From 7bb5b6de6f5d7d0b4971948f649da57f401b24e7 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Fri, 29 Aug 2025 13:45:55 -0700 Subject: [PATCH 001/104] wip evaluation: custom nat evaluation for simple accuracy, with custom input parsing for readability (see readme) --- README.md | 47 ++++- src/vuln_analysis/configs/config-eval.yml | 158 +++++++++++++++++ .../eval-morpheus_23.11-runtime.json | 101 +++++++++++ src/vuln_analysis/functions/custom_input.py | 164 ++++++++++++++++++ .../functions/simple_accuracy.py | 90 ++++++++++ src/vuln_analysis/register.py | 26 ++- 6 files changed, 581 insertions(+), 5 deletions(-) create mode 100644 src/vuln_analysis/configs/config-eval.yml create mode 100644 src/vuln_analysis/data/eval_datasets/eval-morpheus_23.11-runtime.json create mode 100644 src/vuln_analysis/functions/custom_input.py create mode 100644 src/vuln_analysis/functions/simple_accuracy.py diff --git a/README.md b/README.md index c46ebf764..4ec0eae83 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,8 @@ limitations under the License. - [Customizing the embedding model](#customizing-the-embedding-model) - [Steps to configure an alternate embedding provider](#steps-to-configure-an-alternate-embedding-provider) - [Customizing the Output](#customizing-the-output) +- [Evaluation](#evaluation) + - [Creating Evaluation Datasets](#creating-evaluation-datasets) - [Troubleshooting](#troubleshooting) - [Git LFS issues](#git-lfs-issues) - [Container build issues](#container-build-issues) @@ -534,10 +536,6 @@ The configuration defines how the workflow operates, including functions, LLMs, - `base_git_dir`: The directory for storing pulled git repositories used for code analysis. Default is `.cache/am_cache/git`. - `base_code_index_dir`: The directory used for storing code index files. Default is `./cache/am_cache/code_index`. - `cve_fetch_intel`: Fetches details about CVEs from NIST and CVE Details websites. - - `max_retries`: Maximum number of retries on client and server errors. Default is 10. - - `retry_on_client_errors`: Whether to retry on client errors. Default is true. - - `request_timeout`: Timeout for individual HTTP requests in seconds. Default is 300. - - `intel_source_timeout`: Timeout for each intel source (across all retries) in seconds. None means no timeout. Default is None. - `cve_process_sbom`: Prepares and validates input SBOM. - `cve_check_vuln_deps`: Cross-references every entry in the SBOM for known vulnerabilities. If none are found, the agent is skipped for that CVE. - `skip`: If true, skips this check and runs all CVEs through the agent. Useful when the input CVEs to the workflow have been validated by a dedicated vulnerability scanner. Default is false. @@ -755,6 +753,47 @@ The workflow can then be updated to use the new function: Additional output options will be added in the future. +## Evaluation + +Run `nat eval` using the `config-eval.yml` configuration to get an accuracy score between 0 to 1. + +``` +nat eval --config_file=configs/config-eval.yml +``` + +Ground truth datasets should be entered as json files in `data/eval_datasets/`, where each file corresponds to a different image. For each image, different evaluation runs can be defined as workflows; each workflow lists the CVEs that evaluation run will test. + +In `config-eval.yml`, an example evaluation workflow has been set up to run on the morpheus_23.11-runtime, as defined in `data/eval_datasets/eval-morpheus_23.11-runtime.json`, using workflow id `workflow0` which tests five CVEs. + +### Creating Evaluation Datasets +To run evaluation on a another image, a new json file should be created using the following format: +``` +{ + "container_image": { + "name": "nvcr.io/my/image", + "tag": "my-tag", + "source_info": [], #include source info + "sbom_info": {} #include sbom info + }, + "workflows": { + "my_workflow": { + "description": "My workflow description", + "ground_truth": [ + { + "vuln_id": "CVE-0000-00001", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + } # list other CVEs + ] + } # create other workflows + } +} +``` +In `config-eval.yml`, the following should be configured: +- `file_path`: Evaluation dataset specifying image metadata and workflow with CVEs +- `kwargs: workflow_id`: The specific workflow to run +- `output_dir`: Evaluation output file path + ## Troubleshooting Several common issues can arise when running the workflow. Here are some common issues and their solutions. diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml new file mode 100644 index 000000000..7ddd239f7 --- /dev/null +++ b/src/vuln_analysis/configs/config-eval.yml @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +general: + use_uvloop: true + +functions: + cve_generate_vdbs: + _type: cve_generate_vdbs + agent_name: cve_agent_executor # Used to determine which tools are enabled + embedder_name: nim_embedder + base_git_dir: .cache/am_cache/git + base_vdb_dir: .cache/am_cache/vdb + base_code_index_dir: .cache/am_cache/code_index + cve_fetch_intel: + _type: cve_fetch_intel + cve_process_sbom: + _type: cve_process_sbom + cve_check_vuln_deps : + _type: cve_check_vuln_deps + skip: false + cve_checklist: + _type: cve_checklist + llm_name: checklist_llm + Container Image Code QA System: + _type: local_vdb_retriever + embedder_name: nim_embedder + llm_name: code_vdb_retriever_llm + vdb_type: code + return_source_documents: false + Container Image Developer Guide QA System: + _type: local_vdb_retriever + embedder_name: nim_embedder + llm_name: doc_vdb_retriever_llm + vdb_type: doc + return_source_documents: false + Lexical Search Container Image Code QA System: + _type: lexical_code_search + top_k: 5 + Internet Search: + _type: serp_wrapper + max_retries: 5 + cve_agent_executor: + _type: cve_agent_executor + llm_name: cve_agent_executor_llm + tool_names: + - Container Image Code QA System + - Container Image Developer Guide QA System + # - Lexical Search Container Image Code QA System # Uncomment to enable lexical search + - Internet Search + max_concurrency: null + max_iterations: 10 + prompt_examples: false + replace_exceptions: true + replace_exceptions_value: "I do not have a definitive answer for this checklist item." + return_intermediate_steps: false + verbose: false + cve_summarize: + _type: cve_summarize + llm_name: summarize_llm + cve_justify: + _type: cve_justify + llm_name: justify_llm + cve_file_output: + _type: cve_file_output + file_path: .tmp/output.json + markdown_dir: .tmp/vulnerability_markdown_reports + overwrite: true + +llms: + checklist_llm: + _type: nim + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CHECKLIST_MODEL_NAME:-nvdev/meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + code_vdb_retriever_llm: + _type: nim + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CODE_VDB_RETRIEVER_MODEL_NAME:-nvdev/meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + doc_vdb_retriever_llm: + _type: nim + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${DOC_VDB_RETRIEVER_MODEL_NAME:-nvdev/meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + cve_agent_executor_llm: + _type: nim + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CVE_AGENT_EXECUTOR_MODEL_NAME:-nvdev/meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + summarize_llm: + _type: nim + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${SUMMARIZE_MODEL_NAME:-nvdev/meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + justify_llm: + _type: nim + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${JUSTIFY_MODEL_NAME:-nvdev/meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + +embedders: + nim_embedder: + _type: nim + base_url: ${NIM_EMBED_BASE_URL:-https://integrate.api.nvidia.com/v1} + model_name: ${EMBEDDER_MODEL_NAME:-nvdev/nvidia/nv-embedqa-e5-v5} + truncate: END + max_batch_size: 128 + +workflow: + _type: cve_agent + cve_generate_vdbs_name: cve_generate_vdbs + cve_fetch_intel_name: cve_fetch_intel + cve_process_sbom_name: cve_process_sbom + cve_check_vuln_deps_name: cve_check_vuln_deps + cve_checklist_name: cve_checklist + cve_agent_executor_name: cve_agent_executor + cve_summarize_name: cve_summarize + cve_justify_name: cve_justify + cve_output_config_name: cve_file_output + +eval: + general: + output_dir: ./.tmp/eval/custom_eval + dataset: + _type: custom + file_path: data/eval_datasets/eval-morpheus_23.11-runtime.json + function: vuln_analysis.functions.custom_input.convert_better_config_to_eval_input + kwargs: + workflow_id: "workflow0" #workflow0 produces 0.4 accuracy, workflow1 produces 1.0 accuracy + + evaluators: + accuracy_eval: + _type: accuracy \ No newline at end of file diff --git a/src/vuln_analysis/data/eval_datasets/eval-morpheus_23.11-runtime.json b/src/vuln_analysis/data/eval_datasets/eval-morpheus_23.11-runtime.json new file mode 100644 index 000000000..562a0e46f --- /dev/null +++ b/src/vuln_analysis/data/eval_datasets/eval-morpheus_23.11-runtime.json @@ -0,0 +1,101 @@ +{ + "container_image": { + "name": "nvcr.io/nvidia/morpheus/morpheus", + "tag": "23.11-runtime", + "source_info": [ + { + "type": "code", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "ref": "branch-23.11", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ], + "exclude": [ + "tests/**/*" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "ref": "branch-23.11", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + } + ], + "sbom_info": { + "_type": "file", + "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom" + } + }, + "workflows": { + "workflow0": { + "description": "Real ground truth dataset, yields 0.4 accuracy", + "ground_truth": [ + { + "vuln_id": "GHSA-3f63-hfp8-52jq", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + }, + { + "vuln_id": "CVE-2023-36632", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "GHSA-cxfr-5q3r-2rc2", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "GHSA-554w-xh4j-8w64", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "CVE-2023-51767", + "status": "AFFECTED", + "label": "vulnerable" + } + ] + }, + "workflow1": { + "description": "Toy dataset for testing, yields 1.0 accuracy", + "ground_truth": [ + { + "vuln_id": "GHSA-3f63-hfp8-52jq", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + }, + { + "vuln_id": "CVE-2023-36632", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + }, + { + "vuln_id": "GHSA-cxfr-5q3r-2rc2", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "GHSA-554w-xh4j-8w64", + "status": "UNKNOWN", + "label": "unknown" + }, + { + "vuln_id": "CVE-2023-51767", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + } + ] + } + } + } \ No newline at end of file diff --git a/src/vuln_analysis/functions/custom_input.py b/src/vuln_analysis/functions/custom_input.py new file mode 100644 index 000000000..305138283 --- /dev/null +++ b/src/vuln_analysis/functions/custom_input.py @@ -0,0 +1,164 @@ +import json +from pathlib import Path + +from nat.eval.evaluator.evaluator_model import EvalInput +from nat.eval.evaluator.evaluator_model import EvalInputItem + + +def convert_better_config_to_eval_input(file_path: Path, workflow_id: str | None = None, max_workflows: int | None = None) -> EvalInput: + """ + To run just this script outside of the nat pipeline: python3 custom_input.py custom_config.json --workflow-ids workflow0 --preview -o my_eval.json + Note: need to run inside vuln_analysis docker container where nat is installed + """ + # Load the container image config JSON + with open(file_path, 'r', encoding='utf-8') as f: + config_data = json.load(f) + + # Extract container image and workflows + container_image = config_data.get('container_image', {}) + workflows = config_data.get('workflows', {}) + + if not container_image: + print("No container_image found in config file") + return EvalInput(eval_input_items=[]) + + if not workflows: + print("No workflows found in config file") + return EvalInput(eval_input_items=[]) + + # Filter by workflow ID if specified + if workflow_id is not None: + if workflow_id not in workflows: + print(f"No workflow found with ID: {workflow_id}") + print(f"Available workflows: {list(workflows.keys())}") + return EvalInput(eval_input_items=[]) + + target_workflow = workflows[workflow_id] + print(f"Processing workflow: {workflow_id}") + else: + # Use first workflow if no specific ID specified + workflow_id = list(workflows.keys())[0] + target_workflow = workflows[workflow_id] + print(f"No workflow ID specified, using first available: {workflow_id}") + + # Process the workflow + ground_truth = target_workflow.get('ground_truth', []) + + # Create scan structure with vulnerability IDs from ground truth + scan_vulns = [] + ground_truth_mapping = {} + + for gt_item in ground_truth: + vuln_id = gt_item.get('vuln_id') + status = gt_item.get('status') + + if vuln_id and status: + # Add to scan structure + scan_vulns.append({"vuln_id": vuln_id}) + + # Add to ground truth mapping + ground_truth_mapping[vuln_id] = status + + # Create the question structure (container_image + scan) + question_structure = { + "image": container_image, + "scan": { + "vulns": scan_vulns + } + } + + # Convert to stringified format + question_string = json.dumps(question_structure, separators=(',', ':')) + answer_string = json.dumps(ground_truth_mapping, separators=(',', ':')) + + # Create evaluation item + eval_item = EvalInputItem( + id=0, + input_obj=question_string, + expected_output_obj=answer_string, + full_dataset_entry={ + "id": 0, + "question": question_string, + "answer": answer_string, + "workflow_id": workflow_id, + "original_workflow": target_workflow + } + ) + + return EvalInput(eval_input_items=[eval_item]) + + +# For if you want to look at the eval input string +def save_as_baseline_string_format(eval_input: EvalInput, output_path: Path) -> None: + baseline_data = [] + + for item in eval_input.eval_input_items: + baseline_item = { + "id": item.id, + "question": item.input_obj, + "answer": item.expected_output_obj + } + baseline_data.append(baseline_item) + + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(baseline_data, f, indent=4) + + print(f"Saved baseline string format to: {output_path}") + + +def main(): + import argparse + + parser = argparse.ArgumentParser( + description='Convert better_config.json to evaluation format', + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument('input_file', help='Path to better_config.json file') + parser.add_argument('-o', '--output', help='Output path for baseline string format (optional)') + parser.add_argument('--workflow-id', help='Specific workflow ID to process (e.g., --workflow-id workflow0)') + parser.add_argument('--max-workflows', type=int, help='Maximum number of workflows to process') + parser.add_argument('--preview', action='store_true', help='Show preview of first converted item') + + args = parser.parse_args() + + try: + input_path = Path(args.input_file) + + # Convert to EvalInput format + eval_input = convert_better_config_to_eval_input(input_path, args.workflow_id, args.max_workflows) + + if args.preview and eval_input.eval_input_items: + first_item = eval_input.eval_input_items[0] + print(f"ID: {first_item.id}") + print(f"\nQuestion (first 200 chars):") + print(f"{first_item.input_obj[:200]}...") + print(f"\nAnswer:") + print(f"{first_item.expected_output_obj}") + print("="*80) + + # Save as baseline string format if output specified + if args.output: + output_path = Path(args.output) + save_as_baseline_string_format(eval_input, output_path) + else: + # Default output path in eval directory + eval_dir = Path(__file__).parent + default_output = eval_dir / "converted_baseline_string.json" + save_as_baseline_string_format(eval_input, default_output) + + + except FileNotFoundError as e: + print(f"Error: Input file not found - {e}") + return 1 + except json.JSONDecodeError as e: + print(f"Error: Invalid JSON in input file - {e}") + return 1 + except Exception as e: + print(f"Unexpected error: {e}") + return 1 + + return 0 + + +if __name__ == "__main__": + exit(main()) \ No newline at end of file diff --git a/src/vuln_analysis/functions/simple_accuracy.py b/src/vuln_analysis/functions/simple_accuracy.py new file mode 100644 index 000000000..9e8524a86 --- /dev/null +++ b/src/vuln_analysis/functions/simple_accuracy.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +import asyncio +import json + +from tqdm import tqdm + +from aiq.eval.evaluator.evaluator_model import EvalInput +from aiq.eval.evaluator.evaluator_model import EvalOutput +from aiq.eval.evaluator.evaluator_model import EvalOutputItem +from aiq.eval.evaluator.evaluator_model import EvalInputItem +from aiq.eval.utils.tqdm_position_registry import TqdmPositionRegistry + + +_LABEL_MAP = {'TRUE': 'AFFECTED', 'FALSE': 'NOT AFFECTED', 'UNKNOWN': 'UNKNOWN'} + + +class SimpleAccuracyEvaluator: + '''Simple accuracy evaluator class''' + + def __init__(self, max_concurrency: int): + self.max_concurrency = max_concurrency + self.semaphore = asyncio.Semaphore(self.max_concurrency) + + async def evaluate(self, eval_input: EvalInput) -> EvalOutput: + '''Evaluate accuracy funciont''' + + async def process_item(item): + """Compute simple accuracy for an individual item""" + ground_truth_map = eval(item.expected_output_obj) + pipeline_result = json.loads(item.output_obj)['output'] + + accuracies = [] + generated_answers = [] + question = [] + for result in pipeline_result: + vuln_id = result['vuln_id'] + question.append(vuln_id) + affected_status = result.get('justification', {}).get('status', 'MISSING') + status_result = _LABEL_MAP.get(affected_status, 'MISSING') + generated_answers.append(status_result) + + # ORIGINAL LOGIC - ground truth MISSING factors into accuracy calculation + # accuracies.append(float(status_result == ground_truth_map.get(vuln_id, 'MISSING'))) + + # NEW LOGIC - Only include in accuracy calculation if ground truth exists (not MISSING) + ground_truth_status = ground_truth_map.get(vuln_id) + if ground_truth_status is not None: + accuracies.append(float(status_result == ground_truth_status)) + + reasoning = { + "question": question, + "answer": [ground_truth_map.get(q, 'MISSING') for q in question], + "generated_answer": generated_answers, + } + + return sum(accuracies)/len(accuracies), reasoning + + async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: + async with self.semaphore: + result = await process_item(item) + pbar.update(1) + return result + + try: + tqdm_position = TqdmPositionRegistry.claim() + pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Accuracy", position=tqdm_position) + results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) + finally: + pbar.close() + TqdmPositionRegistry.release(tqdm_position) + + sample_scores, sample_reasonings = zip(*results) if results else ([], []) + + avg_score = round(sum(sample_scores) / len(sample_scores), 2) if sample_scores else 0.0 + + eval_output_items = [ + EvalOutputItem(id=item.id, score=score, reasoning=reasoning) + for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings) + ] + + return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) \ No newline at end of file diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index 4b093db0a..9981e75c0 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -46,6 +46,14 @@ from vuln_analysis.utils.llm_engine_utils import postprocess_engine_output from vuln_analysis.utils.llm_engine_utils import preprocess_engine_input + +# Register evaluator helper functions +from aiq.builder.builder import EvalBuilder +from aiq.builder.evaluator import EvaluatorInfo +from aiq.cli.register_workflow import register_evaluator +from aiq.data_models.evaluator import EvaluatorBaseConfig + + logger = logging.getLogger(__name__) @@ -69,6 +77,22 @@ class CVEAgentWorkflowConfig(FunctionBaseConfig, name="cve_agent"): description="Workflow function description") +#evaluator registration (actual code is in functions/simple_accuracy.py) +class SimpleAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="accuracy"): + '''Configuration for custom accuracy evaluator''' + pass + + +@register_evaluator(config_type=SimpleAccuracyEvaluatorConfig) +async def simple_accuracy_evaluator(config: SimpleAccuracyEvaluatorConfig, builder: EvalBuilder): + '''Register simple accuracy evaluator''' + from .functions.simple_accuracy import SimpleAccuracyEvaluator + evaluator = SimpleAccuracyEvaluator(builder.get_max_concurrency()) + + yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Simple Accuracy Evaluator") + + + @register_function(config_type=CVEAgentWorkflowConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def cve_agent_workflow(config: CVEAgentWorkflowConfig, builder: Builder): @@ -232,4 +256,4 @@ async def _response_fn(input_message: AgentMorpheusInput) -> AgentMorpheusOutput except GeneratorExit: logger.info("Workflow exited early!") finally: - logger.info("Cleaning up cve-agent workflow.") + logger.info("Cleaning up cve-agent workflow.") \ No newline at end of file From bf613be8edef8211a2d5be2ca61adcab53d549b6 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Fri, 29 Aug 2025 14:57:44 -0700 Subject: [PATCH 002/104] putting back error handling info in readme that got deleted by accident --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 4ec0eae83..0a3f4fd0c 100644 --- a/README.md +++ b/README.md @@ -536,6 +536,10 @@ The configuration defines how the workflow operates, including functions, LLMs, - `base_git_dir`: The directory for storing pulled git repositories used for code analysis. Default is `.cache/am_cache/git`. - `base_code_index_dir`: The directory used for storing code index files. Default is `./cache/am_cache/code_index`. - `cve_fetch_intel`: Fetches details about CVEs from NIST and CVE Details websites. + - `max_retries`: Maximum number of retries on client and server errors. Default is 10. + - `retry_on_client_errors`: Whether to retry on client errors. Default is true. + - `request_timeout`: Timeout for individual HTTP requests in seconds. Default is 300. + - `intel_source_timeout`: Timeout for each intel source (across all retries) in seconds. None means no timeout. Default is None. - `cve_process_sbom`: Prepares and validates input SBOM. - `cve_check_vuln_deps`: Cross-references every entry in the SBOM for known vulnerabilities. If none are found, the agent is skipped for that CVE. - `skip`: If true, skips this check and runs all CVEs through the agent. Useful when the input CVEs to the workflow have been validated by a dedicated vulnerability scanner. Default is false. From 3c66a49987c58725cb9667dd118a1c241f5d9775 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Fri, 29 Aug 2025 15:01:45 -0700 Subject: [PATCH 003/104] removed wrong comment on accuracy numbers --- src/vuln_analysis/configs/config-eval.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml index 7ddd239f7..1f0a54260 100644 --- a/src/vuln_analysis/configs/config-eval.yml +++ b/src/vuln_analysis/configs/config-eval.yml @@ -151,7 +151,7 @@ eval: file_path: data/eval_datasets/eval-morpheus_23.11-runtime.json function: vuln_analysis.functions.custom_input.convert_better_config_to_eval_input kwargs: - workflow_id: "workflow0" #workflow0 produces 0.4 accuracy, workflow1 produces 1.0 accuracy + workflow_id: "workflow0" evaluators: accuracy_eval: From 7ffb912f92efd68f47fbc61f049f17382969c5c6 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Tue, 2 Sep 2025 17:48:02 -0700 Subject: [PATCH 004/104] readme update for running multiple eval runs --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0a3f4fd0c..d447a9540 100644 --- a/README.md +++ b/README.md @@ -794,10 +794,22 @@ To run evaluation on a another image, a new json file should be created using th } ``` In `config-eval.yml`, the following should be configured: -- `file_path`: Evaluation dataset specifying image metadata and workflow with CVEs +- `file_path`: Evaluation dataset specifying image and workflow (test set of CVEs) - `kwargs: workflow_id`: The specific workflow to run - `output_dir`: Evaluation output file path +### Running Multiple Evaluations +Use the `--reps` flag to run evaluation multiple times on the same image and workflow. +``` +nat eval --config_file=configs/config-eval.yml --reps=3 +``` +**WIP / Known issues** +- nginx llm_cache needs to be disabled for runs to have different results. I was commenting out lines in `nginx_cache.conf` and `nginx/templates/routes/*` but I need to find a better way to toggle it off for eval; I think I can make an eval .env file to override the model provider base URLs in docker-compose so they bypass nginx. +- when I disable llm_cache, I get a bunch of "Too many requests" errors even when using nvdev endpoint + + +The output file will show individual accuracy scores for each run, and the average accuracy across all runs. + ## Troubleshooting Several common issues can arise when running the workflow. Here are some common issues and their solutions. From a3a117f70f6687c9d253c343423823de99c8c33b Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Wed, 3 Sep 2025 15:10:52 -0700 Subject: [PATCH 005/104] add to readme how to run multiple reps of eval --- README.md | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d447a9540..914b6a17f 100644 --- a/README.md +++ b/README.md @@ -799,16 +799,37 @@ In `config-eval.yml`, the following should be configured: - `output_dir`: Evaluation output file path ### Running Multiple Evaluations -Use the `--reps` flag to run evaluation multiple times on the same image and workflow. + +Use the `--reps` flag to run evaluation multiple times on the same image and workflow. Note that **caching must be disabled first** (see [Disable Caching](#disable-caching)). + ``` nat eval --config_file=configs/config-eval.yml --reps=3 ``` -**WIP / Known issues** -- nginx llm_cache needs to be disabled for runs to have different results. I was commenting out lines in `nginx_cache.conf` and `nginx/templates/routes/*` but I need to find a better way to toggle it off for eval; I think I can make an eval .env file to override the model provider base URLs in docker-compose so they bypass nginx. -- when I disable llm_cache, I get a bunch of "Too many requests" errors even when using nvdev endpoint +The output file will show individual accuracy scores for each run, as well as the average accuracy across all runs. + +#### Disable Caching +LLM and embedder caching must be disabled for multiple evaluation runs, to prevent getting the same results every time. + +##### Option 1 - Global Override +Globally override all LLM and embedder caching by setting the `NVIDIA_API_BASE` variable to directly hit the model endpoint instead of going through nginx caching. + +``` +NVIDIA_API_BASE=https://integrate.api.nvidia.com/v1 nat eval --config_file=configs/config-eval.yml --reps=3 +``` + +##### Option 2 - Disable Caching on Specific Calls +Users may want to disable caching for only specific parts of the pipeline. For example, more in-depth experiments may require disabling caching on specific LLMs. Or, you may want to disable all LLM call caching but keeping embedder caching so the embedder does not have to regenerate embeddings for the static image source every evaluation run. + +You can accomplish specific caching overrides in `config-eval.yml` by assigning the `base_url` variable of the target component with `"https://integrate.api.nvidia.com/v1"`. Note that you can create multiple config files to keep track of different evaluation experiments. + + +For quick testing, you can also change configuration at runtime using the `--override` flag, which uses dot notation. Below is an example that disables caching on the checklist_llm. + +``` +nat eval --config_file=configs/config-eval.yml --reps=3 --override llms.checklist_llm.base_url https://integrate.api.nvidia.com/v1 +``` -The output file will show individual accuracy scores for each run, and the average accuracy across all runs. ## Troubleshooting From 2fc3535e234bdc73a52d983250b2099d71737fb8 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Wed, 3 Sep 2025 15:27:03 -0700 Subject: [PATCH 006/104] test commit --- src/vuln_analysis/configs/config-eval.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml index 1f0a54260..43d27edd8 100644 --- a/src/vuln_analysis/configs/config-eval.yml +++ b/src/vuln_analysis/configs/config-eval.yml @@ -155,4 +155,4 @@ eval: evaluators: accuracy_eval: - _type: accuracy \ No newline at end of file + _type: accuracy #test \ No newline at end of file From 530f918f5c2deb6fd7167722fd6ea58af7fbc69f Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Wed, 3 Sep 2025 15:27:33 -0700 Subject: [PATCH 007/104] test commit --- src/vuln_analysis/configs/config-eval.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml index 43d27edd8..1f0a54260 100644 --- a/src/vuln_analysis/configs/config-eval.yml +++ b/src/vuln_analysis/configs/config-eval.yml @@ -155,4 +155,4 @@ eval: evaluators: accuracy_eval: - _type: accuracy #test \ No newline at end of file + _type: accuracy \ No newline at end of file From f1d24f1796863e1b0e9a407fe39f28b422ed434e Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Wed, 3 Sep 2025 17:20:48 -0700 Subject: [PATCH 008/104] nat custom evaluator for label accuracy --- src/vuln_analysis/configs/config-eval.yml | 12 +- .../eval-morpheus_23.11-runtime.json | 15 ++ src/vuln_analysis/functions/label_accuracy.py | 88 +++++++++ .../functions/parse_eval_input.py | 169 ++++++++++++++++++ src/vuln_analysis/register.py | 31 +++- 5 files changed, 302 insertions(+), 13 deletions(-) create mode 100644 src/vuln_analysis/functions/label_accuracy.py create mode 100644 src/vuln_analysis/functions/parse_eval_input.py diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml index 1f0a54260..8a62d8a37 100644 --- a/src/vuln_analysis/configs/config-eval.yml +++ b/src/vuln_analysis/configs/config-eval.yml @@ -145,14 +145,16 @@ workflow: eval: general: - output_dir: ./.tmp/eval/custom_eval + output_dir: ./.tmp/eval/my_eval dataset: _type: custom file_path: data/eval_datasets/eval-morpheus_23.11-runtime.json - function: vuln_analysis.functions.custom_input.convert_better_config_to_eval_input + function: vuln_analysis.functions.parse_eval_input.convert_better_config_to_eval_input kwargs: - workflow_id: "workflow0" + workflow_id: "workflow-small" evaluators: - accuracy_eval: - _type: accuracy \ No newline at end of file + status_accuracy_eval: + _type: status_accuracy + label_accuracy_eval: + _type: label_accuracy \ No newline at end of file diff --git a/src/vuln_analysis/data/eval_datasets/eval-morpheus_23.11-runtime.json b/src/vuln_analysis/data/eval_datasets/eval-morpheus_23.11-runtime.json index 562a0e46f..dfb5bbaa4 100644 --- a/src/vuln_analysis/data/eval_datasets/eval-morpheus_23.11-runtime.json +++ b/src/vuln_analysis/data/eval_datasets/eval-morpheus_23.11-runtime.json @@ -96,6 +96,21 @@ "label": "code_not_reachable" } ] + }, + "workflow-small": { + "description": "only 1 cve", + "ground_truth": [ + { + "vuln_id": "GHSA-3f63-hfp8-52jq", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + }, + { + "vuln_id": "CVE-2023-51767", + "status": "AFFECTED", + "label": "vulnerable" + } + ] } } } \ No newline at end of file diff --git a/src/vuln_analysis/functions/label_accuracy.py b/src/vuln_analysis/functions/label_accuracy.py new file mode 100644 index 000000000..5d5f31f14 --- /dev/null +++ b/src/vuln_analysis/functions/label_accuracy.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +import asyncio +import json + +from tqdm import tqdm + +from aiq.eval.evaluator.evaluator_model import EvalInput +from aiq.eval.evaluator.evaluator_model import EvalOutput +from aiq.eval.evaluator.evaluator_model import EvalOutputItem +from aiq.eval.evaluator.evaluator_model import EvalInputItem +from aiq.eval.utils.tqdm_position_registry import TqdmPositionRegistry + + +class LabelAccuracyEvaluator: + '''Label accuracy evaluator class that calculates accuracy of justification labels''' + + def __init__(self, max_concurrency: int): + self.max_concurrency = max_concurrency + self.semaphore = asyncio.Semaphore(self.max_concurrency) + + async def evaluate(self, eval_input: EvalInput) -> EvalOutput: + '''Evaluate label accuracy''' + + async def process_item(item): + """Compute label accuracy for an individual item""" + ground_truth_map = eval(item.expected_output_obj) + pipeline_result = json.loads(item.output_obj)['output'] + + label_accuracies = [] + generated_answers = [] + question = [] + + for result in pipeline_result: + vuln_id = result['vuln_id'] + question.append(vuln_id) + + # Extract label from pipeline result + justification_label = result.get('justification', {}).get('label', 'MISSING') + generated_answers.append(justification_label) + + # Calculate label accuracy + # Ground truth labels are stored as "{vuln_id}_label" in the ground truth map + # This is populated by custom_input.py from the eval dataset JSON files + ground_truth_label = ground_truth_map.get(f"{vuln_id}_label") + if ground_truth_label is not None: + label_accuracies.append(float(justification_label == ground_truth_label)) + + reasoning = { + "question": question, + "answer": [ground_truth_map.get(f"{q}_label", 'MISSING') for q in question], + "generated_answer": generated_answers, + } + + return sum(label_accuracies)/len(label_accuracies) if label_accuracies else 0.0, reasoning + + async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: + async with self.semaphore: + result = await process_item(item) + pbar.update(1) + return result + + try: + tqdm_position = TqdmPositionRegistry.claim() + pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Label Accuracy", position=tqdm_position) + results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) + finally: + pbar.close() + TqdmPositionRegistry.release(tqdm_position) + + sample_scores, sample_reasonings = zip(*results) if results else ([], []) + + avg_score = round(sum(sample_scores) / len(sample_scores), 2) if sample_scores else 0.0 + + eval_output_items = [ + EvalOutputItem(id=item.id, score=score, reasoning=reasoning) + for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings) + ] + + return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) \ No newline at end of file diff --git a/src/vuln_analysis/functions/parse_eval_input.py b/src/vuln_analysis/functions/parse_eval_input.py new file mode 100644 index 000000000..dd71ebb4b --- /dev/null +++ b/src/vuln_analysis/functions/parse_eval_input.py @@ -0,0 +1,169 @@ +import json +from pathlib import Path + +from nat.eval.evaluator.evaluator_model import EvalInput +from nat.eval.evaluator.evaluator_model import EvalInputItem + + +def convert_better_config_to_eval_input(file_path: Path, workflow_id: str | None = None, max_workflows: int | None = None) -> EvalInput: + """ + To run just this script outside of the nat pipeline: python3 custom_input.py custom_config.json --workflow-ids workflow0 --preview -o my_eval.json + Note: need to run inside vuln_analysis docker container where nat is installed + """ + # Load the container image config JSON + with open(file_path, 'r', encoding='utf-8') as f: + config_data = json.load(f) + + # Extract container image and workflows + container_image = config_data.get('container_image', {}) + workflows = config_data.get('workflows', {}) + + if not container_image: + print("No container_image found in config file") + return EvalInput(eval_input_items=[]) + + if not workflows: + print("No workflows found in config file") + return EvalInput(eval_input_items=[]) + + # Filter by workflow ID if specified + if workflow_id is not None: + if workflow_id not in workflows: + print(f"No workflow found with ID: {workflow_id}") + print(f"Available workflows: {list(workflows.keys())}") + return EvalInput(eval_input_items=[]) + + target_workflow = workflows[workflow_id] + print(f"Processing workflow: {workflow_id}") + else: + # Use first workflow if no specific ID specified + workflow_id = list(workflows.keys())[0] + target_workflow = workflows[workflow_id] + print(f"No workflow ID specified, using first available: {workflow_id}") + + # Process the workflow + ground_truth = target_workflow.get('ground_truth', []) + + # Create scan structure with vulnerability IDs from ground truth + scan_vulns = [] + ground_truth_mapping = {} + + for gt_item in ground_truth: + vuln_id = gt_item.get('vuln_id') + status = gt_item.get('status') + label = gt_item.get('label') + + if vuln_id and status: + # Add to scan structure + scan_vulns.append({"vuln_id": vuln_id}) + + # Add to ground truth mapping + ground_truth_mapping[vuln_id] = status + + # Add label to ground truth mapping if available + if label: + ground_truth_mapping[f"{vuln_id}_label"] = label + + # Create the question structure (container_image + scan) + question_structure = { + "image": container_image, + "scan": { + "vulns": scan_vulns + } + } + + # Convert to stringified format + question_string = json.dumps(question_structure, separators=(',', ':')) + answer_string = json.dumps(ground_truth_mapping, separators=(',', ':')) + + # Create evaluation item + eval_item = EvalInputItem( + id=0, + input_obj=question_string, + expected_output_obj=answer_string, + full_dataset_entry={ + "id": 0, + "question": question_string, + "answer": answer_string, + "workflow_id": workflow_id, + "original_workflow": target_workflow + } + ) + + return EvalInput(eval_input_items=[eval_item]) + + +# For if you want to look at the eval input string +def save_as_baseline_string_format(eval_input: EvalInput, output_path: Path) -> None: + baseline_data = [] + + for item in eval_input.eval_input_items: + baseline_item = { + "id": item.id, + "question": item.input_obj, + "answer": item.expected_output_obj + } + baseline_data.append(baseline_item) + + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(baseline_data, f, indent=4) + + print(f"Saved baseline string format to: {output_path}") + + +def main(): + import argparse + + parser = argparse.ArgumentParser( + description='Convert better_config.json to evaluation format', + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument('input_file', help='Path to better_config.json file') + parser.add_argument('-o', '--output', help='Output path for baseline string format (optional)') + parser.add_argument('--workflow-id', help='Specific workflow ID to process (e.g., --workflow-id workflow0)') + parser.add_argument('--max-workflows', type=int, help='Maximum number of workflows to process') + parser.add_argument('--preview', action='store_true', help='Show preview of first converted item') + + args = parser.parse_args() + + try: + input_path = Path(args.input_file) + + # Convert to EvalInput format + eval_input = convert_better_config_to_eval_input(input_path, args.workflow_id, args.max_workflows) + + if args.preview and eval_input.eval_input_items: + first_item = eval_input.eval_input_items[0] + print(f"ID: {first_item.id}") + print(f"\nQuestion (first 200 chars):") + print(f"{first_item.input_obj[:200]}...") + print(f"\nAnswer:") + print(f"{first_item.expected_output_obj}") + print("="*80) + + # Save as baseline string format if output specified + if args.output: + output_path = Path(args.output) + save_as_baseline_string_format(eval_input, output_path) + else: + # Default output path in eval directory + eval_dir = Path(__file__).parent + default_output = eval_dir / "converted_baseline_string.json" + save_as_baseline_string_format(eval_input, default_output) + + + except FileNotFoundError as e: + print(f"Error: Input file not found - {e}") + return 1 + except json.JSONDecodeError as e: + print(f"Error: Invalid JSON in input file - {e}") + return 1 + except Exception as e: + print(f"Unexpected error: {e}") + return 1 + + return 0 + + +if __name__ == "__main__": + exit(main()) \ No newline at end of file diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index 9981e75c0..9e87b1433 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -78,18 +78,33 @@ class CVEAgentWorkflowConfig(FunctionBaseConfig, name="cve_agent"): #evaluator registration (actual code is in functions/simple_accuracy.py) -class SimpleAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="accuracy"): - '''Configuration for custom accuracy evaluator''' +class StatusAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="status_accuracy"): + '''Configuration for status accuracy evaluator''' pass -@register_evaluator(config_type=SimpleAccuracyEvaluatorConfig) -async def simple_accuracy_evaluator(config: SimpleAccuracyEvaluatorConfig, builder: EvalBuilder): - '''Register simple accuracy evaluator''' - from .functions.simple_accuracy import SimpleAccuracyEvaluator - evaluator = SimpleAccuracyEvaluator(builder.get_max_concurrency()) +@register_evaluator(config_type=StatusAccuracyEvaluatorConfig) +async def status_accuracy_evaluator(config: StatusAccuracyEvaluatorConfig, builder: EvalBuilder): + '''Register status accuracy evaluator''' + from .functions.simple_accuracy import StatusAccuracyEvaluator + evaluator = StatusAccuracyEvaluator(builder.get_max_concurrency()) - yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Simple Accuracy Evaluator") + yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Status Accuracy Evaluator") + + +#evaluator registration (actual code is in functions/label_accuracy.py) +class LabelAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="label_accuracy"): + '''Configuration for label accuracy evaluator''' + pass + + +@register_evaluator(config_type=LabelAccuracyEvaluatorConfig) +async def label_accuracy_evaluator(config: LabelAccuracyEvaluatorConfig, builder: EvalBuilder): + '''Register label accuracy evaluator''' + from .functions.label_accuracy import LabelAccuracyEvaluator + evaluator = LabelAccuracyEvaluator(builder.get_max_concurrency()) + + yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Label Accuracy Evaluator") From 9d8a6772859ee15990e78a73adb1034967871fc7 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Wed, 3 Sep 2025 17:22:02 -0700 Subject: [PATCH 009/104] renamed eval custom input parser --- src/vuln_analysis/functions/custom_input.py | 164 -------------------- 1 file changed, 164 deletions(-) delete mode 100644 src/vuln_analysis/functions/custom_input.py diff --git a/src/vuln_analysis/functions/custom_input.py b/src/vuln_analysis/functions/custom_input.py deleted file mode 100644 index 305138283..000000000 --- a/src/vuln_analysis/functions/custom_input.py +++ /dev/null @@ -1,164 +0,0 @@ -import json -from pathlib import Path - -from nat.eval.evaluator.evaluator_model import EvalInput -from nat.eval.evaluator.evaluator_model import EvalInputItem - - -def convert_better_config_to_eval_input(file_path: Path, workflow_id: str | None = None, max_workflows: int | None = None) -> EvalInput: - """ - To run just this script outside of the nat pipeline: python3 custom_input.py custom_config.json --workflow-ids workflow0 --preview -o my_eval.json - Note: need to run inside vuln_analysis docker container where nat is installed - """ - # Load the container image config JSON - with open(file_path, 'r', encoding='utf-8') as f: - config_data = json.load(f) - - # Extract container image and workflows - container_image = config_data.get('container_image', {}) - workflows = config_data.get('workflows', {}) - - if not container_image: - print("No container_image found in config file") - return EvalInput(eval_input_items=[]) - - if not workflows: - print("No workflows found in config file") - return EvalInput(eval_input_items=[]) - - # Filter by workflow ID if specified - if workflow_id is not None: - if workflow_id not in workflows: - print(f"No workflow found with ID: {workflow_id}") - print(f"Available workflows: {list(workflows.keys())}") - return EvalInput(eval_input_items=[]) - - target_workflow = workflows[workflow_id] - print(f"Processing workflow: {workflow_id}") - else: - # Use first workflow if no specific ID specified - workflow_id = list(workflows.keys())[0] - target_workflow = workflows[workflow_id] - print(f"No workflow ID specified, using first available: {workflow_id}") - - # Process the workflow - ground_truth = target_workflow.get('ground_truth', []) - - # Create scan structure with vulnerability IDs from ground truth - scan_vulns = [] - ground_truth_mapping = {} - - for gt_item in ground_truth: - vuln_id = gt_item.get('vuln_id') - status = gt_item.get('status') - - if vuln_id and status: - # Add to scan structure - scan_vulns.append({"vuln_id": vuln_id}) - - # Add to ground truth mapping - ground_truth_mapping[vuln_id] = status - - # Create the question structure (container_image + scan) - question_structure = { - "image": container_image, - "scan": { - "vulns": scan_vulns - } - } - - # Convert to stringified format - question_string = json.dumps(question_structure, separators=(',', ':')) - answer_string = json.dumps(ground_truth_mapping, separators=(',', ':')) - - # Create evaluation item - eval_item = EvalInputItem( - id=0, - input_obj=question_string, - expected_output_obj=answer_string, - full_dataset_entry={ - "id": 0, - "question": question_string, - "answer": answer_string, - "workflow_id": workflow_id, - "original_workflow": target_workflow - } - ) - - return EvalInput(eval_input_items=[eval_item]) - - -# For if you want to look at the eval input string -def save_as_baseline_string_format(eval_input: EvalInput, output_path: Path) -> None: - baseline_data = [] - - for item in eval_input.eval_input_items: - baseline_item = { - "id": item.id, - "question": item.input_obj, - "answer": item.expected_output_obj - } - baseline_data.append(baseline_item) - - with open(output_path, 'w', encoding='utf-8') as f: - json.dump(baseline_data, f, indent=4) - - print(f"Saved baseline string format to: {output_path}") - - -def main(): - import argparse - - parser = argparse.ArgumentParser( - description='Convert better_config.json to evaluation format', - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument('input_file', help='Path to better_config.json file') - parser.add_argument('-o', '--output', help='Output path for baseline string format (optional)') - parser.add_argument('--workflow-id', help='Specific workflow ID to process (e.g., --workflow-id workflow0)') - parser.add_argument('--max-workflows', type=int, help='Maximum number of workflows to process') - parser.add_argument('--preview', action='store_true', help='Show preview of first converted item') - - args = parser.parse_args() - - try: - input_path = Path(args.input_file) - - # Convert to EvalInput format - eval_input = convert_better_config_to_eval_input(input_path, args.workflow_id, args.max_workflows) - - if args.preview and eval_input.eval_input_items: - first_item = eval_input.eval_input_items[0] - print(f"ID: {first_item.id}") - print(f"\nQuestion (first 200 chars):") - print(f"{first_item.input_obj[:200]}...") - print(f"\nAnswer:") - print(f"{first_item.expected_output_obj}") - print("="*80) - - # Save as baseline string format if output specified - if args.output: - output_path = Path(args.output) - save_as_baseline_string_format(eval_input, output_path) - else: - # Default output path in eval directory - eval_dir = Path(__file__).parent - default_output = eval_dir / "converted_baseline_string.json" - save_as_baseline_string_format(eval_input, default_output) - - - except FileNotFoundError as e: - print(f"Error: Input file not found - {e}") - return 1 - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON in input file - {e}") - return 1 - except Exception as e: - print(f"Unexpected error: {e}") - return 1 - - return 0 - - -if __name__ == "__main__": - exit(main()) \ No newline at end of file From e3617d4f06108e8c48ae105cba0f4cc26f73e238 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Wed, 3 Sep 2025 17:27:32 -0700 Subject: [PATCH 010/104] changed status accuracy evaluator name to status_accuracy from simple_accuracy --- ...{simple_accuracy.py => status_accuracy.py} | 25 ++++++++++++------- src/vuln_analysis/register.py | 4 +-- 2 files changed, 18 insertions(+), 11 deletions(-) rename src/vuln_analysis/functions/{simple_accuracy.py => status_accuracy.py} (80%) diff --git a/src/vuln_analysis/functions/simple_accuracy.py b/src/vuln_analysis/functions/status_accuracy.py similarity index 80% rename from src/vuln_analysis/functions/simple_accuracy.py rename to src/vuln_analysis/functions/status_accuracy.py index 9e8524a86..8d7feedfd 100644 --- a/src/vuln_analysis/functions/simple_accuracy.py +++ b/src/vuln_analysis/functions/status_accuracy.py @@ -23,8 +23,8 @@ _LABEL_MAP = {'TRUE': 'AFFECTED', 'FALSE': 'NOT AFFECTED', 'UNKNOWN': 'UNKNOWN'} -class SimpleAccuracyEvaluator: - '''Simple accuracy evaluator class''' +class StatusAccuracyEvaluator: + '''Status accuracy evaluator class that calculates status accuracy (AFFECTED/NOT AFFECTED)''' def __init__(self, max_concurrency: int): self.max_concurrency = max_concurrency @@ -38,15 +38,19 @@ async def process_item(item): ground_truth_map = eval(item.expected_output_obj) pipeline_result = json.loads(item.output_obj)['output'] - accuracies = [] - generated_answers = [] + status_accuracies = [] + generated_status_answers = [] question = [] + for result in pipeline_result: vuln_id = result['vuln_id'] question.append(vuln_id) + + # Extract status from pipeline result affected_status = result.get('justification', {}).get('status', 'MISSING') + status_result = _LABEL_MAP.get(affected_status, 'MISSING') - generated_answers.append(status_result) + generated_status_answers.append(status_result) # ORIGINAL LOGIC - ground truth MISSING factors into accuracy calculation # accuracies.append(float(status_result == ground_truth_map.get(vuln_id, 'MISSING'))) @@ -54,15 +58,18 @@ async def process_item(item): # NEW LOGIC - Only include in accuracy calculation if ground truth exists (not MISSING) ground_truth_status = ground_truth_map.get(vuln_id) if ground_truth_status is not None: - accuracies.append(float(status_result == ground_truth_status)) + status_accuracies.append(float(status_result == ground_truth_status)) + # Calculate average accuracy + status_avg = sum(status_accuracies)/len(status_accuracies) if status_accuracies else 0.0 + reasoning = { "question": question, - "answer": [ground_truth_map.get(q, 'MISSING') for q in question], - "generated_answer": generated_answers, + "status_answer": [ground_truth_map.get(q, 'MISSING') for q in question], + "generated_status_answer": generated_status_answers, } - return sum(accuracies)/len(accuracies), reasoning + return status_avg, reasoning async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: async with self.semaphore: diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index 9e87b1433..42c90f96b 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -77,7 +77,7 @@ class CVEAgentWorkflowConfig(FunctionBaseConfig, name="cve_agent"): description="Workflow function description") -#evaluator registration (actual code is in functions/simple_accuracy.py) +#evaluator registration (actual code is in functions/status_accuracy.py) class StatusAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="status_accuracy"): '''Configuration for status accuracy evaluator''' pass @@ -86,7 +86,7 @@ class StatusAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="status_accuracy") @register_evaluator(config_type=StatusAccuracyEvaluatorConfig) async def status_accuracy_evaluator(config: StatusAccuracyEvaluatorConfig, builder: EvalBuilder): '''Register status accuracy evaluator''' - from .functions.simple_accuracy import StatusAccuracyEvaluator + from .functions.status_accuracy import StatusAccuracyEvaluator evaluator = StatusAccuracyEvaluator(builder.get_max_concurrency()) yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Status Accuracy Evaluator") From 57639473d01aa2463e55572b6306a8ea7f3d69f4 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Thu, 4 Sep 2025 14:49:08 -0700 Subject: [PATCH 011/104] function renames + update readme with label accuracy evaluator --- README.md | 30 ++++++++---- src/vuln_analysis/configs/config-eval.yml | 4 +- .../eval-morpheus_23.11-runtime.json | 8 ++-- src/vuln_analysis/functions/label_accuracy.py | 16 +++---- .../functions/parse_eval_input.py | 48 +++++++++---------- .../functions/status_accuracy.py | 10 ++-- 6 files changed, 63 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 914b6a17f..ec114feda 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,8 @@ limitations under the License. - [Steps to configure an alternate embedding provider](#steps-to-configure-an-alternate-embedding-provider) - [Customizing the Output](#customizing-the-output) - [Evaluation](#evaluation) - - [Creating Evaluation Datasets](#creating-evaluation-datasets) + - [Inputting Evaluation Test Sets](#inputting-evaluation-test-sets) + - [Running Multiple Evaluations](#running-multiple-evaluations) - [Troubleshooting](#troubleshooting) - [Git LFS issues](#git-lfs-issues) - [Container build issues](#container-build-issues) @@ -758,19 +759,28 @@ The workflow can then be updated to use the new function: Additional output options will be added in the future. ## Evaluation +In this simple evaluation workflow, we evaluate the pipeline's ability to identify vulnerability **status** and **label**. -Run `nat eval` using the `config-eval.yml` configuration to get an accuracy score between 0 to 1. +Each CVE has one ground truth status and one corresponding label. \\ +Valid statuses include: `AFFECTED` or `NOT AFFECTED`. \\ +Valid labels include: `vulnerable` if the CVE status is `AFFECTED`, or one of 10 VEX statuses if the status is `NOT AFFECTED` (see **Justification Assignment** under the [Key Components section](#key-components)) + +To kick off one pipeline run and ensuing evaluation, run `nat eval` using the `config-eval.yml` configuration. ``` nat eval --config_file=configs/config-eval.yml ``` -Ground truth datasets should be entered as json files in `data/eval_datasets/`, where each file corresponds to a different image. For each image, different evaluation runs can be defined as workflows; each workflow lists the CVEs that evaluation run will test. +This will write the files `status_accuracy_eval_output.json` and `label_accuracy_eval_output.json` to the configured output directory (`.tmp/eval/my_eval` in this example), containing accuracy scores for status and label respectively. Scores range from 0 to 1, with 0 indicating no CVEs were categorized correctly, and 1 indicating all CVEs were categorized correctly. + +### Inputting Evaluation Test Sets + +Ground truth datasets should be put in the `data/eval_datasets/` directory, in json format. Each json file corresponds to a different image. Additionally, for each image, different evaluation experiments can be defined as test sets where each test set lists the CVEs to be tested. + +In `config-eval.yml`, an example evaluation experiment has been set up on the morpheus_23.11-runtime image, as defined in `data/eval_datasets/eval-morpheus_23.11-runtime.json`, using test set id `test-set0`. -In `config-eval.yml`, an example evaluation workflow has been set up to run on the morpheus_23.11-runtime, as defined in `data/eval_datasets/eval-morpheus_23.11-runtime.json`, using workflow id `workflow0` which tests five CVEs. +To run evaluation on your image, a new json file should be created. The file should follow the following format: -### Creating Evaluation Datasets -To run evaluation on a another image, a new json file should be created using the following format: ``` { "container_image": { @@ -779,8 +789,8 @@ To run evaluation on a another image, a new json file should be created using th "source_info": [], #include source info "sbom_info": {} #include sbom info }, - "workflows": { - "my_workflow": { + "test_sets": { + "my_test_set": { "description": "My workflow description", "ground_truth": [ { @@ -794,8 +804,8 @@ To run evaluation on a another image, a new json file should be created using th } ``` In `config-eval.yml`, the following should be configured: -- `file_path`: Evaluation dataset specifying image and workflow (test set of CVEs) -- `kwargs: workflow_id`: The specific workflow to run +- `file_path`: Evaluation dataset specifying image and test set +- `kwargs: test_set_id`: The specific test set to run - `output_dir`: Evaluation output file path ### Running Multiple Evaluations diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml index 8a62d8a37..afbc953d9 100644 --- a/src/vuln_analysis/configs/config-eval.yml +++ b/src/vuln_analysis/configs/config-eval.yml @@ -149,9 +149,9 @@ eval: dataset: _type: custom file_path: data/eval_datasets/eval-morpheus_23.11-runtime.json - function: vuln_analysis.functions.parse_eval_input.convert_better_config_to_eval_input + function: vuln_analysis.functions.parse_eval_input.parse_input kwargs: - workflow_id: "workflow-small" + test_set_id: "test-set-small" evaluators: status_accuracy_eval: diff --git a/src/vuln_analysis/data/eval_datasets/eval-morpheus_23.11-runtime.json b/src/vuln_analysis/data/eval_datasets/eval-morpheus_23.11-runtime.json index dfb5bbaa4..27ea4ea04 100644 --- a/src/vuln_analysis/data/eval_datasets/eval-morpheus_23.11-runtime.json +++ b/src/vuln_analysis/data/eval_datasets/eval-morpheus_23.11-runtime.json @@ -36,8 +36,8 @@ "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom" } }, - "workflows": { - "workflow0": { + "test_set": { + "test-set0": { "description": "Real ground truth dataset, yields 0.4 accuracy", "ground_truth": [ { @@ -67,7 +67,7 @@ } ] }, - "workflow1": { + "test-set1": { "description": "Toy dataset for testing, yields 1.0 accuracy", "ground_truth": [ { @@ -97,7 +97,7 @@ } ] }, - "workflow-small": { + "test-set-small": { "description": "only 1 cve", "ground_truth": [ { diff --git a/src/vuln_analysis/functions/label_accuracy.py b/src/vuln_analysis/functions/label_accuracy.py index 5d5f31f14..c1abf0b0a 100644 --- a/src/vuln_analysis/functions/label_accuracy.py +++ b/src/vuln_analysis/functions/label_accuracy.py @@ -13,15 +13,15 @@ from tqdm import tqdm -from aiq.eval.evaluator.evaluator_model import EvalInput -from aiq.eval.evaluator.evaluator_model import EvalOutput -from aiq.eval.evaluator.evaluator_model import EvalOutputItem -from aiq.eval.evaluator.evaluator_model import EvalInputItem -from aiq.eval.utils.tqdm_position_registry import TqdmPositionRegistry +from nat.eval.evaluator.evaluator_model import EvalInput +from nat.eval.evaluator.evaluator_model import EvalOutput +from nat.eval.evaluator.evaluator_model import EvalOutputItem +from nat.eval.evaluator.evaluator_model import EvalInputItem +from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry class LabelAccuracyEvaluator: - '''Label accuracy evaluator class that calculates accuracy of justification labels''' + '''Evaluator class for justification labels accuracy''' def __init__(self, max_concurrency: int): self.max_concurrency = max_concurrency @@ -31,7 +31,7 @@ async def evaluate(self, eval_input: EvalInput) -> EvalOutput: '''Evaluate label accuracy''' async def process_item(item): - """Compute label accuracy for an individual item""" + """Compute simple accuracy for an individual item""" ground_truth_map = eval(item.expected_output_obj) pipeline_result = json.loads(item.output_obj)['output'] @@ -49,7 +49,7 @@ async def process_item(item): # Calculate label accuracy # Ground truth labels are stored as "{vuln_id}_label" in the ground truth map - # This is populated by custom_input.py from the eval dataset JSON files + # This is populated by parse_eval_input.py from the eval dataset JSON files ground_truth_label = ground_truth_map.get(f"{vuln_id}_label") if ground_truth_label is not None: label_accuracies.append(float(justification_label == ground_truth_label)) diff --git a/src/vuln_analysis/functions/parse_eval_input.py b/src/vuln_analysis/functions/parse_eval_input.py index dd71ebb4b..74e280140 100644 --- a/src/vuln_analysis/functions/parse_eval_input.py +++ b/src/vuln_analysis/functions/parse_eval_input.py @@ -5,44 +5,44 @@ from nat.eval.evaluator.evaluator_model import EvalInputItem -def convert_better_config_to_eval_input(file_path: Path, workflow_id: str | None = None, max_workflows: int | None = None) -> EvalInput: +def parse_input(file_path: Path, test_set_id: str | None = None, max_test_sets: int | None = None) -> EvalInput: """ - To run just this script outside of the nat pipeline: python3 custom_input.py custom_config.json --workflow-ids workflow0 --preview -o my_eval.json + To run just this script outside of the nat pipeline: python3 parse_eval_input.py custom_config.json --test-set-id workflow0 --preview -o my_eval.json Note: need to run inside vuln_analysis docker container where nat is installed """ # Load the container image config JSON with open(file_path, 'r', encoding='utf-8') as f: config_data = json.load(f) - # Extract container image and workflows + # Extract container image and test sets container_image = config_data.get('container_image', {}) - workflows = config_data.get('workflows', {}) + test_sets = config_data.get('test_set', {}) if not container_image: print("No container_image found in config file") return EvalInput(eval_input_items=[]) - if not workflows: - print("No workflows found in config file") + if not test_sets: + print("No test sets found in config file") return EvalInput(eval_input_items=[]) - # Filter by workflow ID if specified - if workflow_id is not None: - if workflow_id not in workflows: - print(f"No workflow found with ID: {workflow_id}") - print(f"Available workflows: {list(workflows.keys())}") + # Filter by test set ID if specified + if test_set_id is not None: + if test_set_id not in test_sets: + print(f"No test set found with ID: {test_set_id}") + print(f"Available test sets: {list(test_sets.keys())}") return EvalInput(eval_input_items=[]) - target_workflow = workflows[workflow_id] - print(f"Processing workflow: {workflow_id}") + target_test_set = test_sets[test_set_id] + print(f"Processing test set: {test_set_id}") else: - # Use first workflow if no specific ID specified - workflow_id = list(workflows.keys())[0] - target_workflow = workflows[workflow_id] - print(f"No workflow ID specified, using first available: {workflow_id}") + # Use first test set if no specific ID specified + test_set_id = list(test_sets.keys())[0] + target_test_set = test_sets[test_set_id] + print(f"No test set ID specified, using first available: {test_set_id}") - # Process the workflow - ground_truth = target_workflow.get('ground_truth', []) + # Process the test set + ground_truth = target_test_set.get('ground_truth', []) # Create scan structure with vulnerability IDs from ground truth scan_vulns = [] @@ -85,8 +85,8 @@ def convert_better_config_to_eval_input(file_path: Path, workflow_id: str | None "id": 0, "question": question_string, "answer": answer_string, - "workflow_id": workflow_id, - "original_workflow": target_workflow + "test_set_id": test_set_id, + "original_test_set": target_test_set } ) @@ -120,8 +120,8 @@ def main(): ) parser.add_argument('input_file', help='Path to better_config.json file') parser.add_argument('-o', '--output', help='Output path for baseline string format (optional)') - parser.add_argument('--workflow-id', help='Specific workflow ID to process (e.g., --workflow-id workflow0)') - parser.add_argument('--max-workflows', type=int, help='Maximum number of workflows to process') + parser.add_argument('--test-set-id', help='Specific test set ID to process (e.g., --test-set-id workflow0)') + parser.add_argument('--max-test-sets', type=int, help='Maximum number of test sets to process') parser.add_argument('--preview', action='store_true', help='Show preview of first converted item') args = parser.parse_args() @@ -130,7 +130,7 @@ def main(): input_path = Path(args.input_file) # Convert to EvalInput format - eval_input = convert_better_config_to_eval_input(input_path, args.workflow_id, args.max_workflows) + eval_input = parse_input(input_path, args.test_set_id, args.max_test_sets) if args.preview and eval_input.eval_input_items: first_item = eval_input.eval_input_items[0] diff --git a/src/vuln_analysis/functions/status_accuracy.py b/src/vuln_analysis/functions/status_accuracy.py index 8d7feedfd..712c4cf8b 100644 --- a/src/vuln_analysis/functions/status_accuracy.py +++ b/src/vuln_analysis/functions/status_accuracy.py @@ -13,11 +13,11 @@ from tqdm import tqdm -from aiq.eval.evaluator.evaluator_model import EvalInput -from aiq.eval.evaluator.evaluator_model import EvalOutput -from aiq.eval.evaluator.evaluator_model import EvalOutputItem -from aiq.eval.evaluator.evaluator_model import EvalInputItem -from aiq.eval.utils.tqdm_position_registry import TqdmPositionRegistry +from nat.eval.evaluator.evaluator_model import EvalInput +from nat.eval.evaluator.evaluator_model import EvalOutput +from nat.eval.evaluator.evaluator_model import EvalOutputItem +from nat.eval.evaluator.evaluator_model import EvalInputItem +from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry _LABEL_MAP = {'TRUE': 'AFFECTED', 'FALSE': 'NOT AFFECTED', 'UNKNOWN': 'UNKNOWN'} From 22c09106c77df1884a8f3be6061e4e2c3042b773 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Thu, 4 Sep 2025 14:51:34 -0700 Subject: [PATCH 012/104] readme typo --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ec114feda..8d581c5fb 100644 --- a/README.md +++ b/README.md @@ -761,8 +761,8 @@ Additional output options will be added in the future. ## Evaluation In this simple evaluation workflow, we evaluate the pipeline's ability to identify vulnerability **status** and **label**. -Each CVE has one ground truth status and one corresponding label. \\ -Valid statuses include: `AFFECTED` or `NOT AFFECTED`. \\ +Each CVE has one ground truth status and one corresponding label. +Valid statuses include: `AFFECTED` or `NOT AFFECTED`. Valid labels include: `vulnerable` if the CVE status is `AFFECTED`, or one of 10 VEX statuses if the status is `NOT AFFECTED` (see **Justification Assignment** under the [Key Components section](#key-components)) To kick off one pipeline run and ensuing evaluation, run `nat eval` using the `config-eval.yml` configuration. From 77639c74b14afe8145386e5a01e7e0b279910524 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Thu, 4 Sep 2025 14:54:40 -0700 Subject: [PATCH 013/104] more README typos --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8d581c5fb..b46194c43 100644 --- a/README.md +++ b/README.md @@ -763,9 +763,9 @@ In this simple evaluation workflow, we evaluate the pipeline's ability to identi Each CVE has one ground truth status and one corresponding label. Valid statuses include: `AFFECTED` or `NOT AFFECTED`. -Valid labels include: `vulnerable` if the CVE status is `AFFECTED`, or one of 10 VEX statuses if the status is `NOT AFFECTED` (see **Justification Assignment** under the [Key Components section](#key-components)) +Valid labels include: `vulnerable` if the CVE status is `AFFECTED`, or one of 10 VEX statuses if the status is `NOT AFFECTED` (see **Justification Assignment** under the [Key Components section](#key-components)). -To kick off one pipeline run and ensuing evaluation, run `nat eval` using the `config-eval.yml` configuration. +To kick off one pipeline run and the ensuing evaluation, run `nat eval` using the `config-eval.yml` configuration. ``` nat eval --config_file=configs/config-eval.yml @@ -779,7 +779,7 @@ Ground truth datasets should be put in the `data/eval_datasets/` directory, in j In `config-eval.yml`, an example evaluation experiment has been set up on the morpheus_23.11-runtime image, as defined in `data/eval_datasets/eval-morpheus_23.11-runtime.json`, using test set id `test-set0`. -To run evaluation on your image, a new json file should be created. The file should follow the following format: +To run evaluation on your image, a new ground truth json file should be created. The file should follow the following format: ``` { @@ -806,11 +806,11 @@ To run evaluation on your image, a new json file should be created. The file sho In `config-eval.yml`, the following should be configured: - `file_path`: Evaluation dataset specifying image and test set - `kwargs: test_set_id`: The specific test set to run -- `output_dir`: Evaluation output file path +- `output_dir`: Evaluation output directory path ### Running Multiple Evaluations -Use the `--reps` flag to run evaluation multiple times on the same image and workflow. Note that **caching must be disabled first** (see [Disable Caching](#disable-caching)). +Use the `--reps` flag to run evaluation multiple times on the same image and workflow. Note that **caching must be disabled first** (see [Disable Caching](#disable-caching) below). ``` nat eval --config_file=configs/config-eval.yml --reps=3 From 5988743050ac6ec8a3cd00489e277b9f95bd6881 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Thu, 4 Sep 2025 17:01:50 -0700 Subject: [PATCH 014/104] implemented more in-depth evaluators for within-experiment multi repetition evaluation: response spread and consistency. based on VAPR logic --- README.md | 10 ++ src/vuln_analysis/configs/config-eval.yml | 8 +- .../functions/eval_consistency.py | 129 ++++++++++++++++ ...bel_accuracy.py => eval_label_accuracy.py} | 0 .../functions/eval_response_spread.py | 139 ++++++++++++++++++ ...us_accuracy.py => eval_status_accuracy.py} | 0 src/vuln_analysis/register.py | 38 ++++- 7 files changed, 319 insertions(+), 5 deletions(-) create mode 100644 src/vuln_analysis/functions/eval_consistency.py rename src/vuln_analysis/functions/{label_accuracy.py => eval_label_accuracy.py} (100%) create mode 100644 src/vuln_analysis/functions/eval_response_spread.py rename src/vuln_analysis/functions/{status_accuracy.py => eval_status_accuracy.py} (100%) diff --git a/README.md b/README.md index b46194c43..ca9731b77 100644 --- a/README.md +++ b/README.md @@ -840,6 +840,16 @@ For quick testing, you can also change configuration at runtime using the `--ove nat eval --config_file=configs/config-eval.yml --reps=3 --override llms.checklist_llm.base_url https://integrate.api.nvidia.com/v1 ``` +### Additional Evaluators + +#### Within-Experiment +- **Measure Response Spread:** For multiple reps (you can technically run it with one rep but the spread will just be perfect score of 1). Calculates a response spread across all the reps, per CVE. Outputs a response spread for each CVE, and also provides an average response spread across all CVEs. Response spread is from 0 to 1, with 1 being perfect match (no spread). It's calculated based on the natural language summary field. It's calculated via creating a bag of words (word frequency vector) and calculating the cosine similarity between the centroid and the response produced by each rep. It then uses the configured aggregation method (average, min, max) to produce the final output. +- **Measure Consistency:** For multiple reps (you can technically run it with one rep but the consistency will be perfect score of 1). Calculates consistency on status and labels per CVE. Consistency is captured by 'alikeablity' which is calculated by this formula: Alikeability = sum(p^2) where p is the probability of each unique value. So if you did 3 reps and had labels: {vulnerable, code_not_reachable, code_not_reachable}, you would have two p's, causing the probabilities array to be {1/3, 2/3}. Taking the square of each of those values and then summing them: (1/3)^2 + (2/3)^2 = 0.56, which is a mid range consistency. Notice that if you had a single p (all label values were the same), your consistency would be 1.0, or a perfect consistency. This evaluator outputs two consistency scores per CVE (one for status, one for labels), as well as the average status consistency score, the average label consistency score, and the overall combined average of the two averages. + +#### Cross-Experiment Comparison +- **Aggregate Responses:** WIP +- **Compare Labels:** WIP + ## Troubleshooting diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml index afbc953d9..665db26a5 100644 --- a/src/vuln_analysis/configs/config-eval.yml +++ b/src/vuln_analysis/configs/config-eval.yml @@ -157,4 +157,10 @@ eval: status_accuracy_eval: _type: status_accuracy label_accuracy_eval: - _type: label_accuracy \ No newline at end of file + _type: label_accuracy + response_spread_eval: + _type: response_spread + kwargs: + aggregation: "average" + consistency_eval: + _type: consistency \ No newline at end of file diff --git a/src/vuln_analysis/functions/eval_consistency.py b/src/vuln_analysis/functions/eval_consistency.py new file mode 100644 index 000000000..ed1669cce --- /dev/null +++ b/src/vuln_analysis/functions/eval_consistency.py @@ -0,0 +1,129 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +import asyncio +import json +import numpy as np +from collections import Counter + +from tqdm import tqdm + +from nat.eval.evaluator.evaluator_model import EvalInput +from nat.eval.evaluator.evaluator_model import EvalOutput +from nat.eval.evaluator.evaluator_model import EvalOutputItem +from nat.eval.evaluator.evaluator_model import EvalInputItem +from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry + + +class ConsistencyEvaluator: + '''Evaluator class for measuring consistency using alikeability metric''' + + def __init__(self, max_concurrency: int): + self.max_concurrency = max_concurrency + self.semaphore = asyncio.Semaphore(self.max_concurrency) + + async def evaluate(self, eval_input: EvalInput) -> EvalOutput: + '''Evaluate consistency across multiple runs''' + + async def process_item(item): + """Calculate consistency for an individual item across multiple runs""" + pipeline_result = json.loads(item.output_obj)['output'] + + # Group responses by vuln_id to get multiple responses per CVE + vuln_responses = {} + + for result in pipeline_result: + vuln_id = result['vuln_id'] + # We can measure consistency on different fields + label = result.get('justification', {}).get('label', 'MISSING') + status = result.get('justification', {}).get('status', 'MISSING') + + if vuln_id not in vuln_responses: + vuln_responses[vuln_id] = {'labels': [], 'statuses': []} + + vuln_responses[vuln_id]['labels'].append(label) + vuln_responses[vuln_id]['statuses'].append(status) + + # Calculate consistency for each CVE using alikeability (entropy-based) + label_consistencies = {} + status_consistencies = {} + all_label_consistencies = [] + all_status_consistencies = [] + + for vuln_id, responses in vuln_responses.items(): + # Calculate label consistency + labels = responses['labels'] + if len(labels) > 1: + # Alikeability calculation: sum of squared probabilities + label_counts = Counter(labels) + n = len(labels) + probs = [count / n for count in label_counts.values()] + label_consistency = sum(p * p for p in probs) + else: + label_consistency = 1.0 # Perfect consistency if only one response + + label_consistencies[vuln_id] = label_consistency + all_label_consistencies.append(label_consistency) + + # Calculate status consistency + statuses = responses['statuses'] + if len(statuses) > 1: + # Alikeability calculation: sum of squared probabilities + status_counts = Counter(statuses) + n = len(statuses) + probs = [count / n for count in status_counts.values()] + status_consistency = sum(p * p for p in probs) + else: + status_consistency = 1.0 # Perfect consistency if only one response + + status_consistencies[vuln_id] = status_consistency + all_status_consistencies.append(status_consistency) + + # Calculate overall average consistency + overall_label_consistency = sum(all_label_consistencies) / len(all_label_consistencies) if all_label_consistencies else 1.0 + overall_status_consistency = sum(all_status_consistencies) / len(all_status_consistencies) if all_status_consistencies else 1.0 + + reasoning = { + "label_consistencies": label_consistencies, + "status_consistencies": status_consistencies, + "overall_label_consistency": overall_label_consistency, + "overall_status_consistency": overall_status_consistency, + "num_cves": len(vuln_responses), + "total_responses": sum(len(responses['labels']) for responses in vuln_responses.values()) + } + + # Return average of label and status consistency + overall_consistency = (overall_label_consistency + overall_status_consistency) / 2 + return overall_consistency, reasoning + + async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: + async with self.semaphore: + result = await process_item(item) + pbar.update(1) + return result + + try: + tqdm_position = TqdmPositionRegistry.claim() + pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Consistency", position=tqdm_position) + results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) + finally: + pbar.close() + TqdmPositionRegistry.release(tqdm_position) + + sample_scores, sample_reasonings = zip(*results) if results else ([], []) + + avg_score = round(sum(sample_scores) / len(sample_scores), 4) if sample_scores else 1.0 + + eval_output_items = [ + EvalOutputItem(id=item.id, score=score, reasoning=reasoning) + for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings) + ] + + return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) diff --git a/src/vuln_analysis/functions/label_accuracy.py b/src/vuln_analysis/functions/eval_label_accuracy.py similarity index 100% rename from src/vuln_analysis/functions/label_accuracy.py rename to src/vuln_analysis/functions/eval_label_accuracy.py diff --git a/src/vuln_analysis/functions/eval_response_spread.py b/src/vuln_analysis/functions/eval_response_spread.py new file mode 100644 index 000000000..7e6bc455b --- /dev/null +++ b/src/vuln_analysis/functions/eval_response_spread.py @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +import asyncio +import json +import numpy as np +from collections import Counter +from sklearn.feature_extraction.text import CountVectorizer + +from tqdm import tqdm + +from nat.eval.evaluator.evaluator_model import EvalInput +from nat.eval.evaluator.evaluator_model import EvalOutput +from nat.eval.evaluator.evaluator_model import EvalOutputItem +from nat.eval.evaluator.evaluator_model import EvalInputItem +from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry + + +class ResponseSpreadEvaluator: + '''Evaluator class for measuring response spread using cosine similarity''' + + def __init__(self, max_concurrency: int, aggregation: str = "average", stopwords: str = None): + self.max_concurrency = max_concurrency + self.semaphore = asyncio.Semaphore(self.max_concurrency) + self.aggregation = aggregation # "average", "min", or "max" + self.stopwords = stopwords + + async def evaluate(self, eval_input: EvalInput) -> EvalOutput: + '''Evaluate response spread across multiple runs''' + + async def process_item(item): + """Calculate response spread for an individual item across multiple runs""" + pipeline_result = json.loads(item.output_obj)['output'] + + # Group responses by vuln_id to get multiple responses per CVE + vuln_responses = {} + + for result in pipeline_result: + vuln_id = result['vuln_id'] + summary = result.get('summary', '') + + if vuln_id not in vuln_responses: + vuln_responses[vuln_id] = [] + vuln_responses[vuln_id].append(summary) + + # Calculate response spread for each CVE + response_spreads = {} + all_spreads = [] + + for vuln_id, responses in vuln_responses.items(): + if len(responses) < 2: + # Need at least 2 responses to calculate spread + response_spreads[vuln_id] = 1.0 # Perfect similarity if only one response + all_spreads.append(1.0) + continue + + # Calculate cosine similarity spread using bag-of-words + try: + vectorizer = CountVectorizer(stop_words=self.stopwords) + X = vectorizer.fit_transform(responses).toarray() + + if X.shape[0] < 2: + response_spreads[vuln_id] = 1.0 + all_spreads.append(1.0) + continue + + # Calculate centroid (average of all response vectors) + centroid = np.sum(X, axis=0) + + # Calculate cosine similarities between each response and centroid + dot_products = np.dot(X, centroid) + norms = np.linalg.norm(X, axis=1) * np.linalg.norm(centroid) + + # Avoid division by zero + norms = np.where(norms == 0, 1, norms) + cos_similarities = dot_products / norms + + # Apply aggregation + if self.aggregation == "average": + spread_score = float(np.average(cos_similarities)) + elif self.aggregation == "max": + spread_score = float(np.max(cos_similarities)) + elif self.aggregation == "min": + spread_score = float(np.min(cos_similarities)) + else: + spread_score = float(np.average(cos_similarities)) + + response_spreads[vuln_id] = spread_score + all_spreads.append(spread_score) + + except Exception as e: + # If calculation fails, default to 1.0 (perfect similarity) + response_spreads[vuln_id] = 1.0 + all_spreads.append(1.0) + + # Calculate overall average spread + overall_spread = sum(all_spreads) / len(all_spreads) if all_spreads else 1.0 + + reasoning = { + "response_spreads": response_spreads, + "overall_spread": overall_spread, + "aggregation_method": self.aggregation, + "num_cves": len(vuln_responses), + "total_responses": sum(len(responses) for responses in vuln_responses.values()) + } + + return overall_spread, reasoning + + async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: + async with self.semaphore: + result = await process_item(item) + pbar.update(1) + return result + + try: + tqdm_position = TqdmPositionRegistry.claim() + pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Response Spread", position=tqdm_position) + results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) + finally: + pbar.close() + TqdmPositionRegistry.release(tqdm_position) + + sample_scores, sample_reasonings = zip(*results) if results else ([], []) + + avg_score = round(sum(sample_scores) / len(sample_scores), 4) if sample_scores else 1.0 + + eval_output_items = [ + EvalOutputItem(id=item.id, score=score, reasoning=reasoning) + for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings) + ] + + return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) diff --git a/src/vuln_analysis/functions/status_accuracy.py b/src/vuln_analysis/functions/eval_status_accuracy.py similarity index 100% rename from src/vuln_analysis/functions/status_accuracy.py rename to src/vuln_analysis/functions/eval_status_accuracy.py diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index 42c90f96b..e6a76566f 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -77,7 +77,7 @@ class CVEAgentWorkflowConfig(FunctionBaseConfig, name="cve_agent"): description="Workflow function description") -#evaluator registration (actual code is in functions/status_accuracy.py) +#evaluator registration (actual code is in functions/eval_status_accuracy.py) class StatusAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="status_accuracy"): '''Configuration for status accuracy evaluator''' pass @@ -86,13 +86,13 @@ class StatusAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="status_accuracy") @register_evaluator(config_type=StatusAccuracyEvaluatorConfig) async def status_accuracy_evaluator(config: StatusAccuracyEvaluatorConfig, builder: EvalBuilder): '''Register status accuracy evaluator''' - from .functions.status_accuracy import StatusAccuracyEvaluator + from .functions.eval_status_accuracy import StatusAccuracyEvaluator evaluator = StatusAccuracyEvaluator(builder.get_max_concurrency()) yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Status Accuracy Evaluator") -#evaluator registration (actual code is in functions/label_accuracy.py) +#evaluator registration (actual code is in functions/eval_label_accuracy.py) class LabelAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="label_accuracy"): '''Configuration for label accuracy evaluator''' pass @@ -101,12 +101,42 @@ class LabelAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="label_accuracy"): @register_evaluator(config_type=LabelAccuracyEvaluatorConfig) async def label_accuracy_evaluator(config: LabelAccuracyEvaluatorConfig, builder: EvalBuilder): '''Register label accuracy evaluator''' - from .functions.label_accuracy import LabelAccuracyEvaluator + from .functions.eval_label_accuracy import LabelAccuracyEvaluator evaluator = LabelAccuracyEvaluator(builder.get_max_concurrency()) yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Label Accuracy Evaluator") +#evaluator registration (actual code is in functions/eval_response_spread.py) +class ResponseSpreadEvaluatorConfig(EvaluatorBaseConfig, name="response_spread"): + '''Configuration for response spread evaluator''' + pass + + +@register_evaluator(config_type=ResponseSpreadEvaluatorConfig) +async def response_spread_evaluator(config: ResponseSpreadEvaluatorConfig, builder: EvalBuilder): + '''Register response spread evaluator''' + from .functions.eval_response_spread import ResponseSpreadEvaluator + evaluator = ResponseSpreadEvaluator(builder.get_max_concurrency()) + + yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Response Spread Evaluator") + + +#evaluator registration (actual code is in functions/eval_consistency.py) +class ConsistencyEvaluatorConfig(EvaluatorBaseConfig, name="consistency"): + '''Configuration for consistency evaluator''' + pass + + +@register_evaluator(config_type=ConsistencyEvaluatorConfig) +async def consistency_evaluator(config: ConsistencyEvaluatorConfig, builder: EvalBuilder): + '''Register consistency evaluator''' + from .functions.eval_consistency import ConsistencyEvaluator + evaluator = ConsistencyEvaluator(builder.get_max_concurrency()) + + yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Consistency Evaluator") + + @register_function(config_type=CVEAgentWorkflowConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def cve_agent_workflow(config: CVEAgentWorkflowConfig, builder: Builder): From 65b6a8e2a3df9c3c3ac456a1a45b69605f1defe0 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Fri, 5 Sep 2025 15:27:56 -0700 Subject: [PATCH 015/104] new eval dataset input file format --- .../data/eval_datasets/eval-dataset.json | 181 ++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 src/vuln_analysis/data/eval_datasets/eval-dataset.json diff --git a/src/vuln_analysis/data/eval_datasets/eval-dataset.json b/src/vuln_analysis/data/eval_datasets/eval-dataset.json new file mode 100644 index 000000000..fb3b6056c --- /dev/null +++ b/src/vuln_analysis/data/eval_datasets/eval-dataset.json @@ -0,0 +1,181 @@ +{ + "containers": { + "morpheus-23.11": { + "container_image": { + "name": "nvcr.io/nvidia/morpheus/morpheus", + "tag": "23.11-runtime", + "source_info": [ + { + "type": "code", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "ref": "branch-23.11", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ], + "exclude": [ + "tests/**/*" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "ref": "branch-23.11", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + } + ], + "sbom_info": { + "_type": "file", + "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom" + } + }, + "test_sets": { + "morpheus-23.11-test-set": { + "description": "morpheus-23.11 only test set", + "ground_truth": [ + { + "vuln_id": "GHSA-3f63-hfp8-52jq", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + }, + { + "vuln_id": "CVE-2023-36632", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "GHSA-cxfr-5q3r-2rc2", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "GHSA-554w-xh4j-8w64", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "CVE-2023-51767", + "status": "AFFECTED", + "label": "vulnerable" + } + ] + }, + "test-set-common": { + "description": "Common test set for both containers", + "ground_truth": [ + { + "vuln_id": "GHSA-3f63-hfp8-52jq", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + }, + { + "vuln_id": "CVE-2023-36632", + "status": "AFFECTED", + "label": "vulnerable" + } + ] + } + } + }, + "morpheus-24.03": { + "container_image": { + "name": "nvcr.io/nvidia/morpheus/morpheus", + "tag": "v24.03.02-runtime", + "source_info": [ + { + "type": "code", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "ref": "v24.03.02", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ], + "exclude": [ + "tests/**/*" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "ref": "v24.03.02", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + } + ], + "sbom_info": { + "_type": "file", + "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v24.03.02-runtime.sbom" + } + }, + "test_sets": { + "morpheus-24.03-test-set": { + "description": "morpheus-24.03 only test set (toy data, not real ground truth)", + "ground_truth": [ + { + "vuln_id": "GHSA-3f63-hfp8-52jq", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + }, + { + "vuln_id": "CVE-2023-50782", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "CVE-2023-36632", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "CVE-2023-43804", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + }, + { + "vuln_id": "GHSA-cxfr-5q3r-2rc2", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "GHSA-554w-xh4j-8w64", + "status": "AFFECTED", + "label": "vulnerable" + } + ] + }, + "test-set-common": { + "description": "Common test set for both containers", + "ground_truth": [ + { + "vuln_id": "GHSA-3f63-hfp8-52jq", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + }, + { + "vuln_id": "CVE-2023-36632", + "status": "AFFECTED", + "label": "vulnerable" + } + ] + } + } + } + } +} \ No newline at end of file From ef053dfcc1053fb8b39ca7ba17ee64a9fbb80cbe Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Fri, 5 Sep 2025 17:17:30 -0700 Subject: [PATCH 016/104] Ground truth input file structure changed to list all containers; readme updated to reflect this. All eval functions are in a new eval directory. Custom evaluator registrations are moved to a new eval_registry.py file. --- README.md | 68 +++--- src/vuln_analysis/configs/config-eval.yml | 10 +- .../data/eval_datasets/eval-dataset.json | 181 --------------- .../data/eval_datasets/eval_dataset.json | 186 ++++++++++++++- ....11-runtime.json => old_eval_dataset.json} | 5 - .../data/eval_datasets/profiler_dataset.json | 7 + .../{functions => eval}/eval_consistency.py | 4 +- .../eval_label_accuracy.py | 1 + .../eval_response_spread.py | 4 +- .../eval_status_accuracy.py | 2 +- src/vuln_analysis/eval/evaluator_register.py | 80 +++++++ src/vuln_analysis/eval/parse_eval_input.py | 212 ++++++++++++++++++ .../functions/parse_eval_input.py | 169 -------------- src/vuln_analysis/register.py | 69 +----- 14 files changed, 523 insertions(+), 475 deletions(-) delete mode 100644 src/vuln_analysis/data/eval_datasets/eval-dataset.json rename src/vuln_analysis/data/eval_datasets/{eval-morpheus_23.11-runtime.json => old_eval_dataset.json} (95%) create mode 100644 src/vuln_analysis/data/eval_datasets/profiler_dataset.json rename src/vuln_analysis/{functions => eval}/eval_consistency.py (97%) rename src/vuln_analysis/{functions => eval}/eval_label_accuracy.py (97%) rename src/vuln_analysis/{functions => eval}/eval_response_spread.py (98%) rename src/vuln_analysis/{functions => eval}/eval_status_accuracy.py (98%) create mode 100644 src/vuln_analysis/eval/evaluator_register.py create mode 100644 src/vuln_analysis/eval/parse_eval_input.py delete mode 100644 src/vuln_analysis/functions/parse_eval_input.py diff --git a/README.md b/README.md index ca9731b77..b4ced277b 100644 --- a/README.md +++ b/README.md @@ -759,13 +759,13 @@ The workflow can then be updated to use the new function: Additional output options will be added in the future. ## Evaluation -In this simple evaluation workflow, we evaluate the pipeline's ability to identify vulnerability **status** and **label**. +In this example evaluation workflow, we evaluate the pipeline's ability to identify vulnerability **status** and **label**. Each CVE has one ground truth status and one corresponding label. Valid statuses include: `AFFECTED` or `NOT AFFECTED`. Valid labels include: `vulnerable` if the CVE status is `AFFECTED`, or one of 10 VEX statuses if the status is `NOT AFFECTED` (see **Justification Assignment** under the [Key Components section](#key-components)). -To kick off one pipeline run and the ensuing evaluation, run `nat eval` using the `config-eval.yml` configuration. +To run evaluaton, run `nat eval` using the `config-eval.yml` configuration. ``` nat eval --config_file=configs/config-eval.yml @@ -774,39 +774,32 @@ nat eval --config_file=configs/config-eval.yml This will write the files `status_accuracy_eval_output.json` and `label_accuracy_eval_output.json` to the configured output directory (`.tmp/eval/my_eval` in this example), containing accuracy scores for status and label respectively. Scores range from 0 to 1, with 0 indicating no CVEs were categorized correctly, and 1 indicating all CVEs were categorized correctly. ### Inputting Evaluation Test Sets +Ground truth datasets should be put in `data/eval_datasets/eval_dataset.json`. The file contains all containers and CVEs to be tested, and has the following structure: + +``` +containers: + container1 + container1 metadata + test-setA: {list of CVEs} + test-setB: {list of CVEs} + container2 + container2 metadata + test-setB: {list of CVEs} + test-setC: {list of CVEs} +``` + +In `config-eval.yml`, an example evaluation experiment has been set up on the morpheus_23.11-runtime and morpheus_24.03-runtime images, using the `test-set-common` test set. + +In `config-eval.yml`, the `test_set_id` and `container_ids` should be configured. For example: +```yml +test_set_id: "test-setB" +container_ids: ["container1", "container2"] +``` + +Note that in order to run a particular test set on a set of images, that test set ID must be defined for all images (eg. here `test-setB` is valid, but `test-setA` would be invalid because `container2` does not define a ground truth for that test set). + +You can also configure the evaluation output directory using the `output_dir` variable. -Ground truth datasets should be put in the `data/eval_datasets/` directory, in json format. Each json file corresponds to a different image. Additionally, for each image, different evaluation experiments can be defined as test sets where each test set lists the CVEs to be tested. - -In `config-eval.yml`, an example evaluation experiment has been set up on the morpheus_23.11-runtime image, as defined in `data/eval_datasets/eval-morpheus_23.11-runtime.json`, using test set id `test-set0`. - -To run evaluation on your image, a new ground truth json file should be created. The file should follow the following format: - -``` -{ - "container_image": { - "name": "nvcr.io/my/image", - "tag": "my-tag", - "source_info": [], #include source info - "sbom_info": {} #include sbom info - }, - "test_sets": { - "my_test_set": { - "description": "My workflow description", - "ground_truth": [ - { - "vuln_id": "CVE-0000-00001", - "status": "NOT AFFECTED", - "label": "code_not_reachable" - } # list other CVEs - ] - } # create other workflows - } -} -``` -In `config-eval.yml`, the following should be configured: -- `file_path`: Evaluation dataset specifying image and test set -- `kwargs: test_set_id`: The specific test set to run -- `output_dir`: Evaluation output directory path ### Running Multiple Evaluations @@ -842,13 +835,12 @@ nat eval --config_file=configs/config-eval.yml --reps=3 --override llms.checklis ### Additional Evaluators -#### Within-Experiment +WIP: i need to make these descriptions more clear - **Measure Response Spread:** For multiple reps (you can technically run it with one rep but the spread will just be perfect score of 1). Calculates a response spread across all the reps, per CVE. Outputs a response spread for each CVE, and also provides an average response spread across all CVEs. Response spread is from 0 to 1, with 1 being perfect match (no spread). It's calculated based on the natural language summary field. It's calculated via creating a bag of words (word frequency vector) and calculating the cosine similarity between the centroid and the response produced by each rep. It then uses the configured aggregation method (average, min, max) to produce the final output. - **Measure Consistency:** For multiple reps (you can technically run it with one rep but the consistency will be perfect score of 1). Calculates consistency on status and labels per CVE. Consistency is captured by 'alikeablity' which is calculated by this formula: Alikeability = sum(p^2) where p is the probability of each unique value. So if you did 3 reps and had labels: {vulnerable, code_not_reachable, code_not_reachable}, you would have two p's, causing the probabilities array to be {1/3, 2/3}. Taking the square of each of those values and then summing them: (1/3)^2 + (2/3)^2 = 0.56, which is a mid range consistency. Notice that if you had a single p (all label values were the same), your consistency would be 1.0, or a perfect consistency. This evaluator outputs two consistency scores per CVE (one for status, one for labels), as well as the average status consistency score, the average label consistency score, and the overall combined average of the two averages. -#### Cross-Experiment Comparison -- **Aggregate Responses:** WIP -- **Compare Labels:** WIP +These are defined in `eval` dir and registered in `evaluator_registry.py` +You can write an evaluator for your desired metrics via writing a custom evaluator (link NAT docs) ## Troubleshooting diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml index 665db26a5..9c0c81a23 100644 --- a/src/vuln_analysis/configs/config-eval.yml +++ b/src/vuln_analysis/configs/config-eval.yml @@ -148,10 +148,12 @@ eval: output_dir: ./.tmp/eval/my_eval dataset: _type: custom - file_path: data/eval_datasets/eval-morpheus_23.11-runtime.json - function: vuln_analysis.functions.parse_eval_input.parse_input + file_path: data/eval_datasets/eval_dataset.json + function: vuln_analysis.eval.parse_eval_input.parse_input kwargs: - test_set_id: "test-set-small" + test_set_id: "test-set-common" + container_ids: ["morpheus-23.11", "morpheus-24.03"] + # comment out container_ids to use all available containers evaluators: status_accuracy_eval: @@ -161,6 +163,6 @@ eval: response_spread_eval: _type: response_spread kwargs: - aggregation: "average" + aggregation: "average" #min, max consistency_eval: _type: consistency \ No newline at end of file diff --git a/src/vuln_analysis/data/eval_datasets/eval-dataset.json b/src/vuln_analysis/data/eval_datasets/eval-dataset.json deleted file mode 100644 index fb3b6056c..000000000 --- a/src/vuln_analysis/data/eval_datasets/eval-dataset.json +++ /dev/null @@ -1,181 +0,0 @@ -{ - "containers": { - "morpheus-23.11": { - "container_image": { - "name": "nvcr.io/nvidia/morpheus/morpheus", - "tag": "23.11-runtime", - "source_info": [ - { - "type": "code", - "git_repo": "https://github.com/nv-morpheus/Morpheus.git", - "ref": "branch-23.11", - "include": [ - "**/*.cpp", - "**/*.cu", - "**/*.cuh", - "**/*.h", - "**/*.hpp", - "**/*.ipynb", - "**/*.py", - "**/*Dockerfile" - ], - "exclude": [ - "tests/**/*" - ] - }, - { - "type": "doc", - "git_repo": "https://github.com/nv-morpheus/Morpheus.git", - "ref": "branch-23.11", - "include": [ - "**/*.md", - "docs/**/*.rst" - ] - } - ], - "sbom_info": { - "_type": "file", - "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom" - } - }, - "test_sets": { - "morpheus-23.11-test-set": { - "description": "morpheus-23.11 only test set", - "ground_truth": [ - { - "vuln_id": "GHSA-3f63-hfp8-52jq", - "status": "NOT AFFECTED", - "label": "code_not_reachable" - }, - { - "vuln_id": "CVE-2023-36632", - "status": "AFFECTED", - "label": "vulnerable" - }, - { - "vuln_id": "GHSA-cxfr-5q3r-2rc2", - "status": "AFFECTED", - "label": "vulnerable" - }, - { - "vuln_id": "GHSA-554w-xh4j-8w64", - "status": "AFFECTED", - "label": "vulnerable" - }, - { - "vuln_id": "CVE-2023-51767", - "status": "AFFECTED", - "label": "vulnerable" - } - ] - }, - "test-set-common": { - "description": "Common test set for both containers", - "ground_truth": [ - { - "vuln_id": "GHSA-3f63-hfp8-52jq", - "status": "NOT AFFECTED", - "label": "code_not_reachable" - }, - { - "vuln_id": "CVE-2023-36632", - "status": "AFFECTED", - "label": "vulnerable" - } - ] - } - } - }, - "morpheus-24.03": { - "container_image": { - "name": "nvcr.io/nvidia/morpheus/morpheus", - "tag": "v24.03.02-runtime", - "source_info": [ - { - "type": "code", - "git_repo": "https://github.com/nv-morpheus/Morpheus.git", - "ref": "v24.03.02", - "include": [ - "**/*.cpp", - "**/*.cu", - "**/*.cuh", - "**/*.h", - "**/*.hpp", - "**/*.ipynb", - "**/*.py", - "**/*Dockerfile" - ], - "exclude": [ - "tests/**/*" - ] - }, - { - "type": "doc", - "git_repo": "https://github.com/nv-morpheus/Morpheus.git", - "ref": "v24.03.02", - "include": [ - "**/*.md", - "docs/**/*.rst" - ] - } - ], - "sbom_info": { - "_type": "file", - "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v24.03.02-runtime.sbom" - } - }, - "test_sets": { - "morpheus-24.03-test-set": { - "description": "morpheus-24.03 only test set (toy data, not real ground truth)", - "ground_truth": [ - { - "vuln_id": "GHSA-3f63-hfp8-52jq", - "status": "NOT AFFECTED", - "label": "code_not_reachable" - }, - { - "vuln_id": "CVE-2023-50782", - "status": "AFFECTED", - "label": "vulnerable" - }, - { - "vuln_id": "CVE-2023-36632", - "status": "AFFECTED", - "label": "vulnerable" - }, - { - "vuln_id": "CVE-2023-43804", - "status": "NOT AFFECTED", - "label": "code_not_reachable" - }, - { - "vuln_id": "GHSA-cxfr-5q3r-2rc2", - "status": "AFFECTED", - "label": "vulnerable" - }, - { - "vuln_id": "GHSA-554w-xh4j-8w64", - "status": "AFFECTED", - "label": "vulnerable" - } - ] - }, - "test-set-common": { - "description": "Common test set for both containers", - "ground_truth": [ - { - "vuln_id": "GHSA-3f63-hfp8-52jq", - "status": "NOT AFFECTED", - "label": "code_not_reachable" - }, - { - "vuln_id": "CVE-2023-36632", - "status": "AFFECTED", - "label": "vulnerable" - } - ] - } - } - } - } -} \ No newline at end of file diff --git a/src/vuln_analysis/data/eval_datasets/eval_dataset.json b/src/vuln_analysis/data/eval_datasets/eval_dataset.json index 1f4597f4e..fb3b6056c 100644 --- a/src/vuln_analysis/data/eval_datasets/eval_dataset.json +++ b/src/vuln_analysis/data/eval_datasets/eval_dataset.json @@ -1,7 +1,181 @@ -[ - { - "id": 1, - "question": "{\"image\":{\"name\":\"nvcr.io\/nvidia\/morpheus\/morpheus\",\"tag\":\"23.11-runtime\",\"source_info\":[{\"type\":\"code\",\"git_repo\":\"https:\/\/github.com\/nv-morpheus\/Morpheus.git\",\"ref\":\"v23.11.01\",\"include\":[\"**\/*.cpp\",\"**\/*.cu\",\"**\/*.cuh\",\"**\/*.h\",\"**\/*.hpp\",\"**\/*.ipynb\",\"**\/*.py\",\"**\/*Dockerfile\"],\"exclude\":[\"tests\/**\/*\"]},{\"type\":\"doc\",\"git_repo\":\"https:\/\/github.com\/nv-morpheus\/Morpheus.git\",\"ref\":\"v23.11.01\",\"include\":[\"**\/*.md\",\"docs\/**\/*.rst\"]}],\"sbom_info\":{\"_type\":\"file\",\"file_path\":\"data\/sboms\/nvcr.io\/nvidia\/morpheus\/morpheus:v23.11.01-runtime.sbom\"}},\"scan\":{\"vulns\":[{\"vuln_id\":\"GHSA-3f63-hfp8-52jq\"},{\"vuln_id\":\"CVE-2023-50782\"}]}}", - "answer": "N/A" +{ + "containers": { + "morpheus-23.11": { + "container_image": { + "name": "nvcr.io/nvidia/morpheus/morpheus", + "tag": "23.11-runtime", + "source_info": [ + { + "type": "code", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "ref": "branch-23.11", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ], + "exclude": [ + "tests/**/*" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "ref": "branch-23.11", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + } + ], + "sbom_info": { + "_type": "file", + "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom" + } + }, + "test_sets": { + "morpheus-23.11-test-set": { + "description": "morpheus-23.11 only test set", + "ground_truth": [ + { + "vuln_id": "GHSA-3f63-hfp8-52jq", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + }, + { + "vuln_id": "CVE-2023-36632", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "GHSA-cxfr-5q3r-2rc2", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "GHSA-554w-xh4j-8w64", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "CVE-2023-51767", + "status": "AFFECTED", + "label": "vulnerable" + } + ] + }, + "test-set-common": { + "description": "Common test set for both containers", + "ground_truth": [ + { + "vuln_id": "GHSA-3f63-hfp8-52jq", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + }, + { + "vuln_id": "CVE-2023-36632", + "status": "AFFECTED", + "label": "vulnerable" + } + ] + } + } + }, + "morpheus-24.03": { + "container_image": { + "name": "nvcr.io/nvidia/morpheus/morpheus", + "tag": "v24.03.02-runtime", + "source_info": [ + { + "type": "code", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "ref": "v24.03.02", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ], + "exclude": [ + "tests/**/*" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "ref": "v24.03.02", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + } + ], + "sbom_info": { + "_type": "file", + "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v24.03.02-runtime.sbom" + } + }, + "test_sets": { + "morpheus-24.03-test-set": { + "description": "morpheus-24.03 only test set (toy data, not real ground truth)", + "ground_truth": [ + { + "vuln_id": "GHSA-3f63-hfp8-52jq", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + }, + { + "vuln_id": "CVE-2023-50782", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "CVE-2023-36632", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "CVE-2023-43804", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + }, + { + "vuln_id": "GHSA-cxfr-5q3r-2rc2", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "GHSA-554w-xh4j-8w64", + "status": "AFFECTED", + "label": "vulnerable" + } + ] + }, + "test-set-common": { + "description": "Common test set for both containers", + "ground_truth": [ + { + "vuln_id": "GHSA-3f63-hfp8-52jq", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + }, + { + "vuln_id": "CVE-2023-36632", + "status": "AFFECTED", + "label": "vulnerable" + } + ] + } + } } -] + } +} \ No newline at end of file diff --git a/src/vuln_analysis/data/eval_datasets/eval-morpheus_23.11-runtime.json b/src/vuln_analysis/data/eval_datasets/old_eval_dataset.json similarity index 95% rename from src/vuln_analysis/data/eval_datasets/eval-morpheus_23.11-runtime.json rename to src/vuln_analysis/data/eval_datasets/old_eval_dataset.json index 27ea4ea04..c48df82bd 100644 --- a/src/vuln_analysis/data/eval_datasets/eval-morpheus_23.11-runtime.json +++ b/src/vuln_analysis/data/eval_datasets/old_eval_dataset.json @@ -104,11 +104,6 @@ "vuln_id": "GHSA-3f63-hfp8-52jq", "status": "NOT AFFECTED", "label": "code_not_reachable" - }, - { - "vuln_id": "CVE-2023-51767", - "status": "AFFECTED", - "label": "vulnerable" } ] } diff --git a/src/vuln_analysis/data/eval_datasets/profiler_dataset.json b/src/vuln_analysis/data/eval_datasets/profiler_dataset.json new file mode 100644 index 000000000..1f4597f4e --- /dev/null +++ b/src/vuln_analysis/data/eval_datasets/profiler_dataset.json @@ -0,0 +1,7 @@ +[ + { + "id": 1, + "question": "{\"image\":{\"name\":\"nvcr.io\/nvidia\/morpheus\/morpheus\",\"tag\":\"23.11-runtime\",\"source_info\":[{\"type\":\"code\",\"git_repo\":\"https:\/\/github.com\/nv-morpheus\/Morpheus.git\",\"ref\":\"v23.11.01\",\"include\":[\"**\/*.cpp\",\"**\/*.cu\",\"**\/*.cuh\",\"**\/*.h\",\"**\/*.hpp\",\"**\/*.ipynb\",\"**\/*.py\",\"**\/*Dockerfile\"],\"exclude\":[\"tests\/**\/*\"]},{\"type\":\"doc\",\"git_repo\":\"https:\/\/github.com\/nv-morpheus\/Morpheus.git\",\"ref\":\"v23.11.01\",\"include\":[\"**\/*.md\",\"docs\/**\/*.rst\"]}],\"sbom_info\":{\"_type\":\"file\",\"file_path\":\"data\/sboms\/nvcr.io\/nvidia\/morpheus\/morpheus:v23.11.01-runtime.sbom\"}},\"scan\":{\"vulns\":[{\"vuln_id\":\"GHSA-3f63-hfp8-52jq\"},{\"vuln_id\":\"CVE-2023-50782\"}]}}", + "answer": "N/A" + } +] diff --git a/src/vuln_analysis/functions/eval_consistency.py b/src/vuln_analysis/eval/eval_consistency.py similarity index 97% rename from src/vuln_analysis/functions/eval_consistency.py rename to src/vuln_analysis/eval/eval_consistency.py index ed1669cce..2ced13614 100644 --- a/src/vuln_analysis/functions/eval_consistency.py +++ b/src/vuln_analysis/eval/eval_consistency.py @@ -89,14 +89,14 @@ async def process_item(item): # Calculate overall average consistency overall_label_consistency = sum(all_label_consistencies) / len(all_label_consistencies) if all_label_consistencies else 1.0 overall_status_consistency = sum(all_status_consistencies) / len(all_status_consistencies) if all_status_consistencies else 1.0 - reasoning = { "label_consistencies": label_consistencies, "status_consistencies": status_consistencies, "overall_label_consistency": overall_label_consistency, "overall_status_consistency": overall_status_consistency, "num_cves": len(vuln_responses), - "total_responses": sum(len(responses['labels']) for responses in vuln_responses.values()) + "total_responses": sum(len(responses['labels']) for responses in vuln_responses.values()), + "test_set_id": item.full_dataset_entry.get('test_set_id', 'unknown'), } # Return average of label and status consistency diff --git a/src/vuln_analysis/functions/eval_label_accuracy.py b/src/vuln_analysis/eval/eval_label_accuracy.py similarity index 97% rename from src/vuln_analysis/functions/eval_label_accuracy.py rename to src/vuln_analysis/eval/eval_label_accuracy.py index c1abf0b0a..91a636ccc 100644 --- a/src/vuln_analysis/functions/eval_label_accuracy.py +++ b/src/vuln_analysis/eval/eval_label_accuracy.py @@ -58,6 +58,7 @@ async def process_item(item): "question": question, "answer": [ground_truth_map.get(f"{q}_label", 'MISSING') for q in question], "generated_answer": generated_answers, + "test_set_id": item.full_dataset_entry.get('test_set_id', 'unknown'), } return sum(label_accuracies)/len(label_accuracies) if label_accuracies else 0.0, reasoning diff --git a/src/vuln_analysis/functions/eval_response_spread.py b/src/vuln_analysis/eval/eval_response_spread.py similarity index 98% rename from src/vuln_analysis/functions/eval_response_spread.py rename to src/vuln_analysis/eval/eval_response_spread.py index 7e6bc455b..e360a49b2 100644 --- a/src/vuln_analysis/functions/eval_response_spread.py +++ b/src/vuln_analysis/eval/eval_response_spread.py @@ -102,13 +102,13 @@ async def process_item(item): # Calculate overall average spread overall_spread = sum(all_spreads) / len(all_spreads) if all_spreads else 1.0 - reasoning = { "response_spreads": response_spreads, "overall_spread": overall_spread, "aggregation_method": self.aggregation, "num_cves": len(vuln_responses), - "total_responses": sum(len(responses) for responses in vuln_responses.values()) + "total_responses": sum(len(responses) for responses in vuln_responses.values()), + "test_set_id": item.full_dataset_entry.get('test_set_id', 'unknown'), } return overall_spread, reasoning diff --git a/src/vuln_analysis/functions/eval_status_accuracy.py b/src/vuln_analysis/eval/eval_status_accuracy.py similarity index 98% rename from src/vuln_analysis/functions/eval_status_accuracy.py rename to src/vuln_analysis/eval/eval_status_accuracy.py index 712c4cf8b..ac3bd7cd4 100644 --- a/src/vuln_analysis/functions/eval_status_accuracy.py +++ b/src/vuln_analysis/eval/eval_status_accuracy.py @@ -62,11 +62,11 @@ async def process_item(item): # Calculate average accuracy status_avg = sum(status_accuracies)/len(status_accuracies) if status_accuracies else 0.0 - reasoning = { "question": question, "status_answer": [ground_truth_map.get(q, 'MISSING') for q in question], "generated_status_answer": generated_status_answers, + "test_set_id": item.full_dataset_entry.get('test_set_id', 'unknown'), } return status_avg, reasoning diff --git a/src/vuln_analysis/eval/evaluator_register.py b/src/vuln_analysis/eval/evaluator_register.py new file mode 100644 index 000000000..bca554f53 --- /dev/null +++ b/src/vuln_analysis/eval/evaluator_register.py @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Register evaluator helper functions +from aiq.builder.builder import EvalBuilder +from aiq.builder.evaluator import EvaluatorInfo +from aiq.cli.register_workflow import register_evaluator +from aiq.data_models.evaluator import EvaluatorBaseConfig + + +#evaluator registration (actual code is in eval/eval_status_accuracy.py) +class StatusAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="status_accuracy"): + '''Configuration for status accuracy evaluator''' + pass + + +@register_evaluator(config_type=StatusAccuracyEvaluatorConfig) +async def status_accuracy_evaluator(config: StatusAccuracyEvaluatorConfig, builder: EvalBuilder): + '''Register status accuracy evaluator''' + from .eval_status_accuracy import StatusAccuracyEvaluator + evaluator = StatusAccuracyEvaluator(builder.get_max_concurrency()) + + yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Status Accuracy Evaluator") + + +#evaluator registration (actual code is in eval/eval_label_accuracy.py) +class LabelAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="label_accuracy"): + '''Configuration for label accuracy evaluator''' + pass + + +@register_evaluator(config_type=LabelAccuracyEvaluatorConfig) +async def label_accuracy_evaluator(config: LabelAccuracyEvaluatorConfig, builder: EvalBuilder): + '''Register label accuracy evaluator''' + from .eval_label_accuracy import LabelAccuracyEvaluator + evaluator = LabelAccuracyEvaluator(builder.get_max_concurrency()) + + yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Label Accuracy Evaluator") + + +#evaluator registration (actual code is in eval/eval_response_spread.py) +class ResponseSpreadEvaluatorConfig(EvaluatorBaseConfig, name="response_spread"): + '''Configuration for response spread evaluator''' + pass + + +@register_evaluator(config_type=ResponseSpreadEvaluatorConfig) +async def response_spread_evaluator(config: ResponseSpreadEvaluatorConfig, builder: EvalBuilder): + '''Register response spread evaluator''' + from .eval_response_spread import ResponseSpreadEvaluator + evaluator = ResponseSpreadEvaluator(builder.get_max_concurrency()) + + yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Response Spread Evaluator") + + +#evaluator registration (actual code is in eval/eval_consistency.py) +class ConsistencyEvaluatorConfig(EvaluatorBaseConfig, name="consistency"): + '''Configuration for consistency evaluator''' + pass + + +@register_evaluator(config_type=ConsistencyEvaluatorConfig) +async def consistency_evaluator(config: ConsistencyEvaluatorConfig, builder: EvalBuilder): + '''Register consistency evaluator''' + from .eval_consistency import ConsistencyEvaluator + evaluator = ConsistencyEvaluator(builder.get_max_concurrency()) + + yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Consistency Evaluator") diff --git a/src/vuln_analysis/eval/parse_eval_input.py b/src/vuln_analysis/eval/parse_eval_input.py new file mode 100644 index 000000000..77fb4d5a5 --- /dev/null +++ b/src/vuln_analysis/eval/parse_eval_input.py @@ -0,0 +1,212 @@ +import json +from pathlib import Path + +from nat.eval.evaluator.evaluator_model import EvalInput +from nat.eval.evaluator.evaluator_model import EvalInputItem + + +def parse_input(file_path: Path, test_set_id: str | None = None, container_ids: list[str] | None = None, max_test_sets: int | None = None) -> EvalInput: + """ + To run just this script outside of the nat pipeline: + python3 parse_eval_input.py eval-dataset.json --test-set-id test-set-small --container-ids morpheus-23.11 --preview -o my_eval.json + Note: need to run inside vuln_analysis docker container where nat is installed + """ + # Load the container image config JSON + with open(file_path, 'r', encoding='utf-8') as f: + config_data = json.load(f) + + # Extract containers + containers = config_data.get('containers', {}) + + if not containers: + print("No containers found in config file") + return EvalInput(eval_input_items=[]) + + # Filter containers if specific IDs provided + if container_ids is not None: + available_containers = {k: v for k, v in containers.items() if k in container_ids} + if not available_containers: + print(f"No containers found with IDs: {container_ids}") + print(f"Available containers: {list(containers.keys())}") + return EvalInput(eval_input_items=[]) + else: + available_containers = containers + print(f"No container IDs specified, using all available containers: {list(containers.keys())}") + + # Find test set across all containers (test set must exist in all containers) + if test_set_id is not None: + # Check if test set exists in ALL specified containers + missing_containers = [] + for container_id, container_data in available_containers.items(): + test_sets = container_data.get('test_sets', {}) + if test_set_id not in test_sets: + missing_containers.append(container_id) + + if missing_containers: + print(f"Test set '{test_set_id}' not found in containers: {missing_containers}") + print("Available test sets across all containers:") + for container_id, container_data in available_containers.items(): + test_sets = container_data.get('test_sets', {}) + print(f" {container_id}: {list(test_sets.keys())}") + return EvalInput(eval_input_items=[]) + + print(f"Processing test set: {test_set_id}") + else: + print("Error: No test set ID specified") + print("Available test sets per container:") + for container_id, container_data in available_containers.items(): + test_sets = container_data.get('test_sets', {}) + print(f" {container_id}: {list(test_sets.keys())}") + return EvalInput(eval_input_items=[]) + + # Create evaluation items for each container + eval_items = [] + + for container_id, container_data in available_containers.items(): + container_image = container_data.get('container_image', {}) + test_sets = container_data.get('test_sets', {}) + + if not container_image: + print(f"No container_image found for container: {container_id}") + continue + + # Get the test set (we already verified it exists in all containers) + target_test_set = test_sets[test_set_id] + ground_truth = target_test_set.get('ground_truth', []) + + # Create scan structure with vulnerability IDs from ground truth + scan_vulns = [] + ground_truth_mapping = {} + + for gt_item in ground_truth: + vuln_id = gt_item.get('vuln_id') + status = gt_item.get('status') + label = gt_item.get('label') + + if vuln_id and status: + # Add to scan structure + scan_vulns.append({"vuln_id": vuln_id}) + + # Add to ground truth mapping + ground_truth_mapping[vuln_id] = status + + # Add label to ground truth mapping if available + if label: + ground_truth_mapping[f"{vuln_id}_label"] = label + + # Create the question structure (container_image + scan) + question_structure = { + "image": container_image, + "scan": { + "vulns": scan_vulns + } + } + + # Convert to stringified format + question_string = json.dumps(question_structure, separators=(',', ':')) + answer_string = json.dumps(ground_truth_mapping, separators=(',', ':')) + + # Create evaluation item + eval_item = EvalInputItem( + id=container_id, + input_obj=question_string, + expected_output_obj=answer_string, + full_dataset_entry={ + "id": container_id, + "question": question_string, + "answer": answer_string, + "test_set_id": test_set_id, + "container_id": container_id, + "original_test_set": target_test_set + } + ) + + eval_items.append(eval_item) + + if not eval_items: + print("No valid evaluation items created") + return EvalInput(eval_input_items=[]) + + print(f"Created {len(eval_items)} evaluation items for test set '{test_set_id}'") + return EvalInput(eval_input_items=eval_items) + + +# For if you want to look at the eval input string +def save_as_baseline_string_format(eval_input: EvalInput, output_path: Path) -> None: + baseline_data = [] + + for item in eval_input.eval_input_items: + baseline_item = { + "id": item.id, + "question": item.input_obj, + "answer": item.expected_output_obj + } + baseline_data.append(baseline_item) + + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(baseline_data, f, indent=4) + + print(f"Saved baseline string format to: {output_path}") + + +def main(): + import argparse + + parser = argparse.ArgumentParser( + description='Parse consolidated evaluation dataset with multiple containers and test sets', + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument('input_file', help='Path to eval-dataset.json file') + parser.add_argument('-o', '--output', help='Output path for baseline string format (optional)') + parser.add_argument('--test-set-id', help='Specific test set ID to process (e.g., --test-set-id test-set-small)') + parser.add_argument('--container-ids', nargs='+', help='Specific container IDs to process (e.g., --container-ids morpheus-23.11 morpheus-24.03)') + parser.add_argument('--max-test-sets', type=int, help='Maximum number of test sets to process') + parser.add_argument('--preview', action='store_true', help='Show preview of first converted item') + + args = parser.parse_args() + + try: + input_path = Path(args.input_file) + + # Convert to EvalInput format + eval_input = parse_input(input_path, args.test_set_id, args.container_ids, args.max_test_sets) + + if args.preview and eval_input.eval_input_items: + print(f"Total evaluation items: {len(eval_input.eval_input_items)}") + for i, item in enumerate(eval_input.eval_input_items[:3]): # Show first 3 items + print(f"\n--- Item {i} ---") + print(f"ID: {item.id}") + print(f"Container ID: {item.full_dataset_entry.get('container_id', 'unknown')}") + print(f"Test Set ID: {item.full_dataset_entry.get('test_set_id', 'unknown')}") + print(f"Question (first 200 chars):") + print(f"{item.input_obj[:200]}...") + print(f"Answer:") + print(f"{item.expected_output_obj}") + print("="*80) + + # Save as baseline string format if output specified + if args.output: + output_path = Path(args.output) + save_as_baseline_string_format(eval_input, output_path) + else: + # Default output path in eval directory + eval_dir = Path(__file__).parent + default_output = eval_dir / "eval_input_string.json" + save_as_baseline_string_format(eval_input, default_output) + + + except FileNotFoundError as e: + print(f"Error: Input file not found - {e}") + return 1 + except json.JSONDecodeError as e: + print(f"Error: Invalid JSON in input file - {e}") + return 1 + except Exception as e: + print(f"Unexpected error: {e}") + return 1 + + return 0 + + +if __name__ == "__main__": + exit(main()) \ No newline at end of file diff --git a/src/vuln_analysis/functions/parse_eval_input.py b/src/vuln_analysis/functions/parse_eval_input.py deleted file mode 100644 index 74e280140..000000000 --- a/src/vuln_analysis/functions/parse_eval_input.py +++ /dev/null @@ -1,169 +0,0 @@ -import json -from pathlib import Path - -from nat.eval.evaluator.evaluator_model import EvalInput -from nat.eval.evaluator.evaluator_model import EvalInputItem - - -def parse_input(file_path: Path, test_set_id: str | None = None, max_test_sets: int | None = None) -> EvalInput: - """ - To run just this script outside of the nat pipeline: python3 parse_eval_input.py custom_config.json --test-set-id workflow0 --preview -o my_eval.json - Note: need to run inside vuln_analysis docker container where nat is installed - """ - # Load the container image config JSON - with open(file_path, 'r', encoding='utf-8') as f: - config_data = json.load(f) - - # Extract container image and test sets - container_image = config_data.get('container_image', {}) - test_sets = config_data.get('test_set', {}) - - if not container_image: - print("No container_image found in config file") - return EvalInput(eval_input_items=[]) - - if not test_sets: - print("No test sets found in config file") - return EvalInput(eval_input_items=[]) - - # Filter by test set ID if specified - if test_set_id is not None: - if test_set_id not in test_sets: - print(f"No test set found with ID: {test_set_id}") - print(f"Available test sets: {list(test_sets.keys())}") - return EvalInput(eval_input_items=[]) - - target_test_set = test_sets[test_set_id] - print(f"Processing test set: {test_set_id}") - else: - # Use first test set if no specific ID specified - test_set_id = list(test_sets.keys())[0] - target_test_set = test_sets[test_set_id] - print(f"No test set ID specified, using first available: {test_set_id}") - - # Process the test set - ground_truth = target_test_set.get('ground_truth', []) - - # Create scan structure with vulnerability IDs from ground truth - scan_vulns = [] - ground_truth_mapping = {} - - for gt_item in ground_truth: - vuln_id = gt_item.get('vuln_id') - status = gt_item.get('status') - label = gt_item.get('label') - - if vuln_id and status: - # Add to scan structure - scan_vulns.append({"vuln_id": vuln_id}) - - # Add to ground truth mapping - ground_truth_mapping[vuln_id] = status - - # Add label to ground truth mapping if available - if label: - ground_truth_mapping[f"{vuln_id}_label"] = label - - # Create the question structure (container_image + scan) - question_structure = { - "image": container_image, - "scan": { - "vulns": scan_vulns - } - } - - # Convert to stringified format - question_string = json.dumps(question_structure, separators=(',', ':')) - answer_string = json.dumps(ground_truth_mapping, separators=(',', ':')) - - # Create evaluation item - eval_item = EvalInputItem( - id=0, - input_obj=question_string, - expected_output_obj=answer_string, - full_dataset_entry={ - "id": 0, - "question": question_string, - "answer": answer_string, - "test_set_id": test_set_id, - "original_test_set": target_test_set - } - ) - - return EvalInput(eval_input_items=[eval_item]) - - -# For if you want to look at the eval input string -def save_as_baseline_string_format(eval_input: EvalInput, output_path: Path) -> None: - baseline_data = [] - - for item in eval_input.eval_input_items: - baseline_item = { - "id": item.id, - "question": item.input_obj, - "answer": item.expected_output_obj - } - baseline_data.append(baseline_item) - - with open(output_path, 'w', encoding='utf-8') as f: - json.dump(baseline_data, f, indent=4) - - print(f"Saved baseline string format to: {output_path}") - - -def main(): - import argparse - - parser = argparse.ArgumentParser( - description='Convert better_config.json to evaluation format', - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument('input_file', help='Path to better_config.json file') - parser.add_argument('-o', '--output', help='Output path for baseline string format (optional)') - parser.add_argument('--test-set-id', help='Specific test set ID to process (e.g., --test-set-id workflow0)') - parser.add_argument('--max-test-sets', type=int, help='Maximum number of test sets to process') - parser.add_argument('--preview', action='store_true', help='Show preview of first converted item') - - args = parser.parse_args() - - try: - input_path = Path(args.input_file) - - # Convert to EvalInput format - eval_input = parse_input(input_path, args.test_set_id, args.max_test_sets) - - if args.preview and eval_input.eval_input_items: - first_item = eval_input.eval_input_items[0] - print(f"ID: {first_item.id}") - print(f"\nQuestion (first 200 chars):") - print(f"{first_item.input_obj[:200]}...") - print(f"\nAnswer:") - print(f"{first_item.expected_output_obj}") - print("="*80) - - # Save as baseline string format if output specified - if args.output: - output_path = Path(args.output) - save_as_baseline_string_format(eval_input, output_path) - else: - # Default output path in eval directory - eval_dir = Path(__file__).parent - default_output = eval_dir / "converted_baseline_string.json" - save_as_baseline_string_format(eval_input, default_output) - - - except FileNotFoundError as e: - print(f"Error: Input file not found - {e}") - return 1 - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON in input file - {e}") - return 1 - except Exception as e: - print(f"Unexpected error: {e}") - return 1 - - return 0 - - -if __name__ == "__main__": - exit(main()) \ No newline at end of file diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index e6a76566f..17f62369f 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -46,13 +46,8 @@ from vuln_analysis.utils.llm_engine_utils import postprocess_engine_output from vuln_analysis.utils.llm_engine_utils import preprocess_engine_input - -# Register evaluator helper functions -from aiq.builder.builder import EvalBuilder -from aiq.builder.evaluator import EvaluatorInfo -from aiq.cli.register_workflow import register_evaluator -from aiq.data_models.evaluator import EvaluatorBaseConfig - +# Import evaluator registrations +from vuln_analysis.eval import evaluator_register logger = logging.getLogger(__name__) @@ -77,66 +72,6 @@ class CVEAgentWorkflowConfig(FunctionBaseConfig, name="cve_agent"): description="Workflow function description") -#evaluator registration (actual code is in functions/eval_status_accuracy.py) -class StatusAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="status_accuracy"): - '''Configuration for status accuracy evaluator''' - pass - - -@register_evaluator(config_type=StatusAccuracyEvaluatorConfig) -async def status_accuracy_evaluator(config: StatusAccuracyEvaluatorConfig, builder: EvalBuilder): - '''Register status accuracy evaluator''' - from .functions.eval_status_accuracy import StatusAccuracyEvaluator - evaluator = StatusAccuracyEvaluator(builder.get_max_concurrency()) - - yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Status Accuracy Evaluator") - - -#evaluator registration (actual code is in functions/eval_label_accuracy.py) -class LabelAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="label_accuracy"): - '''Configuration for label accuracy evaluator''' - pass - - -@register_evaluator(config_type=LabelAccuracyEvaluatorConfig) -async def label_accuracy_evaluator(config: LabelAccuracyEvaluatorConfig, builder: EvalBuilder): - '''Register label accuracy evaluator''' - from .functions.eval_label_accuracy import LabelAccuracyEvaluator - evaluator = LabelAccuracyEvaluator(builder.get_max_concurrency()) - - yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Label Accuracy Evaluator") - - -#evaluator registration (actual code is in functions/eval_response_spread.py) -class ResponseSpreadEvaluatorConfig(EvaluatorBaseConfig, name="response_spread"): - '''Configuration for response spread evaluator''' - pass - - -@register_evaluator(config_type=ResponseSpreadEvaluatorConfig) -async def response_spread_evaluator(config: ResponseSpreadEvaluatorConfig, builder: EvalBuilder): - '''Register response spread evaluator''' - from .functions.eval_response_spread import ResponseSpreadEvaluator - evaluator = ResponseSpreadEvaluator(builder.get_max_concurrency()) - - yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Response Spread Evaluator") - - -#evaluator registration (actual code is in functions/eval_consistency.py) -class ConsistencyEvaluatorConfig(EvaluatorBaseConfig, name="consistency"): - '''Configuration for consistency evaluator''' - pass - - -@register_evaluator(config_type=ConsistencyEvaluatorConfig) -async def consistency_evaluator(config: ConsistencyEvaluatorConfig, builder: EvalBuilder): - '''Register consistency evaluator''' - from .functions.eval_consistency import ConsistencyEvaluator - evaluator = ConsistencyEvaluator(builder.get_max_concurrency()) - - yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Consistency Evaluator") - - @register_function(config_type=CVEAgentWorkflowConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def cve_agent_workflow(config: CVEAgentWorkflowConfig, builder: Builder): From 6e4f1458ec332d198e0ac901ff2c9da5360cf5a0 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Sun, 7 Sep 2025 18:24:53 -0700 Subject: [PATCH 017/104] fix readme description for within-experiment custom evaluators --- README.md | 13 +++++++------ src/vuln_analysis/configs/config-eval.yml | 13 +++++++------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index b4ced277b..88c1e41f9 100644 --- a/README.md +++ b/README.md @@ -833,14 +833,15 @@ For quick testing, you can also change configuration at runtime using the `--ove nat eval --config_file=configs/config-eval.yml --reps=3 --override llms.checklist_llm.base_url https://integrate.api.nvidia.com/v1 ``` -### Additional Evaluators +#### Evaluation Metrics for Multiple Runs +Below are two example evaluation metrics for evaluating response spread and consistency across multiple evaluation runs. Uncomment the last section in `config-eval.yml` to enable these evaluators. You can view their source code in the `eval` directory. -WIP: i need to make these descriptions more clear -- **Measure Response Spread:** For multiple reps (you can technically run it with one rep but the spread will just be perfect score of 1). Calculates a response spread across all the reps, per CVE. Outputs a response spread for each CVE, and also provides an average response spread across all CVEs. Response spread is from 0 to 1, with 1 being perfect match (no spread). It's calculated based on the natural language summary field. It's calculated via creating a bag of words (word frequency vector) and calculating the cosine similarity between the centroid and the response produced by each rep. It then uses the configured aggregation method (average, min, max) to produce the final output. -- **Measure Consistency:** For multiple reps (you can technically run it with one rep but the consistency will be perfect score of 1). Calculates consistency on status and labels per CVE. Consistency is captured by 'alikeablity' which is calculated by this formula: Alikeability = sum(p^2) where p is the probability of each unique value. So if you did 3 reps and had labels: {vulnerable, code_not_reachable, code_not_reachable}, you would have two p's, causing the probabilities array to be {1/3, 2/3}. Taking the square of each of those values and then summing them: (1/3)^2 + (2/3)^2 = 0.56, which is a mid range consistency. Notice that if you had a single p (all label values were the same), your consistency would be 1.0, or a perfect consistency. This evaluator outputs two consistency scores per CVE (one for status, one for labels), as well as the average status consistency score, the average label consistency score, and the overall combined average of the two averages. +You can write similar evaluators to calculate any desired custom metrics; read more at the NeMo Agent Toolkit documentation [custom evaluators](https://docs.nvidia.com/nemo/agent-toolkit/1.2/reference/evaluate.html#adding-custom-evaluators). -These are defined in `eval` dir and registered in `evaluator_registry.py` -You can write an evaluator for your desired metrics via writing a custom evaluator (link NAT docs) + +- **Measure Response Spread:** Measures how similar the response summary field is across multiple runs for the same CVE. The summary field is in natural language; this evaluator converts each response into a bag-of-words frequency vector, then computes cosine similarity between each run and the centroid of all runs. Results are aggregated via a selected aggregation method (average, min, or max) configured in `config-eval.yml`. The final output is a spread score from 0 to 1 for each CVE, with 1 indicating no variation across runs and 0 indicating maximum variation. + +- **Measure Consistency:** Measures how consistent the model is in categorical outputs (status and label) across multiple runs for the same CVE. Computes a custom score to capture consistency, which we define as the sum of p², where p is the probability of each unique value. For example, if labels for one CVE across 3 runs are `{vulnerable, code_not_reachable, code_not_reachable}`, then the probabilities are `{1/3, 2/3}`, and the consistency score is `(1/3)² + (2/3)² = 0.56`. A score of **1.0** means perfect consistency (all runs yielded exact same values). The final output is a per-CVE consistency score for status and a per-CVE consistency score for labels. ## Troubleshooting diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml index 9c0c81a23..23097da5d 100644 --- a/src/vuln_analysis/configs/config-eval.yml +++ b/src/vuln_analysis/configs/config-eval.yml @@ -160,9 +160,10 @@ eval: _type: status_accuracy label_accuracy_eval: _type: label_accuracy - response_spread_eval: - _type: response_spread - kwargs: - aggregation: "average" #min, max - consistency_eval: - _type: consistency \ No newline at end of file + # Uncomment to enable response spread and consistency evaluators + # response_spread_eval: + # _type: response_spread + # kwargs: + # aggregation: "average" #min, max + # consistency_eval: + # _type: consistency \ No newline at end of file From 26a257f671e441889ac61f6dbe7d2734287323de Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Sun, 7 Sep 2025 18:26:29 -0700 Subject: [PATCH 018/104] readme typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 88c1e41f9..c9493e9c7 100644 --- a/README.md +++ b/README.md @@ -836,7 +836,7 @@ nat eval --config_file=configs/config-eval.yml --reps=3 --override llms.checklis #### Evaluation Metrics for Multiple Runs Below are two example evaluation metrics for evaluating response spread and consistency across multiple evaluation runs. Uncomment the last section in `config-eval.yml` to enable these evaluators. You can view their source code in the `eval` directory. -You can write similar evaluators to calculate any desired custom metrics; read more at the NeMo Agent Toolkit documentation [custom evaluators](https://docs.nvidia.com/nemo/agent-toolkit/1.2/reference/evaluate.html#adding-custom-evaluators). +You can write similar evaluators to calculate any desired custom metrics; read more at the NeMo Agent Toolkit documentation for [custom evaluators](https://docs.nvidia.com/nemo/agent-toolkit/1.2/reference/evaluate.html#adding-custom-evaluators). - **Measure Response Spread:** Measures how similar the response summary field is across multiple runs for the same CVE. The summary field is in natural language; this evaluator converts each response into a bag-of-words frequency vector, then computes cosine similarity between each run and the centroid of all runs. Results are aggregated via a selected aggregation method (average, min, or max) configured in `config-eval.yml`. The final output is a spread score from 0 to 1 for each CVE, with 1 indicating no variation across runs and 0 indicating maximum variation. From c7ad0f6245312b72adec1a21c8439d24e7011fd3 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Sun, 7 Sep 2025 18:27:41 -0700 Subject: [PATCH 019/104] readme typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c9493e9c7..7706f4886 100644 --- a/README.md +++ b/README.md @@ -841,7 +841,7 @@ You can write similar evaluators to calculate any desired custom metrics; read m - **Measure Response Spread:** Measures how similar the response summary field is across multiple runs for the same CVE. The summary field is in natural language; this evaluator converts each response into a bag-of-words frequency vector, then computes cosine similarity between each run and the centroid of all runs. Results are aggregated via a selected aggregation method (average, min, or max) configured in `config-eval.yml`. The final output is a spread score from 0 to 1 for each CVE, with 1 indicating no variation across runs and 0 indicating maximum variation. -- **Measure Consistency:** Measures how consistent the model is in categorical outputs (status and label) across multiple runs for the same CVE. Computes a custom score to capture consistency, which we define as the sum of p², where p is the probability of each unique value. For example, if labels for one CVE across 3 runs are `{vulnerable, code_not_reachable, code_not_reachable}`, then the probabilities are `{1/3, 2/3}`, and the consistency score is `(1/3)² + (2/3)² = 0.56`. A score of **1.0** means perfect consistency (all runs yielded exact same values). The final output is a per-CVE consistency score for status and a per-CVE consistency score for labels. +- **Measure Consistency:** Measures how consistent the model is in categorical outputs (status and label) across multiple runs for the same CVE. Computes a custom score to capture consistency, which we define as the sum of p², where p is the probability of each unique value. For example, if labels for one CVE across 3 runs are `{vulnerable, code_not_reachable, code_not_reachable}`, then the probabilities are `{1/3, 2/3}`, and the consistency score is `(1/3)² + (2/3)² = 0.56`. A score of 1.0 means perfect consistency (all runs yielded exact same values). The final output is a per-CVE consistency score for status and a per-CVE consistency score for labels. ## Troubleshooting From 6275d523890fd4386ccaf44d7ded2f207e01a0c1 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Sun, 7 Sep 2025 18:29:52 -0700 Subject: [PATCH 020/104] removed alikeability word from comments, calling it consistency instead --- src/vuln_analysis/eval/eval_consistency.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/vuln_analysis/eval/eval_consistency.py b/src/vuln_analysis/eval/eval_consistency.py index 2ced13614..6d61cd4b7 100644 --- a/src/vuln_analysis/eval/eval_consistency.py +++ b/src/vuln_analysis/eval/eval_consistency.py @@ -23,7 +23,7 @@ class ConsistencyEvaluator: - '''Evaluator class for measuring consistency using alikeability metric''' + '''Evaluator class for measuring consistency using custom metric''' def __init__(self, max_concurrency: int): self.max_concurrency = max_concurrency @@ -51,7 +51,7 @@ async def process_item(item): vuln_responses[vuln_id]['labels'].append(label) vuln_responses[vuln_id]['statuses'].append(status) - # Calculate consistency for each CVE using alikeability (entropy-based) + # Calculate consistency for each CVE label_consistencies = {} status_consistencies = {} all_label_consistencies = [] @@ -61,7 +61,7 @@ async def process_item(item): # Calculate label consistency labels = responses['labels'] if len(labels) > 1: - # Alikeability calculation: sum of squared probabilities + # Consistency calculation: sum of squared probabilities label_counts = Counter(labels) n = len(labels) probs = [count / n for count in label_counts.values()] @@ -75,7 +75,7 @@ async def process_item(item): # Calculate status consistency statuses = responses['statuses'] if len(statuses) > 1: - # Alikeability calculation: sum of squared probabilities + # Consistency calculation: sum of squared probabilities status_counts = Counter(statuses) n = len(statuses) probs = [count / n for count in status_counts.values()] From d0a087ca25849bc49a82ca17dfb59296d3988578 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Sun, 7 Sep 2025 18:32:09 -0700 Subject: [PATCH 021/104] readme typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7706f4886..5ca19d87f 100644 --- a/README.md +++ b/README.md @@ -822,7 +822,7 @@ NVIDIA_API_BASE=https://integrate.api.nvidia.com/v1 nat eval --config_file=confi ``` ##### Option 2 - Disable Caching on Specific Calls -Users may want to disable caching for only specific parts of the pipeline. For example, more in-depth experiments may require disabling caching on specific LLMs. Or, you may want to disable all LLM call caching but keeping embedder caching so the embedder does not have to regenerate embeddings for the static image source every evaluation run. +Users may want to disable caching for only specific parts of the pipeline. For example, more in-depth experiments may require disabling caching on specific LLMs. Or, you may want to disable all LLM call caching but keep embedder caching so the embedder does not have to regenerate embeddings for the static image source every evaluation run. You can accomplish specific caching overrides in `config-eval.yml` by assigning the `base_url` variable of the target component with `"https://integrate.api.nvidia.com/v1"`. Note that you can create multiple config files to keep track of different evaluation experiments. From 59d25022cdf8bca7856e1fad1c8d51275403c3f5 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Sun, 7 Sep 2025 18:34:51 -0700 Subject: [PATCH 022/104] fix model endpoints --- src/vuln_analysis/configs/config-eval.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml index 23097da5d..81d9f9991 100644 --- a/src/vuln_analysis/configs/config-eval.yml +++ b/src/vuln_analysis/configs/config-eval.yml @@ -83,42 +83,42 @@ llms: checklist_llm: _type: nim base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} - model_name: ${CHECKLIST_MODEL_NAME:-nvdev/meta/llama-3.1-70b-instruct} + model_name: ${CHECKLIST_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 max_tokens: 2000 top_p: 0.01 code_vdb_retriever_llm: _type: nim base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} - model_name: ${CODE_VDB_RETRIEVER_MODEL_NAME:-nvdev/meta/llama-3.1-70b-instruct} + model_name: ${CODE_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 max_tokens: 2000 top_p: 0.01 doc_vdb_retriever_llm: _type: nim base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} - model_name: ${DOC_VDB_RETRIEVER_MODEL_NAME:-nvdev/meta/llama-3.1-70b-instruct} + model_name: ${DOC_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 max_tokens: 2000 top_p: 0.01 cve_agent_executor_llm: _type: nim base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} - model_name: ${CVE_AGENT_EXECUTOR_MODEL_NAME:-nvdev/meta/llama-3.1-70b-instruct} + model_name: ${CVE_AGENT_EXECUTOR_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 max_tokens: 2000 top_p: 0.01 summarize_llm: _type: nim base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} - model_name: ${SUMMARIZE_MODEL_NAME:-nvdev/meta/llama-3.1-70b-instruct} + model_name: ${SUMMARIZE_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 max_tokens: 1024 top_p: 0.01 justify_llm: _type: nim base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} - model_name: ${JUSTIFY_MODEL_NAME:-nvdev/meta/llama-3.1-70b-instruct} + model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 max_tokens: 1024 top_p: 0.01 @@ -127,7 +127,7 @@ embedders: nim_embedder: _type: nim base_url: ${NIM_EMBED_BASE_URL:-https://integrate.api.nvidia.com/v1} - model_name: ${EMBEDDER_MODEL_NAME:-nvdev/nvidia/nv-embedqa-e5-v5} + model_name: ${EMBEDDER_MODEL_NAME:-nvidia/nv-embedqa-e5-v5} truncate: END max_batch_size: 128 From fb21f6e4be8dddd7ebc438c7ef8a70c98fde6c8c Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Sun, 7 Sep 2025 18:42:21 -0700 Subject: [PATCH 023/104] change aiq imports to nat --- src/vuln_analysis/eval/evaluator_register.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/vuln_analysis/eval/evaluator_register.py b/src/vuln_analysis/eval/evaluator_register.py index bca554f53..d9f65aac1 100644 --- a/src/vuln_analysis/eval/evaluator_register.py +++ b/src/vuln_analysis/eval/evaluator_register.py @@ -14,10 +14,10 @@ # limitations under the License. # Register evaluator helper functions -from aiq.builder.builder import EvalBuilder -from aiq.builder.evaluator import EvaluatorInfo -from aiq.cli.register_workflow import register_evaluator -from aiq.data_models.evaluator import EvaluatorBaseConfig +from nat.builder.builder import EvalBuilder +from nat.builder.evaluator import EvaluatorInfo +from nat.cli.register_workflow import register_evaluator +from nat.data_models.evaluator import EvaluatorBaseConfig #evaluator registration (actual code is in eval/eval_status_accuracy.py) From 8397ef71afae181caa2a201a80e7ae4d0667d161 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Sun, 7 Sep 2025 18:44:07 -0700 Subject: [PATCH 024/104] stop tracking old_eval_datasets --- .../data/eval_datasets/old_eval_dataset.json | 111 ------------------ 1 file changed, 111 deletions(-) delete mode 100644 src/vuln_analysis/data/eval_datasets/old_eval_dataset.json diff --git a/src/vuln_analysis/data/eval_datasets/old_eval_dataset.json b/src/vuln_analysis/data/eval_datasets/old_eval_dataset.json deleted file mode 100644 index c48df82bd..000000000 --- a/src/vuln_analysis/data/eval_datasets/old_eval_dataset.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "container_image": { - "name": "nvcr.io/nvidia/morpheus/morpheus", - "tag": "23.11-runtime", - "source_info": [ - { - "type": "code", - "git_repo": "https://github.com/nv-morpheus/Morpheus.git", - "ref": "branch-23.11", - "include": [ - "**/*.cpp", - "**/*.cu", - "**/*.cuh", - "**/*.h", - "**/*.hpp", - "**/*.ipynb", - "**/*.py", - "**/*Dockerfile" - ], - "exclude": [ - "tests/**/*" - ] - }, - { - "type": "doc", - "git_repo": "https://github.com/nv-morpheus/Morpheus.git", - "ref": "branch-23.11", - "include": [ - "**/*.md", - "docs/**/*.rst" - ] - } - ], - "sbom_info": { - "_type": "file", - "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom" - } - }, - "test_set": { - "test-set0": { - "description": "Real ground truth dataset, yields 0.4 accuracy", - "ground_truth": [ - { - "vuln_id": "GHSA-3f63-hfp8-52jq", - "status": "NOT AFFECTED", - "label": "code_not_reachable" - }, - { - "vuln_id": "CVE-2023-36632", - "status": "AFFECTED", - "label": "vulnerable" - }, - { - "vuln_id": "GHSA-cxfr-5q3r-2rc2", - "status": "AFFECTED", - "label": "vulnerable" - }, - { - "vuln_id": "GHSA-554w-xh4j-8w64", - "status": "AFFECTED", - "label": "vulnerable" - }, - { - "vuln_id": "CVE-2023-51767", - "status": "AFFECTED", - "label": "vulnerable" - } - ] - }, - "test-set1": { - "description": "Toy dataset for testing, yields 1.0 accuracy", - "ground_truth": [ - { - "vuln_id": "GHSA-3f63-hfp8-52jq", - "status": "NOT AFFECTED", - "label": "code_not_reachable" - }, - { - "vuln_id": "CVE-2023-36632", - "status": "NOT AFFECTED", - "label": "code_not_reachable" - }, - { - "vuln_id": "GHSA-cxfr-5q3r-2rc2", - "status": "AFFECTED", - "label": "vulnerable" - }, - { - "vuln_id": "GHSA-554w-xh4j-8w64", - "status": "UNKNOWN", - "label": "unknown" - }, - { - "vuln_id": "CVE-2023-51767", - "status": "NOT AFFECTED", - "label": "code_not_reachable" - } - ] - }, - "test-set-small": { - "description": "only 1 cve", - "ground_truth": [ - { - "vuln_id": "GHSA-3f63-hfp8-52jq", - "status": "NOT AFFECTED", - "label": "code_not_reachable" - } - ] - } - } - } \ No newline at end of file From 6cf22496650aa86b7779ec1b83e61c8aeafd801f Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Sun, 7 Sep 2025 18:55:30 -0700 Subject: [PATCH 025/104] small changes to custom evaluator examples --- src/vuln_analysis/eval/eval_consistency.py | 26 +++--- .../eval/eval_response_spread.py | 82 +++++++++---------- 2 files changed, 51 insertions(+), 57 deletions(-) diff --git a/src/vuln_analysis/eval/eval_consistency.py b/src/vuln_analysis/eval/eval_consistency.py index 6d61cd4b7..2bfc7a38a 100644 --- a/src/vuln_analysis/eval/eval_consistency.py +++ b/src/vuln_analysis/eval/eval_consistency.py @@ -35,28 +35,28 @@ async def evaluate(self, eval_input: EvalInput) -> EvalOutput: async def process_item(item): """Calculate consistency for an individual item across multiple runs""" pipeline_result = json.loads(item.output_obj)['output'] - + # Group responses by vuln_id to get multiple responses per CVE vuln_responses = {} - + for result in pipeline_result: vuln_id = result['vuln_id'] - # We can measure consistency on different fields + # Measure consistency on label and status fields label = result.get('justification', {}).get('label', 'MISSING') status = result.get('justification', {}).get('status', 'MISSING') - + if vuln_id not in vuln_responses: vuln_responses[vuln_id] = {'labels': [], 'statuses': []} - + vuln_responses[vuln_id]['labels'].append(label) vuln_responses[vuln_id]['statuses'].append(status) - + # Calculate consistency for each CVE label_consistencies = {} status_consistencies = {} all_label_consistencies = [] all_status_consistencies = [] - + for vuln_id, responses in vuln_responses.items(): # Calculate label consistency labels = responses['labels'] @@ -68,10 +68,10 @@ async def process_item(item): label_consistency = sum(p * p for p in probs) else: label_consistency = 1.0 # Perfect consistency if only one response - + label_consistencies[vuln_id] = label_consistency all_label_consistencies.append(label_consistency) - + # Calculate status consistency statuses = responses['statuses'] if len(statuses) > 1: @@ -82,10 +82,10 @@ async def process_item(item): status_consistency = sum(p * p for p in probs) else: status_consistency = 1.0 # Perfect consistency if only one response - + status_consistencies[vuln_id] = status_consistency all_status_consistencies.append(status_consistency) - + # Calculate overall average consistency overall_label_consistency = sum(all_label_consistencies) / len(all_label_consistencies) if all_label_consistencies else 1.0 overall_status_consistency = sum(all_status_consistencies) / len(all_status_consistencies) if all_status_consistencies else 1.0 @@ -98,11 +98,11 @@ async def process_item(item): "total_responses": sum(len(responses['labels']) for responses in vuln_responses.values()), "test_set_id": item.full_dataset_entry.get('test_set_id', 'unknown'), } - + # Return average of label and status consistency overall_consistency = (overall_label_consistency + overall_status_consistency) / 2 return overall_consistency, reasoning - + async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: async with self.semaphore: result = await process_item(item) diff --git a/src/vuln_analysis/eval/eval_response_spread.py b/src/vuln_analysis/eval/eval_response_spread.py index e360a49b2..d4ca0547d 100644 --- a/src/vuln_analysis/eval/eval_response_spread.py +++ b/src/vuln_analysis/eval/eval_response_spread.py @@ -38,68 +38,62 @@ async def evaluate(self, eval_input: EvalInput) -> EvalOutput: async def process_item(item): """Calculate response spread for an individual item across multiple runs""" pipeline_result = json.loads(item.output_obj)['output'] - + # Group responses by vuln_id to get multiple responses per CVE vuln_responses = {} - + for result in pipeline_result: vuln_id = result['vuln_id'] summary = result.get('summary', '') - + if vuln_id not in vuln_responses: vuln_responses[vuln_id] = [] vuln_responses[vuln_id].append(summary) - + # Calculate response spread for each CVE response_spreads = {} all_spreads = [] - + for vuln_id, responses in vuln_responses.items(): if len(responses) < 2: # Need at least 2 responses to calculate spread response_spreads[vuln_id] = 1.0 # Perfect similarity if only one response all_spreads.append(1.0) continue - + # Calculate cosine similarity spread using bag-of-words - try: - vectorizer = CountVectorizer(stop_words=self.stopwords) - X = vectorizer.fit_transform(responses).toarray() - - if X.shape[0] < 2: - response_spreads[vuln_id] = 1.0 - all_spreads.append(1.0) - continue - - # Calculate centroid (average of all response vectors) - centroid = np.sum(X, axis=0) - - # Calculate cosine similarities between each response and centroid - dot_products = np.dot(X, centroid) - norms = np.linalg.norm(X, axis=1) * np.linalg.norm(centroid) - - # Avoid division by zero - norms = np.where(norms == 0, 1, norms) - cos_similarities = dot_products / norms - - # Apply aggregation - if self.aggregation == "average": - spread_score = float(np.average(cos_similarities)) - elif self.aggregation == "max": - spread_score = float(np.max(cos_similarities)) - elif self.aggregation == "min": - spread_score = float(np.min(cos_similarities)) - else: - spread_score = float(np.average(cos_similarities)) - - response_spreads[vuln_id] = spread_score - all_spreads.append(spread_score) - - except Exception as e: - # If calculation fails, default to 1.0 (perfect similarity) + vectorizer = CountVectorizer(stop_words=self.stopwords) + X = vectorizer.fit_transform(responses).toarray() + + if X.shape[0] < 2: response_spreads[vuln_id] = 1.0 all_spreads.append(1.0) - + continue + + # Calculate centroid (average of all response vectors) + centroid = np.sum(X, axis=0) + + # Calculate cosine similarities between each response and centroid + dot_products = np.dot(X, centroid) + norms = np.linalg.norm(X, axis=1) * np.linalg.norm(centroid) + + # Avoid division by zero + norms = np.where(norms == 0, 1, norms) + cos_similarities = dot_products / norms + + # Apply aggregation + if self.aggregation == "average": + spread_score = float(np.average(cos_similarities)) + elif self.aggregation == "max": + spread_score = float(np.max(cos_similarities)) + elif self.aggregation == "min": + spread_score = float(np.min(cos_similarities)) + else: + spread_score = float(np.average(cos_similarities)) + + response_spreads[vuln_id] = spread_score + all_spreads.append(spread_score) + # Calculate overall average spread overall_spread = sum(all_spreads) / len(all_spreads) if all_spreads else 1.0 reasoning = { @@ -110,9 +104,9 @@ async def process_item(item): "total_responses": sum(len(responses) for responses in vuln_responses.values()), "test_set_id": item.full_dataset_entry.get('test_set_id', 'unknown'), } - + return overall_spread, reasoning - + async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: async with self.semaphore: result = await process_item(item) From 1deaf9000f4d598446771b2bd5642905ba0c3393 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Wed, 10 Sep 2025 13:17:56 -0700 Subject: [PATCH 026/104] slight file structure and naming changes based on pr comments --- src/vuln_analysis/configs/config-eval.yml | 27 ++-- src/vuln_analysis/configs/config-tracing.yml | 2 +- src/vuln_analysis/configs/config.yml | 2 +- .../data/eval_datasets/eval_dataset.json | 2 +- src/vuln_analysis/eval/eval_consistency.py | 129 ----------------- src/vuln_analysis/eval/eval_label_accuracy.py | 89 ------------ .../eval/eval_response_spread.py | 133 ------------------ .../eval/eval_status_accuracy.py | 97 ------------- src/vuln_analysis/eval/evaluator_register.py | 25 ++-- src/vuln_analysis/eval/parse_eval_input.py | 2 +- src/vuln_analysis/register.py | 2 +- 11 files changed, 36 insertions(+), 474 deletions(-) delete mode 100644 src/vuln_analysis/eval/eval_consistency.py delete mode 100644 src/vuln_analysis/eval/eval_label_accuracy.py delete mode 100644 src/vuln_analysis/eval/eval_response_spread.py delete mode 100644 src/vuln_analysis/eval/eval_status_accuracy.py diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml index 81d9f9991..d44448177 100644 --- a/src/vuln_analysis/configs/config-eval.yml +++ b/src/vuln_analysis/configs/config-eval.yml @@ -15,6 +15,11 @@ general: use_uvloop: true + telemetry: + tracing: + weave: + _type: weave + project: "cve-agent" functions: cve_generate_vdbs: @@ -145,25 +150,25 @@ workflow: eval: general: - output_dir: ./.tmp/eval/my_eval + workflow_alias: "first_w&b_run" + output_dir: ./.tmp/eval/my_eval4 dataset: _type: custom file_path: data/eval_datasets/eval_dataset.json function: vuln_analysis.eval.parse_eval_input.parse_input kwargs: test_set_id: "test-set-common" - container_ids: ["morpheus-23.11", "morpheus-24.03"] + container_ids: ["morpheus-23.11"] #["morpheus-23.11", "morpheus-24.03"] # comment out container_ids to use all available containers - + evaluators: - status_accuracy_eval: + status_accuracy: _type: status_accuracy - label_accuracy_eval: + label_accuracy: _type: label_accuracy # Uncomment to enable response spread and consistency evaluators - # response_spread_eval: - # _type: response_spread - # kwargs: - # aggregation: "average" #min, max - # consistency_eval: - # _type: consistency \ No newline at end of file + response_spread: + _type: response_spread + aggregation: "average" #Options: average, min, max + consistency: + _type: consistency diff --git a/src/vuln_analysis/configs/config-tracing.yml b/src/vuln_analysis/configs/config-tracing.yml index d6503f1d9..888077a14 100644 --- a/src/vuln_analysis/configs/config-tracing.yml +++ b/src/vuln_analysis/configs/config-tracing.yml @@ -158,7 +158,7 @@ eval: output_dir: ./.tmp/eval/cve_agent dataset: _type: json - file_path: data/eval_datasets/eval_dataset.json + file_path: data/eval_datasets/profiler_dataset.json profiler: token_uniqueness_forecast: true diff --git a/src/vuln_analysis/configs/config.yml b/src/vuln_analysis/configs/config.yml index cc1e66371..385c9a575 100644 --- a/src/vuln_analysis/configs/config.yml +++ b/src/vuln_analysis/configs/config.yml @@ -148,7 +148,7 @@ eval: output_dir: ./.tmp/eval/cve_agent dataset: _type: json - file_path: data/eval_datasets/eval_dataset.json + file_path: data/eval_datasets/profiler_dataset.json profiler: token_uniqueness_forecast: true diff --git a/src/vuln_analysis/data/eval_datasets/eval_dataset.json b/src/vuln_analysis/data/eval_datasets/eval_dataset.json index fb3b6056c..5142acdee 100644 --- a/src/vuln_analysis/data/eval_datasets/eval_dataset.json +++ b/src/vuln_analysis/data/eval_datasets/eval_dataset.json @@ -178,4 +178,4 @@ } } } -} \ No newline at end of file +} diff --git a/src/vuln_analysis/eval/eval_consistency.py b/src/vuln_analysis/eval/eval_consistency.py deleted file mode 100644 index 2bfc7a38a..000000000 --- a/src/vuln_analysis/eval/eval_consistency.py +++ /dev/null @@ -1,129 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: LicenseRef-NvidiaProprietary -# -# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual -# property and proprietary rights in and to this material, related -# documentation and any modifications thereto. Any use, reproduction, -# disclosure or distribution of this material and related documentation -# without an express license agreement from NVIDIA CORPORATION or -# its affiliates is strictly prohibited. - -import asyncio -import json -import numpy as np -from collections import Counter - -from tqdm import tqdm - -from nat.eval.evaluator.evaluator_model import EvalInput -from nat.eval.evaluator.evaluator_model import EvalOutput -from nat.eval.evaluator.evaluator_model import EvalOutputItem -from nat.eval.evaluator.evaluator_model import EvalInputItem -from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry - - -class ConsistencyEvaluator: - '''Evaluator class for measuring consistency using custom metric''' - - def __init__(self, max_concurrency: int): - self.max_concurrency = max_concurrency - self.semaphore = asyncio.Semaphore(self.max_concurrency) - - async def evaluate(self, eval_input: EvalInput) -> EvalOutput: - '''Evaluate consistency across multiple runs''' - - async def process_item(item): - """Calculate consistency for an individual item across multiple runs""" - pipeline_result = json.loads(item.output_obj)['output'] - - # Group responses by vuln_id to get multiple responses per CVE - vuln_responses = {} - - for result in pipeline_result: - vuln_id = result['vuln_id'] - # Measure consistency on label and status fields - label = result.get('justification', {}).get('label', 'MISSING') - status = result.get('justification', {}).get('status', 'MISSING') - - if vuln_id not in vuln_responses: - vuln_responses[vuln_id] = {'labels': [], 'statuses': []} - - vuln_responses[vuln_id]['labels'].append(label) - vuln_responses[vuln_id]['statuses'].append(status) - - # Calculate consistency for each CVE - label_consistencies = {} - status_consistencies = {} - all_label_consistencies = [] - all_status_consistencies = [] - - for vuln_id, responses in vuln_responses.items(): - # Calculate label consistency - labels = responses['labels'] - if len(labels) > 1: - # Consistency calculation: sum of squared probabilities - label_counts = Counter(labels) - n = len(labels) - probs = [count / n for count in label_counts.values()] - label_consistency = sum(p * p for p in probs) - else: - label_consistency = 1.0 # Perfect consistency if only one response - - label_consistencies[vuln_id] = label_consistency - all_label_consistencies.append(label_consistency) - - # Calculate status consistency - statuses = responses['statuses'] - if len(statuses) > 1: - # Consistency calculation: sum of squared probabilities - status_counts = Counter(statuses) - n = len(statuses) - probs = [count / n for count in status_counts.values()] - status_consistency = sum(p * p for p in probs) - else: - status_consistency = 1.0 # Perfect consistency if only one response - - status_consistencies[vuln_id] = status_consistency - all_status_consistencies.append(status_consistency) - - # Calculate overall average consistency - overall_label_consistency = sum(all_label_consistencies) / len(all_label_consistencies) if all_label_consistencies else 1.0 - overall_status_consistency = sum(all_status_consistencies) / len(all_status_consistencies) if all_status_consistencies else 1.0 - reasoning = { - "label_consistencies": label_consistencies, - "status_consistencies": status_consistencies, - "overall_label_consistency": overall_label_consistency, - "overall_status_consistency": overall_status_consistency, - "num_cves": len(vuln_responses), - "total_responses": sum(len(responses['labels']) for responses in vuln_responses.values()), - "test_set_id": item.full_dataset_entry.get('test_set_id', 'unknown'), - } - - # Return average of label and status consistency - overall_consistency = (overall_label_consistency + overall_status_consistency) / 2 - return overall_consistency, reasoning - - async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: - async with self.semaphore: - result = await process_item(item) - pbar.update(1) - return result - - try: - tqdm_position = TqdmPositionRegistry.claim() - pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Consistency", position=tqdm_position) - results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) - finally: - pbar.close() - TqdmPositionRegistry.release(tqdm_position) - - sample_scores, sample_reasonings = zip(*results) if results else ([], []) - - avg_score = round(sum(sample_scores) / len(sample_scores), 4) if sample_scores else 1.0 - - eval_output_items = [ - EvalOutputItem(id=item.id, score=score, reasoning=reasoning) - for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings) - ] - - return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) diff --git a/src/vuln_analysis/eval/eval_label_accuracy.py b/src/vuln_analysis/eval/eval_label_accuracy.py deleted file mode 100644 index 91a636ccc..000000000 --- a/src/vuln_analysis/eval/eval_label_accuracy.py +++ /dev/null @@ -1,89 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: LicenseRef-NvidiaProprietary -# -# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual -# property and proprietary rights in and to this material, related -# documentation and any modifications thereto. Any use, reproduction, -# disclosure or distribution of this material and related documentation -# without an express license agreement from NVIDIA CORPORATION or -# its affiliates is strictly prohibited. - -import asyncio -import json - -from tqdm import tqdm - -from nat.eval.evaluator.evaluator_model import EvalInput -from nat.eval.evaluator.evaluator_model import EvalOutput -from nat.eval.evaluator.evaluator_model import EvalOutputItem -from nat.eval.evaluator.evaluator_model import EvalInputItem -from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry - - -class LabelAccuracyEvaluator: - '''Evaluator class for justification labels accuracy''' - - def __init__(self, max_concurrency: int): - self.max_concurrency = max_concurrency - self.semaphore = asyncio.Semaphore(self.max_concurrency) - - async def evaluate(self, eval_input: EvalInput) -> EvalOutput: - '''Evaluate label accuracy''' - - async def process_item(item): - """Compute simple accuracy for an individual item""" - ground_truth_map = eval(item.expected_output_obj) - pipeline_result = json.loads(item.output_obj)['output'] - - label_accuracies = [] - generated_answers = [] - question = [] - - for result in pipeline_result: - vuln_id = result['vuln_id'] - question.append(vuln_id) - - # Extract label from pipeline result - justification_label = result.get('justification', {}).get('label', 'MISSING') - generated_answers.append(justification_label) - - # Calculate label accuracy - # Ground truth labels are stored as "{vuln_id}_label" in the ground truth map - # This is populated by parse_eval_input.py from the eval dataset JSON files - ground_truth_label = ground_truth_map.get(f"{vuln_id}_label") - if ground_truth_label is not None: - label_accuracies.append(float(justification_label == ground_truth_label)) - - reasoning = { - "question": question, - "answer": [ground_truth_map.get(f"{q}_label", 'MISSING') for q in question], - "generated_answer": generated_answers, - "test_set_id": item.full_dataset_entry.get('test_set_id', 'unknown'), - } - - return sum(label_accuracies)/len(label_accuracies) if label_accuracies else 0.0, reasoning - - async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: - async with self.semaphore: - result = await process_item(item) - pbar.update(1) - return result - - try: - tqdm_position = TqdmPositionRegistry.claim() - pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Label Accuracy", position=tqdm_position) - results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) - finally: - pbar.close() - TqdmPositionRegistry.release(tqdm_position) - - sample_scores, sample_reasonings = zip(*results) if results else ([], []) - - avg_score = round(sum(sample_scores) / len(sample_scores), 2) if sample_scores else 0.0 - - eval_output_items = [ - EvalOutputItem(id=item.id, score=score, reasoning=reasoning) - for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings) - ] - - return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) \ No newline at end of file diff --git a/src/vuln_analysis/eval/eval_response_spread.py b/src/vuln_analysis/eval/eval_response_spread.py deleted file mode 100644 index d4ca0547d..000000000 --- a/src/vuln_analysis/eval/eval_response_spread.py +++ /dev/null @@ -1,133 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: LicenseRef-NvidiaProprietary -# -# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual -# property and proprietary rights in and to this material, related -# documentation and any modifications thereto. Any use, reproduction, -# disclosure or distribution of this material and related documentation -# without an express license agreement from NVIDIA CORPORATION or -# its affiliates is strictly prohibited. - -import asyncio -import json -import numpy as np -from collections import Counter -from sklearn.feature_extraction.text import CountVectorizer - -from tqdm import tqdm - -from nat.eval.evaluator.evaluator_model import EvalInput -from nat.eval.evaluator.evaluator_model import EvalOutput -from nat.eval.evaluator.evaluator_model import EvalOutputItem -from nat.eval.evaluator.evaluator_model import EvalInputItem -from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry - - -class ResponseSpreadEvaluator: - '''Evaluator class for measuring response spread using cosine similarity''' - - def __init__(self, max_concurrency: int, aggregation: str = "average", stopwords: str = None): - self.max_concurrency = max_concurrency - self.semaphore = asyncio.Semaphore(self.max_concurrency) - self.aggregation = aggregation # "average", "min", or "max" - self.stopwords = stopwords - - async def evaluate(self, eval_input: EvalInput) -> EvalOutput: - '''Evaluate response spread across multiple runs''' - - async def process_item(item): - """Calculate response spread for an individual item across multiple runs""" - pipeline_result = json.loads(item.output_obj)['output'] - - # Group responses by vuln_id to get multiple responses per CVE - vuln_responses = {} - - for result in pipeline_result: - vuln_id = result['vuln_id'] - summary = result.get('summary', '') - - if vuln_id not in vuln_responses: - vuln_responses[vuln_id] = [] - vuln_responses[vuln_id].append(summary) - - # Calculate response spread for each CVE - response_spreads = {} - all_spreads = [] - - for vuln_id, responses in vuln_responses.items(): - if len(responses) < 2: - # Need at least 2 responses to calculate spread - response_spreads[vuln_id] = 1.0 # Perfect similarity if only one response - all_spreads.append(1.0) - continue - - # Calculate cosine similarity spread using bag-of-words - vectorizer = CountVectorizer(stop_words=self.stopwords) - X = vectorizer.fit_transform(responses).toarray() - - if X.shape[0] < 2: - response_spreads[vuln_id] = 1.0 - all_spreads.append(1.0) - continue - - # Calculate centroid (average of all response vectors) - centroid = np.sum(X, axis=0) - - # Calculate cosine similarities between each response and centroid - dot_products = np.dot(X, centroid) - norms = np.linalg.norm(X, axis=1) * np.linalg.norm(centroid) - - # Avoid division by zero - norms = np.where(norms == 0, 1, norms) - cos_similarities = dot_products / norms - - # Apply aggregation - if self.aggregation == "average": - spread_score = float(np.average(cos_similarities)) - elif self.aggregation == "max": - spread_score = float(np.max(cos_similarities)) - elif self.aggregation == "min": - spread_score = float(np.min(cos_similarities)) - else: - spread_score = float(np.average(cos_similarities)) - - response_spreads[vuln_id] = spread_score - all_spreads.append(spread_score) - - # Calculate overall average spread - overall_spread = sum(all_spreads) / len(all_spreads) if all_spreads else 1.0 - reasoning = { - "response_spreads": response_spreads, - "overall_spread": overall_spread, - "aggregation_method": self.aggregation, - "num_cves": len(vuln_responses), - "total_responses": sum(len(responses) for responses in vuln_responses.values()), - "test_set_id": item.full_dataset_entry.get('test_set_id', 'unknown'), - } - - return overall_spread, reasoning - - async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: - async with self.semaphore: - result = await process_item(item) - pbar.update(1) - return result - - try: - tqdm_position = TqdmPositionRegistry.claim() - pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Response Spread", position=tqdm_position) - results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) - finally: - pbar.close() - TqdmPositionRegistry.release(tqdm_position) - - sample_scores, sample_reasonings = zip(*results) if results else ([], []) - - avg_score = round(sum(sample_scores) / len(sample_scores), 4) if sample_scores else 1.0 - - eval_output_items = [ - EvalOutputItem(id=item.id, score=score, reasoning=reasoning) - for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings) - ] - - return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) diff --git a/src/vuln_analysis/eval/eval_status_accuracy.py b/src/vuln_analysis/eval/eval_status_accuracy.py deleted file mode 100644 index ac3bd7cd4..000000000 --- a/src/vuln_analysis/eval/eval_status_accuracy.py +++ /dev/null @@ -1,97 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: LicenseRef-NvidiaProprietary -# -# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual -# property and proprietary rights in and to this material, related -# documentation and any modifications thereto. Any use, reproduction, -# disclosure or distribution of this material and related documentation -# without an express license agreement from NVIDIA CORPORATION or -# its affiliates is strictly prohibited. - -import asyncio -import json - -from tqdm import tqdm - -from nat.eval.evaluator.evaluator_model import EvalInput -from nat.eval.evaluator.evaluator_model import EvalOutput -from nat.eval.evaluator.evaluator_model import EvalOutputItem -from nat.eval.evaluator.evaluator_model import EvalInputItem -from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry - - -_LABEL_MAP = {'TRUE': 'AFFECTED', 'FALSE': 'NOT AFFECTED', 'UNKNOWN': 'UNKNOWN'} - - -class StatusAccuracyEvaluator: - '''Status accuracy evaluator class that calculates status accuracy (AFFECTED/NOT AFFECTED)''' - - def __init__(self, max_concurrency: int): - self.max_concurrency = max_concurrency - self.semaphore = asyncio.Semaphore(self.max_concurrency) - - async def evaluate(self, eval_input: EvalInput) -> EvalOutput: - '''Evaluate accuracy funciont''' - - async def process_item(item): - """Compute simple accuracy for an individual item""" - ground_truth_map = eval(item.expected_output_obj) - pipeline_result = json.loads(item.output_obj)['output'] - - status_accuracies = [] - generated_status_answers = [] - question = [] - - for result in pipeline_result: - vuln_id = result['vuln_id'] - question.append(vuln_id) - - # Extract status from pipeline result - affected_status = result.get('justification', {}).get('status', 'MISSING') - - status_result = _LABEL_MAP.get(affected_status, 'MISSING') - generated_status_answers.append(status_result) - - # ORIGINAL LOGIC - ground truth MISSING factors into accuracy calculation - # accuracies.append(float(status_result == ground_truth_map.get(vuln_id, 'MISSING'))) - - # NEW LOGIC - Only include in accuracy calculation if ground truth exists (not MISSING) - ground_truth_status = ground_truth_map.get(vuln_id) - if ground_truth_status is not None: - status_accuracies.append(float(status_result == ground_truth_status)) - - # Calculate average accuracy - status_avg = sum(status_accuracies)/len(status_accuracies) if status_accuracies else 0.0 - reasoning = { - "question": question, - "status_answer": [ground_truth_map.get(q, 'MISSING') for q in question], - "generated_status_answer": generated_status_answers, - "test_set_id": item.full_dataset_entry.get('test_set_id', 'unknown'), - } - - return status_avg, reasoning - - async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: - async with self.semaphore: - result = await process_item(item) - pbar.update(1) - return result - - try: - tqdm_position = TqdmPositionRegistry.claim() - pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Accuracy", position=tqdm_position) - results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) - finally: - pbar.close() - TqdmPositionRegistry.release(tqdm_position) - - sample_scores, sample_reasonings = zip(*results) if results else ([], []) - - avg_score = round(sum(sample_scores) / len(sample_scores), 2) if sample_scores else 0.0 - - eval_output_items = [ - EvalOutputItem(id=item.id, score=score, reasoning=reasoning) - for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings) - ] - - return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) \ No newline at end of file diff --git a/src/vuln_analysis/eval/evaluator_register.py b/src/vuln_analysis/eval/evaluator_register.py index d9f65aac1..0ab7c1083 100644 --- a/src/vuln_analysis/eval/evaluator_register.py +++ b/src/vuln_analysis/eval/evaluator_register.py @@ -14,13 +14,15 @@ # limitations under the License. # Register evaluator helper functions +from typing import Optional + from nat.builder.builder import EvalBuilder from nat.builder.evaluator import EvaluatorInfo from nat.cli.register_workflow import register_evaluator from nat.data_models.evaluator import EvaluatorBaseConfig -#evaluator registration (actual code is in eval/eval_status_accuracy.py) +#evaluator registration (actual code is in eval/evaluators/status_accuracy.py) class StatusAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="status_accuracy"): '''Configuration for status accuracy evaluator''' pass @@ -29,13 +31,13 @@ class StatusAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="status_accuracy") @register_evaluator(config_type=StatusAccuracyEvaluatorConfig) async def status_accuracy_evaluator(config: StatusAccuracyEvaluatorConfig, builder: EvalBuilder): '''Register status accuracy evaluator''' - from .eval_status_accuracy import StatusAccuracyEvaluator + from .evaluators.status_accuracy import StatusAccuracyEvaluator evaluator = StatusAccuracyEvaluator(builder.get_max_concurrency()) yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Status Accuracy Evaluator") -#evaluator registration (actual code is in eval/eval_label_accuracy.py) +#evaluator registration (actual code is in eval/evaluators/label_accuracy.py) class LabelAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="label_accuracy"): '''Configuration for label accuracy evaluator''' pass @@ -44,28 +46,31 @@ class LabelAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="label_accuracy"): @register_evaluator(config_type=LabelAccuracyEvaluatorConfig) async def label_accuracy_evaluator(config: LabelAccuracyEvaluatorConfig, builder: EvalBuilder): '''Register label accuracy evaluator''' - from .eval_label_accuracy import LabelAccuracyEvaluator + from .evaluators.label_accuracy import LabelAccuracyEvaluator evaluator = LabelAccuracyEvaluator(builder.get_max_concurrency()) yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Label Accuracy Evaluator") -#evaluator registration (actual code is in eval/eval_response_spread.py) +#evaluator registration (actual code is in eval/evaluators/response_spread.py) class ResponseSpreadEvaluatorConfig(EvaluatorBaseConfig, name="response_spread"): '''Configuration for response spread evaluator''' - pass + aggregation: str = "average" # "average", "min", or "max" + stopwords: Optional[str] = None @register_evaluator(config_type=ResponseSpreadEvaluatorConfig) async def response_spread_evaluator(config: ResponseSpreadEvaluatorConfig, builder: EvalBuilder): '''Register response spread evaluator''' - from .eval_response_spread import ResponseSpreadEvaluator - evaluator = ResponseSpreadEvaluator(builder.get_max_concurrency()) + from .evaluators.response_spread import ResponseSpreadEvaluator + evaluator = ResponseSpreadEvaluator(max_concurrency=builder.get_max_concurrency(), + aggregation=config.aggregation, + stopwords=config.stopwords) yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Response Spread Evaluator") -#evaluator registration (actual code is in eval/eval_consistency.py) +#evaluator registration (actual code is in eval/evaluators/consistency.py) class ConsistencyEvaluatorConfig(EvaluatorBaseConfig, name="consistency"): '''Configuration for consistency evaluator''' pass @@ -74,7 +79,7 @@ class ConsistencyEvaluatorConfig(EvaluatorBaseConfig, name="consistency"): @register_evaluator(config_type=ConsistencyEvaluatorConfig) async def consistency_evaluator(config: ConsistencyEvaluatorConfig, builder: EvalBuilder): '''Register consistency evaluator''' - from .eval_consistency import ConsistencyEvaluator + from .evaluators.consistency import ConsistencyEvaluator evaluator = ConsistencyEvaluator(builder.get_max_concurrency()) yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Consistency Evaluator") diff --git a/src/vuln_analysis/eval/parse_eval_input.py b/src/vuln_analysis/eval/parse_eval_input.py index 77fb4d5a5..5ff8d1009 100644 --- a/src/vuln_analysis/eval/parse_eval_input.py +++ b/src/vuln_analysis/eval/parse_eval_input.py @@ -209,4 +209,4 @@ def main(): if __name__ == "__main__": - exit(main()) \ No newline at end of file + exit(main()) diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index 17f62369f..55e190085 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -236,4 +236,4 @@ async def _response_fn(input_message: AgentMorpheusInput) -> AgentMorpheusOutput except GeneratorExit: logger.info("Workflow exited early!") finally: - logger.info("Cleaning up cve-agent workflow.") \ No newline at end of file + logger.info("Cleaning up cve-agent workflow.") From ee1660925f53e79e49b1fe55069ff61d826d315f Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Wed, 10 Sep 2025 13:19:17 -0700 Subject: [PATCH 027/104] moved evaluators to evaluator dir --- .../eval/evaluators/consistency.py | 134 +++++++++++++++++ .../eval/evaluators/label_accuracy.py | 94 ++++++++++++ .../eval/evaluators/response_spread.py | 138 ++++++++++++++++++ .../eval/evaluators/status_accuracy.py | 102 +++++++++++++ 4 files changed, 468 insertions(+) create mode 100644 src/vuln_analysis/eval/evaluators/consistency.py create mode 100644 src/vuln_analysis/eval/evaluators/label_accuracy.py create mode 100644 src/vuln_analysis/eval/evaluators/response_spread.py create mode 100644 src/vuln_analysis/eval/evaluators/status_accuracy.py diff --git a/src/vuln_analysis/eval/evaluators/consistency.py b/src/vuln_analysis/eval/evaluators/consistency.py new file mode 100644 index 000000000..4bf47d45e --- /dev/null +++ b/src/vuln_analysis/eval/evaluators/consistency.py @@ -0,0 +1,134 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import json +import numpy as np +from collections import Counter + +from tqdm import tqdm + +from nat.eval.evaluator.evaluator_model import EvalInput +from nat.eval.evaluator.evaluator_model import EvalOutput +from nat.eval.evaluator.evaluator_model import EvalOutputItem +from nat.eval.evaluator.evaluator_model import EvalInputItem +from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry + + +class ConsistencyEvaluator: + '''Evaluator class for measuring consistency using custom metric''' + + def __init__(self, max_concurrency: int): + self.max_concurrency = max_concurrency + self.semaphore = asyncio.Semaphore(self.max_concurrency) + + async def evaluate(self, eval_input: EvalInput) -> EvalOutput: + '''Evaluate consistency across multiple runs''' + + async def process_item(item): + """Calculate consistency for an individual item across multiple runs""" + pipeline_result = json.loads(item.output_obj)['output'] + + # Group responses by vuln_id to get multiple responses per CVE + vuln_responses = {} + + for result in pipeline_result: + vuln_id = result['vuln_id'] + # Measure consistency on label and status fields + label = result.get('justification', {}).get('label', 'MISSING') + status = result.get('justification', {}).get('status', 'MISSING') + + if vuln_id not in vuln_responses: + vuln_responses[vuln_id] = {'labels': [], 'statuses': []} + + vuln_responses[vuln_id]['labels'].append(label) + vuln_responses[vuln_id]['statuses'].append(status) + + # Calculate consistency for each CVE + label_consistencies = {} + status_consistencies = {} + all_label_consistencies = [] + all_status_consistencies = [] + + for vuln_id, responses in vuln_responses.items(): + # Calculate label consistency + labels = responses['labels'] + if len(labels) > 1: + # Consistency calculation: sum of squared probabilities + label_counts = Counter(labels) + n = len(labels) + probs = [count / n for count in label_counts.values()] + label_consistency = sum(p * p for p in probs) + else: + label_consistency = 1.0 # Perfect consistency if only one response + + label_consistencies[vuln_id] = label_consistency + all_label_consistencies.append(label_consistency) + + # Calculate status consistency + statuses = responses['statuses'] + if len(statuses) > 1: + # Consistency calculation: sum of squared probabilities + status_counts = Counter(statuses) + n = len(statuses) + probs = [count / n for count in status_counts.values()] + status_consistency = sum(p * p for p in probs) + else: + status_consistency = 1.0 # Perfect consistency if only one response + + status_consistencies[vuln_id] = status_consistency + all_status_consistencies.append(status_consistency) + + # Calculate overall average consistency + overall_label_consistency = sum(all_label_consistencies) / len(all_label_consistencies) if all_label_consistencies else 1.0 + overall_status_consistency = sum(all_status_consistencies) / len(all_status_consistencies) if all_status_consistencies else 1.0 + reasoning = { + "label_consistencies": label_consistencies, + "status_consistencies": status_consistencies, + "overall_label_consistency": overall_label_consistency, + "overall_status_consistency": overall_status_consistency, + "num_cves": len(vuln_responses), + "total_responses": sum(len(responses['labels']) for responses in vuln_responses.values()), + "test_set_id": item.full_dataset_entry.get('test_set_id', 'unknown'), + } + + # Return average of label and status consistency + overall_consistency = (overall_label_consistency + overall_status_consistency) / 2 + return overall_consistency, reasoning + + async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: + async with self.semaphore: + result = await process_item(item) + pbar.update(1) + return result + + try: + tqdm_position = TqdmPositionRegistry.claim() + pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Consistency", position=tqdm_position) + results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) + finally: + pbar.close() + TqdmPositionRegistry.release(tqdm_position) + + sample_scores, sample_reasonings = zip(*results) if results else ([], []) + + avg_score = round(sum(sample_scores) / len(sample_scores), 4) if sample_scores else 1.0 + + eval_output_items = [ + EvalOutputItem(id=item.id, score=score, reasoning=reasoning) + for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings) + ] + + return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) diff --git a/src/vuln_analysis/eval/evaluators/label_accuracy.py b/src/vuln_analysis/eval/evaluators/label_accuracy.py new file mode 100644 index 000000000..0e659d3c4 --- /dev/null +++ b/src/vuln_analysis/eval/evaluators/label_accuracy.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import json + +from tqdm import tqdm + +from nat.eval.evaluator.evaluator_model import EvalInput +from nat.eval.evaluator.evaluator_model import EvalOutput +from nat.eval.evaluator.evaluator_model import EvalOutputItem +from nat.eval.evaluator.evaluator_model import EvalInputItem +from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry + + +class LabelAccuracyEvaluator: + '''Evaluator class for justification labels accuracy''' + + def __init__(self, max_concurrency: int): + self.max_concurrency = max_concurrency + self.semaphore = asyncio.Semaphore(self.max_concurrency) + + async def evaluate(self, eval_input: EvalInput) -> EvalOutput: + '''Evaluate label accuracy''' + + async def process_item(item): + """Compute simple accuracy for an individual item""" + ground_truth_map = eval(item.expected_output_obj) + pipeline_result = json.loads(item.output_obj)['output'] + + label_accuracies = [] + generated_answers = [] + question = [] + + for result in pipeline_result: + vuln_id = result['vuln_id'] + question.append(vuln_id) + + # Extract label from pipeline result + justification_label = result.get('justification', {}).get('label', 'MISSING') + generated_answers.append(justification_label) + + # Calculate label accuracy + # Ground truth labels are stored as "{vuln_id}_label" in the ground truth map + # This is populated by parse_eval_input.py from the eval dataset JSON files + ground_truth_label = ground_truth_map.get(f"{vuln_id}_label") + if ground_truth_label is not None: + label_accuracies.append(float(justification_label == ground_truth_label)) + + reasoning = { + "question": question, + "answer": [ground_truth_map.get(f"{q}_label", 'MISSING') for q in question], + "generated_answer": generated_answers, + "test_set_id": item.full_dataset_entry.get('test_set_id', 'unknown'), + } + + return sum(label_accuracies)/len(label_accuracies) if label_accuracies else 0.0, reasoning + + async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: + async with self.semaphore: + result = await process_item(item) + pbar.update(1) + return result + + try: + tqdm_position = TqdmPositionRegistry.claim() + pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Label Accuracy", position=tqdm_position) + results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) + finally: + pbar.close() + TqdmPositionRegistry.release(tqdm_position) + + sample_scores, sample_reasonings = zip(*results) if results else ([], []) + + avg_score = round(sum(sample_scores) / len(sample_scores), 2) if sample_scores else 0.0 + + eval_output_items = [ + EvalOutputItem(id=item.id, score=score, reasoning=reasoning) + for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings) + ] + + return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) diff --git a/src/vuln_analysis/eval/evaluators/response_spread.py b/src/vuln_analysis/eval/evaluators/response_spread.py new file mode 100644 index 000000000..da8218d7d --- /dev/null +++ b/src/vuln_analysis/eval/evaluators/response_spread.py @@ -0,0 +1,138 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import json +import numpy as np +from collections import Counter +from sklearn.feature_extraction.text import CountVectorizer + +from tqdm import tqdm + +from nat.eval.evaluator.evaluator_model import EvalInput +from nat.eval.evaluator.evaluator_model import EvalOutput +from nat.eval.evaluator.evaluator_model import EvalOutputItem +from nat.eval.evaluator.evaluator_model import EvalInputItem +from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry + + +class ResponseSpreadEvaluator: + '''Evaluator class for measuring response spread using cosine similarity''' + + def __init__(self, max_concurrency: int, aggregation: str = "average", stopwords: str = None): + self.max_concurrency = max_concurrency + self.semaphore = asyncio.Semaphore(self.max_concurrency) + self.aggregation = aggregation # "average", "min", or "max" + self.stopwords = stopwords + + async def evaluate(self, eval_input: EvalInput) -> EvalOutput: + '''Evaluate response spread across multiple runs''' + + async def process_item(item): + """Calculate response spread for an individual item across multiple runs""" + pipeline_result = json.loads(item.output_obj)['output'] + + # Group responses by vuln_id to get multiple responses per CVE + vuln_responses = {} + + for result in pipeline_result: + vuln_id = result['vuln_id'] + summary = result.get('summary', '') + + if vuln_id not in vuln_responses: + vuln_responses[vuln_id] = [] + vuln_responses[vuln_id].append(summary) + + # Calculate response spread for each CVE + response_spreads = {} + all_spreads = [] + + for vuln_id, responses in vuln_responses.items(): + if len(responses) < 2: + # Need at least 2 responses to calculate spread + response_spreads[vuln_id] = 1.0 # Perfect similarity if only one response + all_spreads.append(1.0) + continue + + # Calculate cosine similarity spread using bag-of-words + vectorizer = CountVectorizer(stop_words=self.stopwords) + X = vectorizer.fit_transform(responses).toarray() + + if X.shape[0] < 2: + response_spreads[vuln_id] = 1.0 + all_spreads.append(1.0) + continue + + # Calculate centroid (average of all response vectors) + centroid = np.sum(X, axis=0) + + # Calculate cosine similarities between each response and centroid + dot_products = np.dot(X, centroid) + norms = np.linalg.norm(X, axis=1) * np.linalg.norm(centroid) + + # Avoid division by zero + norms = np.where(norms == 0, 1, norms) + cos_similarities = dot_products / norms + + # Apply aggregation + if self.aggregation == "average": + spread_score = float(np.average(cos_similarities)) + elif self.aggregation == "max": + spread_score = float(np.max(cos_similarities)) + elif self.aggregation == "min": + spread_score = float(np.min(cos_similarities)) + else: + spread_score = float(np.average(cos_similarities)) + + response_spreads[vuln_id] = spread_score + all_spreads.append(spread_score) + + # Calculate overall average spread + overall_spread = sum(all_spreads) / len(all_spreads) if all_spreads else 1.0 + reasoning = { + "response_spreads": response_spreads, + "overall_spread": overall_spread, + "aggregation_method": self.aggregation, + "num_cves": len(vuln_responses), + "total_responses": sum(len(responses) for responses in vuln_responses.values()), + "test_set_id": item.full_dataset_entry.get('test_set_id', 'unknown'), + } + + return overall_spread, reasoning + + async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: + async with self.semaphore: + result = await process_item(item) + pbar.update(1) + return result + + try: + tqdm_position = TqdmPositionRegistry.claim() + pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Response Spread", position=tqdm_position) + results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) + finally: + pbar.close() + TqdmPositionRegistry.release(tqdm_position) + + sample_scores, sample_reasonings = zip(*results) if results else ([], []) + + avg_score = round(sum(sample_scores) / len(sample_scores), 4) if sample_scores else 1.0 + + eval_output_items = [ + EvalOutputItem(id=item.id, score=score, reasoning=reasoning) + for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings) + ] + + return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) diff --git a/src/vuln_analysis/eval/evaluators/status_accuracy.py b/src/vuln_analysis/eval/evaluators/status_accuracy.py new file mode 100644 index 000000000..1f4741e12 --- /dev/null +++ b/src/vuln_analysis/eval/evaluators/status_accuracy.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import json + +from tqdm import tqdm + +from nat.eval.evaluator.evaluator_model import EvalInput +from nat.eval.evaluator.evaluator_model import EvalOutput +from nat.eval.evaluator.evaluator_model import EvalOutputItem +from nat.eval.evaluator.evaluator_model import EvalInputItem +from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry + + +_LABEL_MAP = {'TRUE': 'AFFECTED', 'FALSE': 'NOT AFFECTED', 'UNKNOWN': 'UNKNOWN'} + + +class StatusAccuracyEvaluator: + '''Status accuracy evaluator class that calculates status accuracy (AFFECTED/NOT AFFECTED)''' + + def __init__(self, max_concurrency: int): + self.max_concurrency = max_concurrency + self.semaphore = asyncio.Semaphore(self.max_concurrency) + + async def evaluate(self, eval_input: EvalInput) -> EvalOutput: + '''Evaluate accuracy funciont''' + + async def process_item(item): + """Compute simple accuracy for an individual item""" + ground_truth_map = eval(item.expected_output_obj) + pipeline_result = json.loads(item.output_obj)['output'] + + status_accuracies = [] + generated_status_answers = [] + question = [] + + for result in pipeline_result: + vuln_id = result['vuln_id'] + question.append(vuln_id) + + # Extract status from pipeline result + affected_status = result.get('justification', {}).get('status', 'MISSING') + + status_result = _LABEL_MAP.get(affected_status, 'MISSING') + generated_status_answers.append(status_result) + + # ORIGINAL LOGIC - ground truth MISSING factors into accuracy calculation + # accuracies.append(float(status_result == ground_truth_map.get(vuln_id, 'MISSING'))) + + # NEW LOGIC - Only include in accuracy calculation if ground truth exists (not MISSING) + ground_truth_status = ground_truth_map.get(vuln_id) + if ground_truth_status is not None: + status_accuracies.append(float(status_result == ground_truth_status)) + + # Calculate average accuracy + status_avg = sum(status_accuracies)/len(status_accuracies) if status_accuracies else 0.0 + reasoning = { + "question": question, + "status_answer": [ground_truth_map.get(q, 'MISSING') for q in question], + "generated_status_answer": generated_status_answers, + "test_set_id": item.full_dataset_entry.get('test_set_id', 'unknown'), + } + + return status_avg, reasoning + + async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: + async with self.semaphore: + result = await process_item(item) + pbar.update(1) + return result + + try: + tqdm_position = TqdmPositionRegistry.claim() + pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Accuracy", position=tqdm_position) + results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) + finally: + pbar.close() + TqdmPositionRegistry.release(tqdm_position) + + sample_scores, sample_reasonings = zip(*results) if results else ([], []) + + avg_score = round(sum(sample_scores) / len(sample_scores), 2) if sample_scores else 0.0 + + eval_output_items = [ + EvalOutputItem(id=item.id, score=score, reasoning=reasoning) + for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings) + ] + + return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) From fd45a3bc2eee92352b8bdc51f213af7a966d7211 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Wed, 10 Sep 2025 13:28:34 -0700 Subject: [PATCH 028/104] modified gitignore to removed .tmp files --- .tmp/.gitkeep | 0 .../CVE-2023-36632.md | 394 ------------------ 2 files changed, 394 deletions(-) delete mode 100644 .tmp/.gitkeep delete mode 100644 .tmp/vulnerability_markdown_reports/vulnerability_reports_morpheus_23_11-runtime/CVE-2023-36632.md diff --git a/.tmp/.gitkeep b/.tmp/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/.tmp/vulnerability_markdown_reports/vulnerability_reports_morpheus_23_11-runtime/CVE-2023-36632.md b/.tmp/vulnerability_markdown_reports/vulnerability_reports_morpheus_23_11-runtime/CVE-2023-36632.md deleted file mode 100644 index 0e20a23ea..000000000 --- a/.tmp/vulnerability_markdown_reports/vulnerability_reports_morpheus_23_11-runtime/CVE-2023-36632.md +++ /dev/null @@ -1,394 +0,0 @@ - - -# Vulnerability Analysis Report for CVE-2023-36632 -> **Container Analyzed:** `nvcr.io/nvidia/morpheus/morpheus:23.11-runtime` - - -> **SBOM Info:** `type='file' file_path='data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom'` - - -> **Status:** Exploitable -## CVE-2023-36632 -> GHSA-ID: GHSA-gv66-v8c8-v69c - -`Disputed`[source](https://nvd.nist.gov/vuln/detail/CVE-2023-36632) - - - The legacy email.utils.parseaddr function in Python through 3.11.4 allows attackers to trigger "RecursionError: maximum recursion depth exceeded while calling a Python object" via a crafted argument. This argument is plausibly an untrusted value from an application's input data that was supposed to contain a name and an e-mail address. NOTE: email.utils.parseaddr is categorized as a Legacy API in the documentation of the Python email package. Applications should instead use the email.parser.BytesParser or email.parser.Parser class. [references](#ref) - - -### Severity and CVSS Score -| Source | Severity | CVSS Score | Vector | Published At | Last Updated | -|---|---|---|---|---|---| -| [**GHSA**](https://github.com/advisories/GHSA-gv66-v8c8-v69c) |UNKNOWN|N/A|N/A|2023-06-25|2023-11-06 -| [**NVD**](https://nvd.nist.gov/vuln/detail/CVE-2023-36632) |HIGH|7.5|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H|2023-06-25|2024-11-21| -| [**RHSA**](https://access.redhat.com/security/cve/CVE-2023-36632) |N/A|N/A|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N|2023-06-25|N/A| -| [**Ubuntu**](https://ubuntu.com/security/CVE-2023-36632) |HIGH|7.5|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H|2023-06-25|2024-07-24| - - -### EPSS Score -| EPSS | Percentile | -| --- | --- | -| 0.00121 | 0.47904 | - -## Vulnerability Analysis - -### Summary (Exploitable) -Based on the provided Checklist and Findings, the CVE is exploitable. Specifically, the application does not handle exceptions like `RecursionError` when processing input data (Checklist Item 1), and the Docker container's error handling mechanisms are not explicitly designed to handle `RecursionError` exceptions (Checklist Item 2). Additionally, there is a potential source of uncontrolled recursion in the `ProducerConsumerQueue` class due to the use of a recursive lock (Checklist Item 3). These findings suggest that the CVE is exploitable, as the application and Docker container do not have adequate measures in place to prevent or handle `RecursionError` exceptions. - -### Justification - - ->label: vulnerable - -The analysis concludes that the CVE is exploitable due to the application's and Docker container's inadequate handling of `RecursionError` exceptions and the presence of a potential source of uncontrolled recursion, indicating that the package is vulnerable to the CVE and needs to be patched. - ---- -### Vulnerable SBOM Dependencies - - -| SBOM Name | SBOM Version | Dependency Name | Dependency Version | Relation | System | -| --- | --- | --- | --- | --- | --- | -| python | 3.10.12 | python | 3.10.12 | SELF | binary | -| python | 3.10.13 | python | 3.10.13 | SELF | binary | -### Checklist -1. [Assess Input Data Handling](#checklist-step-1) - - 1. [Docker Container Code QA System](#checklist-step-1.1) - 2. [Docker Container Code QA System](#checklist-step-1.2) - 3. [Docker Container Developer Guide QA System](#checklist-step-1.3) -2. [Review Error Handling Mechanisms](#checklist-step-2) - - 1. [Docker Container Code QA System](#checklist-step-2.1) - 2. [Docker Container Developer Guide QA System](#checklist-step-2.2) - 3. [Internet Search](#checklist-step-2.3) -3. [Inspect Code for Uncontrolled Recursion](#checklist-step-3) - - 1. [Docker Container Code QA System](#checklist-step-3.1) - 2. [Docker Container Code QA System](#checklist-step-3.2) - -## Checklist Details - -## Step 1 : Assess Input Data Handling - - -> **Input**: *Assess Input Data Handling: Evaluate how the application handles exceptions like `RecursionError`, as the exploit triggers this specific error. Check if the application processes potentially untrusted input data that could include crafted arguments designed to exploit this vulnerability.* - -> **Response**: *The application does not handle exceptions like `RecursionError` when processing input data, but it does handle `RecursionError` in the logging part of the application. Additionally, the application processes input data, but it's unclear whether this input data is potentially untrusted or could include crafted arguments designed to exploit a vulnerability. The application attempts to validate and sanitize input data, but it's still unclear whether this is sufficient to prevent exploitation. The purpose and functionality of the Docker container, and how it handles input data, is not explicitly explained in the provided context.* - -### Step 1.1 : *Docker Container Code QA System* - - -#### Action Log -
Thought: To assess the input data handling of the application, I need to understand how the application processes input data and if it has any mechanisms in place to handle exceptions like `RecursionError`. I also need to check if the application processes potentially untrusted input data that could include crafted arguments designed to exploit this vulnerability.
-
-Action: Docker Container Code QA System
-Action Input: Does the application handle exceptions like `RecursionError` when processing input data?
-
- - -#### Tool Input -
Does the application handle exceptions like `RecursionError` when processing input data?
-
- - -#### Tool Output -
No, the application does not handle exceptions like `RecursionError` when processing input data. According to the provided context, `RecursionError` is only handled in the logging part of the application, specifically when writing log messages. It is not handled in the input data processing part.
- - Source Documents - - | ID | Type | source |... content_type |... language | Page Content | -| --- | --- | --- | --- | --- | --- | -| None | Document | morpheus/service/vdb/milvus_vector_db_service.py | functions_classes | python |
View ContentReturns ------- typing.Any The search result, which can vary depending on the query and options. Raises ------ RuntimeError If an error occurs during the search operation. If query argument is `None` and `data` keyword argument doesn't exist. If `data` keyword arguement is `None`. """ logger.debug("Searching in collection: %s, query=%s, kwargs=%s", self.\_name, query, kwargs) return self.\_client.query(collection\_name=self.\_name, filter=query, \*\*kwargs)
| -| None | Document | morpheus/stages/input/appshield_source_stage.py | functions_classes | python |
View Contentself.\_encoding = encoding self.\_input\_count = None self.\_watcher = DirectoryWatcher(input\_glob=input\_glob, watch\_directory=watch\_directory, max\_files=max\_files, sort\_glob=sort\_glob, recursive=recursive, queue\_max\_size=queue\_max\_size, batch\_timeout=batch\_timeout) @property def name(self) -> str: return "from-appshield" @property def input\_count(self) -> int: """Return None for no max intput count""" return self.\_input\_count def supports\_cpp\_node(self): return False
| -| None | Document | morpheus/stages/input/appshield_source_stage.py | functions_classes | python |
View ContentIf true the list of files matching `input\_glob` will be processed in sorted order. recursive : bool, default = True If true, events will be emitted for the files in subdirectories matching `input\_glob`. queue\_max\_size : int, default = 128 Maximum queue size to hold the file paths to be processed that match `input\_glob`. batch\_timeout : float, default = 5.0 Timeout to retrieve batch messages from the queue. encoding : str, default = latin1 Encoding to read a file. """
| -| None | Document | morpheus/utils/logger.py | functions_classes | python |
View Contentfile = self.\_stderr if is\_error else self.\_stdout color\_kwargs = self.\_determine\_color(record.levelno) with tqdm.external\_write\_mode(file=file, nolock=False): # Write the message click.echo(click.style(msg, \*\*color\_kwargs), file=file, err=is\_error) self.flush() # See issue 36272 https://bugs.python.org/issue36272 except (KeyboardInterrupt, SystemExit, RecursionError): # noqa raise except Exception: # pylint: disable=broad-except self.handleError(record)
| - - - - -[back to top](#checklist-toc) - -### Step 1.2 : *Docker Container Code QA System* - - -#### Action Log -
Thought: The application does not handle exceptions like `RecursionError` when processing input data. However, it does handle `RecursionError` in the logging part of the application. This suggests that the application may not be robust against crafted arguments designed to exploit this vulnerability.
-
-Action: Docker Container Code QA System
-Action Input: Does the application process potentially untrusted input data that could include crafted arguments designed to exploit this vulnerability?
-
- - -#### Tool Input -
Does the application process potentially untrusted input data that could include crafted arguments designed to exploit this vulnerability?
-
- - -#### Tool Output -
Based on the provided context, it appears that the application does process input data, specifically `user_inputs` and `input_names`, which are used to create mappings and validate input names. However, it's unclear whether this input data is potentially untrusted or could include crafted arguments designed to exploit a vulnerability.
-
-The tests `ProcessInputNamesMismatchMultipleMappings` and `ProcessInputNamesMismatchSingleMapping` suggest that the application is designed to handle invalid input and throw exceptions when input names do not match. This implies that the application is attempting to validate and sanitize input data, but it's still unclear whether this is sufficient to prevent exploitation.
-
-Therefore, the answer to the question is: I don't know.
- - Source Documents - - | ID | Type | source |... content_type |... language | Page Content | -| --- | --- | --- | --- | --- | --- | -| None | Document | morpheus/_lib/tests/llm/test_utils.cpp | functions_classes | cpp |
View ContentTEST\_F(TestLLMUtils, ProcessInputNamesMismatchMultipleMappings) { auto user\_inputs = llm::user\_input\_mappings\_t{{"/ext1", "input1"}, {"/ext2", "input3"}}; auto input\_names = std::vector<std::string>{"input1", "input2"}; EXPECT\_THROW(llm::process\_input\_names(user\_inputs, input\_names), std::invalid\_argument); }
| -| None | Document | morpheus/_lib/include/morpheus/messages/memory/inference_memory_nlp.hpp | functions_classes | cpp |
View Contentstruct InferenceMemoryNLPInterfaceProxy : public InferenceMemoryInterfaceProxy { /\*\* \* @brief Create and initialize an InferenceMemoryNLP object, and return a shared pointer to the result \* \* @param count : Message count in inference memory object \* @param input\_ids : The token-ids for each string padded with 0s to max\_length \* @param input\_mask : The mask for token-ids result where corresponding positions identify valid token-id values \* @param seq\_ids : Ids used to index from an inference input to a message. Necessary since there can be more inference inputs than messages (i.e., if some messages get broken into multiple inference requests) \* @return std::shared\_ptr<InferenceMemoryNLP> \*/
| -| None | Document | morpheus/_lib/tests/llm/test_utils.cpp | functions_classes | cpp |
View ContentTEST\_F(TestLLMUtils, ProcessInputNamesMismatchSingleMapping) { auto user\_inputs = llm::user\_input\_mappings\_t{{"/ext1", "input2"}}; auto input\_names = std::vector<std::string>{"input1"}; EXPECT\_THROW(llm::process\_input\_names(user\_inputs, input\_names), std::invalid\_argument); }
| -| None | Document | morpheus/_lib/messages/module.cpp | functions_classes | cpp |
View Content&InferenceMemoryNLPInterfaceProxy::set\_input\_mask) .def\_property( "seq\_ids", &InferenceMemoryNLPInterfaceProxy::get\_seq\_ids, &InferenceMemoryNLPInterfaceProxy::set\_seq\_ids);
| - - - - -[back to top](#checklist-toc) - -### Step 1.3 : *Docker Container Developer Guide QA System* - - -#### Action Log -
Thought: The application does not handle exceptions like `RecursionError` when processing input data, but it does handle `RecursionError` in the logging part of the application. Additionally, the application processes input data, but it's unclear whether this input data is potentially untrusted or could include crafted arguments designed to exploit a vulnerability. The application attempts to validate and sanitize input data, but it's still unclear whether this is sufficient to prevent exploitation.
-
-Action: Docker Container Developer Guide QA System
-Action Input: What is the purpose and functionality of the Docker container, and how does it handle input data?
-
- - -#### Tool Input -
What is the purpose and functionality of the Docker container, and how does it handle input data?
-
- - -#### Tool Output -
Unfortunately, the provided context does not explicitly explain the purpose and functionality of the Docker container or how it handles input data. The context mentions that Docker containers can be used to get started with Morpheus, and that pre-built Docker containers can be used or built from source, but it does not provide further details on their purpose or functionality.
-
-However, based on the general knowledge of Docker containers, it can be inferred that the Docker container is likely used to provide a self-contained environment for running the Morpheus pipeline, including the data processing stages and modules such as the `file_batcher`. But without more specific information, it's difficult to provide a more detailed answer.
-
-So, the answer is: I don't know.
- - Source Documents - - | ID | Type | source | Page Content | -| --- | --- | --- | --- | -| None | Document | examples/llm/rag/README.md |
View Content#### Data Input Points The pipeline has multiple data input avenues: 1. \*\*User Uploaded Documents\*\*: Raw documents provided by users for further processing. 2. \*\*Streaming Event Logs\*\*: Logs that are streamed in real-time. 3. \*\*Streaming Data Feeds\*\*: Continuous streams of data that could be dynamic in nature. 4. \*\*RSS Threat Intel Feeds\*\*: RSS-based feeds that might focus on threat intelligence. 5. \*\*LLM Input Query\*\*: Queries that are inputted for processing by the Large Language Model. #### Data Processing Stages The ingested data traverses several processing stages:
| -| None | Document | docs/source/index.rst |
View ContentGetting Started --------------- Using Morpheus ^^^^^^^^^^^^^^ \* :doc:`getting\_started` - Using pre-built Docker containers, building Docker containers from source, and fetching models and datasets \* :doc:`basics/overview` - Brief overview of the command line interface \* :doc:`basics/building\_a\_pipeline` - Introduction to building a pipeline using the command line interface \* :doc:`Morpheus Examples <examples>` - Example pipelines using both the Python API and command line interface \* :doc:`Pretrained Models <models\_and\_datasets>` - Pretrained models with corresponding training, validation scripts, and datasets \* :doc:`Developer Guides <developer\_guide/guides>` - Covers extending Morpheus with custom stages
| -| None | Document | docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md |
View ContentThe `file\_batcher` module is a component that is responsible for loading input files, filtering out files older than the specified time window, and grouping the remaining files by periods that fall within the time window. This module offers configurability for parameters such as batching options, cache directory, file type, filtering null values, data schema, and the timestamp column name. The `file\_batcher` module processes control messages, validates them, and generates a list of files with their timestamps. The module then groups files by the given period, creates control messages for each batch, and sends them downstream for further processing. A node function is used to handle the processing of control messages, and input and output ports are registered to integrate the module into
| -| None | Document | docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md |
View Contentprocessing. A node function is used to handle the processing of control messages, and input and output ports are registered to integrate the module into the data processing pipeline seamlessly.
| - - - - -## Step 2 : Review Error Handling Mechanisms - - -> **Input**: *Review Error Handling Mechanisms: Since the vendor's perspective is that this is neither a vulnerability nor a bug, but rather an expected behavior when limits are exceeded, review the application's error handling mechanisms to ensure they can properly handle `RecursionError` exceptions without causing unintended behavior or crashes.* - -> **Response**: *The Docker container's error handling mechanisms are not explicitly designed to handle `RecursionError` exceptions. However, the container's code does include some error handling mechanisms, such as raising `ValueError` exceptions in certain cases and catching `NotThisMethod` exceptions. To handle `RecursionError` exceptions, it is recommended to ensure that recursive functions have a terminating condition and to avoid exceeding the maximum recursion depth. Additionally, exceptions can be handled using try-except blocks.* - -### Step 2.1 : *Docker Container Code QA System* - - -#### Action Log -
Thought: To review the error handling mechanisms of the Docker container, I need to understand how the container handles `RecursionError` exceptions. I should first check if the container's code or any of its dependencies use a function or component that could potentially raise a `RecursionError`.
-
-Action: Docker Container Code QA System
-Action Input: Does the Docker container's code or any of its dependencies use a function or component that could raise a `RecursionError`?
-
- - -#### Tool Input -
Does the Docker container's code or any of its dependencies use a function or component that could raise a `RecursionError`?
-
- - -#### Tool Output -
I don't know.
- - Source Documents - - | ID | Type | source |... content_type |... language | Page Content | -| --- | --- | --- | --- | --- | --- | -| None | Document | morpheus/service/vdb/milvus_vector_db_service.py | functions_classes | python |
View Content- 'partition\_name' (str, optional): Required when dropping a specific partition within a collection. Specifies the partition name to be dropped. - 'field\_name' (str, optional): Required when dropping an index within a collection. Specifies the field name for which the index is created. - 'index\_name' (str, optional): Required when dropping an index within a collection. Specifies the name of the index to be dropped. Raises ------ ValueError If mandatory arguments are missing or if the provided 'collection' value is invalid. """ logger.debug("Dropping collection: %s, kwargs=%s", name, kwargs)
| -| None | Document | versioneer.py | functions_classes | python |
View Contentfrom\_vcs\_f = handlers.get("pieces\_from\_vcs") if from\_vcs\_f: try: pieces = from\_vcs\_f(cfg.tag\_prefix, root, verbose) ver = render(pieces, cfg.style) if verbose: print("got version from VCS %s" % ver) return ver except NotThisMethod: pass try: if cfg.parentdir\_prefix: ver = versions\_from\_parentdir(cfg.parentdir\_prefix, root, verbose) if verbose: print("got version from parentdir %s" % ver) return ver except NotThisMethod: pass if verbose: print("unable to compute version")
| -| None | Document | models/training-tuning-scripts/root-cause-models/root_cause_bert.py | simplified_code | python |
View Content# See the License for the specific language governing permissions and # limitations under the License. """ Example Usage: python root\_cause\_bert.py \ --trainingdata ../../datasets/training-data/root-cause-training-data.csv \ --unseenerrors ../../datasets/training-data/root-cause-unseen-errors.csv """
| -| None | Document | morpheus/utils/execution_chain.py | functions_classes | python |
View ContentReturns ------- Any The result of the last function in the chain. Raises ------ Exception If the execution fails at any function in the chain. """ local\_state = {} local\_state.update(kwargs) try: returned\_state = None for function in self.functions: signature = inspect.signature(function) func\_kwargs = {key: value for key, value in local\_state.items() if key in signature.parameters} returned\_state = function(\*\*func\_kwargs) local\_state.update(returned\_state)
| - - - - -[back to top](#checklist-toc) - -### Step 2.2 : *Docker Container Developer Guide QA System* - - -#### Action Log -
Thought: The Docker Container Code QA System was unable to find any specific information about the container's code or its dependencies using a function or component that could raise a `RecursionError`. However, it did return some information about error handling in the container's code. For example, in the `morpheus/service/vdb/milvus_vector_db_service.py` file, it appears that the `drop_collection` function raises a `ValueError` if mandatory arguments are missing or if the provided 'collection' value is invalid. Additionally, in the `versioneer.py` file, it seems that the `from_vcs_f` function catches a `NotThisMethod` exception and passes it. Furthermore, in the `models/training-tuning-scripts/root-cause-models/root_cause_bert.py` file, there is no obvious error handling mechanism. Lastly, in the `morpheus/utils/execution_chain.py` file, the `execute` function raises an `Exception` if the execution fails at any function in the chain.
-
-Action: Docker Container Developer Guide QA System
-Action Input: What is the purpose and functionality of the Docker container's error handling mechanisms?
-
- - -#### Tool Input -
What is the purpose and functionality of the Docker container's error handling mechanisms?
-
- - -#### Tool Output -
Unfortunately, I don't know the answer to this question. The provided context does not mention Docker containers or their error handling mechanisms. It appears to be discussing a model and pipeline for root cause analysis, but does not provide information about Docker containers.
- - Source Documents - - | ID | Type | source | Page Content | -| --- | --- | --- | --- | -| None | Document | docs/source/developer_guide/guides/2_real_world_phishing.md |
View Contentnecessary validation checks in the constructor, and raising an exception in the case of mis-configuration. This allows us to fail early rather than after the pipeline has started.
| -| None | Document | models/model-cards/root-cause-analysis-model-card.md |
View Content\* The model is primarily designed for testing purposes and serves as a small pre-trained model specifically used to evaluate and validate the Root Cause Analysis pipeline. This model is an example of customized transformer-based root cause analysis. It can be used for pipeline testing purposes. It needs to be re-trained for specific root cause analysis or predictive maintenance needs with the fine-tuning scripts in the repo. The hyperparameters can be optimised to adjust to get the best results with another dataset. The aim is to get the model to predict some false positives that could be previously unknown error types. Users can use this root cause analysis approach with other log types too. If they have known failures in their logs, they can use them to train along with ordinary logs
| -| None | Document | docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md |
View Contentprocessing. A node function is used to handle the processing of control messages, and input and output ports are registered to integrate the module into the data processing pipeline seamlessly.
| -| None | Document | docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md |
View ContentThe pipeline comprises a series of interconnected modules designed to create a versatile split processing pipeline. This design enables the reception and processing of control messages to perform tasks such as inference against observed events using either generic or user-specific models. Additionally, the pipeline can train new models based on aggregated or predefined training data, offering a more adaptable and user-friendly experience. ### Overview
| - - - - -[back to top](#checklist-toc) - -### Step 2.3 : *Internet Search* - - -#### Action Log -
Thought: The Docker Container Developer Guide QA System was unable to find any specific information about the Docker container's error handling mechanisms. However, it did return some information about error handling in general. For example, in the `docs/source/developer_guide/guides/2_real_world_phishing.md` file, it appears that the pipeline has necessary validation checks in the constructor and raises an exception in the case of mis-configuration. Additionally, in the `models/model-cards/root-cause-analysis-model-card.md` file, it seems that the model is designed for testing purposes and can be used for pipeline testing purposes. Furthermore, in the `docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md` file, the pipeline comprises a series of interconnected modules designed to create a versatile split processing pipeline.
-
-Action: Internet Search
-Action Input: How to handle RecursionError exceptions in Python
-
- - -#### Tool Input -
How to handle RecursionError exceptions in Python
-
- - -#### Tool Output -
- The Python RecursionError is an exception that occurs when the maximum recursion depth is exceeded. Learn how to fix it.
-- I have a reference to a function within an error. That is to say, if an error occurs, then the code re-references the function it is entered it runs again.
-- In this article, we will elucidate the Recursionerror In Python through examples, and we will also explore potential approaches to resolve this issue.
-- Make sure you recursive Python function has a terminating condition. If there is one, then consider the size of the data structure input to the ...
-- If an exception occurs during execution of the try clause, the exception may be handled by an except clause. If the exception is not handled by an except clause ...
-- Working on a bot that will write out text from a .CSV file at random, line by line as I run it, until all lines on that csv have been printed out.
-- I have an issue which I believe is caused by hitting the recursion limit inside of a getattr call. However, I have no idea how to debug this.
-- User code can raise built-in exceptions. This can be used to test an exception handler or to report an error condition “just like” the situation in which the ...
-- In this article, we shall look at why this error occurs and how to handle it in Python. To understand this, we need to first look at tail ...
-- The recursion limit is set too high. I think the problem is that fixRecursionLimit assumes that one python stack entry is no larger than 50 empty js function ...
- - - -## Step 3 : Inspect Code for Uncontrolled Recursion - - -> **Input**: *Inspect Code for Uncontrolled Recursion: Although the vulnerable package version is already verified to be installed, inspect the application code to identify any instances of uncontrolled recursion that could be exacerbated by the `email.utils.parseaddr` function's behavior, potentially leading to resource exhaustion or crashes.* - -> **Response**: *The Docker container does not appear to be vulnerable to uncontrolled recursion due to the `email.utils.parseaddr` function, as it is not used in the application code. However, there is a potential source of uncontrolled recursion in the `ProducerConsumerQueue` class due to the use of a recursive lock.* - -### Step 3.1 : *Docker Container Code QA System* - - -#### Action Log -
Thought: To answer this question, I need to inspect the application code within the Docker container to identify any instances of uncontrolled recursion that could be exacerbated by the `email.utils.parseaddr` function's behavior. I should use the Docker Container Code QA System to search for any usage of the `email.utils.parseaddr` function and analyze the surrounding code for potential recursion issues.
-
-Action: Docker Container Code QA System
-Action Input: Does the application code use the `email.utils.parseaddr` function, and if so, are there any instances of uncontrolled recursion in the surrounding code?
-
- - -#### Tool Input -
Does the application code use the `email.utils.parseaddr` function, and if so, are there any instances of uncontrolled recursion in the surrounding code?
-
- - -#### Tool Output -
No, the provided code does not use the `email.utils.parseaddr` function. Additionally, there is no indication of uncontrolled recursion in the surrounding code. The code appears to be related to file watching, data processing, and logging, but does not involve email parsing or recursive function calls that could lead to uncontrolled recursion.
- - Source Documents - - | ID | Type | source |... content_type |... language | Page Content | -| --- | --- | --- | --- | --- | --- | -| None | Document | morpheus/service/vdb/milvus_vector_db_service.py | functions_classes | python |
View ContentReturns ------- typing.Any The search result, which can vary depending on the query and options. Raises ------ RuntimeError If an error occurs during the search operation. If query argument is `None` and `data` keyword argument doesn't exist. If `data` keyword arguement is `None`. """ logger.debug("Searching in collection: %s, query=%s, kwargs=%s", self.\_name, query, kwargs) return self.\_client.query(collection\_name=self.\_name, filter=query, \*\*kwargs)
| -| None | Document | morpheus/stages/input/appshield_source_stage.py | functions_classes | python |
View Contentself.\_encoding = encoding self.\_input\_count = None self.\_watcher = DirectoryWatcher(input\_glob=input\_glob, watch\_directory=watch\_directory, max\_files=max\_files, sort\_glob=sort\_glob, recursive=recursive, queue\_max\_size=queue\_max\_size, batch\_timeout=batch\_timeout) @property def name(self) -> str: return "from-appshield" @property def input\_count(self) -> int: """Return None for no max intput count""" return self.\_input\_count def supports\_cpp\_node(self): return False
| -| None | Document | models/training-tuning-scripts/dfp-models/dfp-feature-selection-demo.ipynb | | |
View Content"... ... \n", "3234 External security challenge was not satisfied. \n", "3235 NaN \n", "3236 NaN \n", "3237 NaN \n", "3238 NaN \n", "\n", " properties.clientAppUsed \\\n", "0 Mobile Apps and Desktop clients \n", "1 Browser \n", "2 Mobile Apps and Desktop clients \n", "3 Mobile Apps and Desktop clients \n", "4 Mobile Apps and Desktop clients \n", "... ... \n",
| -| None | Document | morpheus/stages/input/azure_source_stage.py | functions_classes | python |
View ContentIf true the list of files matching `input\_glob` will be processed in sorted order. recursive: bool, default = True If true, events will be emitted for the files in subdirectories that match `input\_glob`. queue\_max\_size: int, default = 128 Maximum queue size to hold the file paths to be processed that match `input\_glob`. batch\_timeout: float, default = 5.0 Timeout to retrieve batch messages from the queue. """
| - - - - -[back to top](#checklist-toc) - -### Step 3.2 : *Docker Container Code QA System* - - -#### Action Log -
Thought: Based on the observation from the Docker Container Code QA System, it appears that the application code does not use the `email.utils.parseaddr` function, and there is no indication of uncontrolled recursion in the surrounding code. This suggests that the vulnerability related to uncontrolled recursion in the `email.utils.parseaddr` function may not be applicable to this Docker container.
-
-However, to further confirm this, I would like to investigate if there are any other functions or components in the application code that could potentially lead to uncontrolled recursion.
-
-Action: Docker Container Code QA System
-Action Input: Are there any other functions or components in the application code that could potentially lead to uncontrolled recursion?
-
- - -#### Tool Input -
Are there any other functions or components in the application code that could potentially lead to uncontrolled recursion?
-
- - -#### Tool Output -
Based on the provided code, there is one potential source of uncontrolled recursion: the use of a recursive lock (`threading.RLock`) in the `ProducerConsumerQueue` class. Recursive locks can lead to uncontrolled recursion if not used carefully, as they allow a thread to acquire the lock multiple times without releasing it.
-
-However, in this specific implementation, the recursive lock is used to prevent reentrant deadlocks, which suggests that the developer is aware of the potential risks and has taken steps to mitigate them.
-
-There are no other obvious sources of uncontrolled recursion in the provided code. The `DirectoryWatcher` class is not shown in the code snippet, but it's possible that it could contain recursive functions or methods that could lead to uncontrolled recursion. Without more information, it's impossible to say for sure.
-
-In general, it's always a good idea to carefully review code that uses recursive functions or locks to ensure that they are used safely and correctly.
- - Source Documents - - | ID | Type | source |... content_type |... language | Page Content | -| --- | --- | --- | --- | --- | --- | -| None | Document | morpheus/controllers/monitor_controller.py | functions_classes | python |
View Contentif isinstance(log\_level, LogLevels): # pylint: disable=isinstance-second-argument-not-valid-type log\_level = log\_level.value self.\_log\_level = log\_level self.\_enabled = None # defined on first call to \_is\_enabled @property def delayed\_start(self) -> bool: return self.\_delayed\_start @property def progress(self) -> tqdm: return self.\_progress def is\_enabled(self) -> bool: """ Returns a boolean indicating whether or not the logger is enabled. """ if self.\_enabled is None: self.\_enabled = logger.isEnabledFor(self.\_log\_level) return self.\_enabled
| -| None | Document | morpheus/utils/producer_consumer_queue.py | functions_classes | python |
View Contentclass ProducerConsumerQueue(queue.Queue, typing.Generic[\_T]): """ Custom queue.Queue implementation which supports closing and uses recursive locks Parameters ---------- maxsize : int Maximum size of queue. If maxsize is <= 0, the queue size is infinite. """ def \_\_init\_\_(self, maxsize: int = 0) -> None: super().\_\_init\_\_(maxsize=maxsize) # Use a recursive lock here to prevent reentrant deadlocks self.mutex = threading.RLock() self.not\_empty = threading.Condition(self.mutex) self.not\_full = threading.Condition(self.mutex) self.all\_tasks\_done = threading.Condition(self.mutex) self.\_is\_closed = False
| -| None | Document | morpheus/stages/input/appshield_source_stage.py | functions_classes | python |
View Contentself.\_encoding = encoding self.\_input\_count = None self.\_watcher = DirectoryWatcher(input\_glob=input\_glob, watch\_directory=watch\_directory, max\_files=max\_files, sort\_glob=sort\_glob, recursive=recursive, queue\_max\_size=queue\_max\_size, batch\_timeout=batch\_timeout) @property def name(self) -> str: return "from-appshield" @property def input\_count(self) -> int: """Return None for no max intput count""" return self.\_input\_count def supports\_cpp\_node(self): return False
| -| None | Document | morpheus/utils/directory_watcher.py | functions_classes | python |
View Contentmax\_files: int Max number of files to read. Useful for debugging to limit startup time. Default value of -1 is unlimited. sort\_glob : bool If true the list of files matching `input\_glob` will be processed in sorted order. recursive: bool If true, events will be emitted for the files in subdirectories matching `input\_glob`. queue\_max\_size: int Maximum queue size to hold the file paths to be processed that match `input\_glob`. batch\_timeout: float Timeout to retrieve batch messages from the queue. """
| - - - - -[back to top](#checklist-toc) - ---- -## References -Here are key references for further details on this vulnerability: -- https://www.cve.org/CVERecord?id=CVE-2023-36632 -- https://github.com/python/cpython/issues/103800 -- https://github.com/Daybreak2019/PoC_python3.9_Vul/blob/main/RecursionError-email.utils.parseaddr.py -- https://docs.python.org/3/library/email.html -- https://docs.python.org/3/library/email.utils.html -- https://github.com/advisories/GHSA-gv66-v8c8-v69c -- https://nvd.nist.gov/vuln/detail/CVE-2023-36632 - -[back to top](#cve-intro) From 2b68b79f462f77556d701b7058e0c52ae18868c0 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Wed, 10 Sep 2025 15:45:40 -0700 Subject: [PATCH 029/104] use one register.py file, remove separate evaluator_register.py --- src/vuln_analysis/configs/config-eval.yml | 2 +- src/vuln_analysis/configs/config-tracing.yml | 2 +- src/vuln_analysis/configs/config.yml | 2 +- src/vuln_analysis/eval/evaluator_register.py | 85 ------------------- .../eval/evaluators/consistency.py | 17 ++++ .../eval/evaluators/label_accuracy.py | 17 ++++ .../eval/evaluators/response_spread.py | 20 +++++ .../eval/evaluators/status_accuracy.py | 17 ++++ src/vuln_analysis/register.py | 5 +- 9 files changed, 78 insertions(+), 89 deletions(-) delete mode 100644 src/vuln_analysis/eval/evaluator_register.py diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml index d44448177..84b489bcc 100644 --- a/src/vuln_analysis/configs/config-eval.yml +++ b/src/vuln_analysis/configs/config-eval.yml @@ -151,7 +151,7 @@ workflow: eval: general: workflow_alias: "first_w&b_run" - output_dir: ./.tmp/eval/my_eval4 + output_dir: ./.tmp/evaluators dataset: _type: custom file_path: data/eval_datasets/eval_dataset.json diff --git a/src/vuln_analysis/configs/config-tracing.yml b/src/vuln_analysis/configs/config-tracing.yml index 888077a14..f525b1c4c 100644 --- a/src/vuln_analysis/configs/config-tracing.yml +++ b/src/vuln_analysis/configs/config-tracing.yml @@ -155,7 +155,7 @@ workflow: eval: general: - output_dir: ./.tmp/eval/cve_agent + output_dir: ./.tmp/profiler dataset: _type: json file_path: data/eval_datasets/profiler_dataset.json diff --git a/src/vuln_analysis/configs/config.yml b/src/vuln_analysis/configs/config.yml index 385c9a575..97c6e9e90 100644 --- a/src/vuln_analysis/configs/config.yml +++ b/src/vuln_analysis/configs/config.yml @@ -145,7 +145,7 @@ workflow: eval: general: - output_dir: ./.tmp/eval/cve_agent + output_dir: ./.tmp/profiler dataset: _type: json file_path: data/eval_datasets/profiler_dataset.json diff --git a/src/vuln_analysis/eval/evaluator_register.py b/src/vuln_analysis/eval/evaluator_register.py deleted file mode 100644 index 0ab7c1083..000000000 --- a/src/vuln_analysis/eval/evaluator_register.py +++ /dev/null @@ -1,85 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Register evaluator helper functions -from typing import Optional - -from nat.builder.builder import EvalBuilder -from nat.builder.evaluator import EvaluatorInfo -from nat.cli.register_workflow import register_evaluator -from nat.data_models.evaluator import EvaluatorBaseConfig - - -#evaluator registration (actual code is in eval/evaluators/status_accuracy.py) -class StatusAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="status_accuracy"): - '''Configuration for status accuracy evaluator''' - pass - - -@register_evaluator(config_type=StatusAccuracyEvaluatorConfig) -async def status_accuracy_evaluator(config: StatusAccuracyEvaluatorConfig, builder: EvalBuilder): - '''Register status accuracy evaluator''' - from .evaluators.status_accuracy import StatusAccuracyEvaluator - evaluator = StatusAccuracyEvaluator(builder.get_max_concurrency()) - - yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Status Accuracy Evaluator") - - -#evaluator registration (actual code is in eval/evaluators/label_accuracy.py) -class LabelAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="label_accuracy"): - '''Configuration for label accuracy evaluator''' - pass - - -@register_evaluator(config_type=LabelAccuracyEvaluatorConfig) -async def label_accuracy_evaluator(config: LabelAccuracyEvaluatorConfig, builder: EvalBuilder): - '''Register label accuracy evaluator''' - from .evaluators.label_accuracy import LabelAccuracyEvaluator - evaluator = LabelAccuracyEvaluator(builder.get_max_concurrency()) - - yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Label Accuracy Evaluator") - - -#evaluator registration (actual code is in eval/evaluators/response_spread.py) -class ResponseSpreadEvaluatorConfig(EvaluatorBaseConfig, name="response_spread"): - '''Configuration for response spread evaluator''' - aggregation: str = "average" # "average", "min", or "max" - stopwords: Optional[str] = None - - -@register_evaluator(config_type=ResponseSpreadEvaluatorConfig) -async def response_spread_evaluator(config: ResponseSpreadEvaluatorConfig, builder: EvalBuilder): - '''Register response spread evaluator''' - from .evaluators.response_spread import ResponseSpreadEvaluator - evaluator = ResponseSpreadEvaluator(max_concurrency=builder.get_max_concurrency(), - aggregation=config.aggregation, - stopwords=config.stopwords) - - yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Response Spread Evaluator") - - -#evaluator registration (actual code is in eval/evaluators/consistency.py) -class ConsistencyEvaluatorConfig(EvaluatorBaseConfig, name="consistency"): - '''Configuration for consistency evaluator''' - pass - - -@register_evaluator(config_type=ConsistencyEvaluatorConfig) -async def consistency_evaluator(config: ConsistencyEvaluatorConfig, builder: EvalBuilder): - '''Register consistency evaluator''' - from .evaluators.consistency import ConsistencyEvaluator - evaluator = ConsistencyEvaluator(builder.get_max_concurrency()) - - yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Consistency Evaluator") diff --git a/src/vuln_analysis/eval/evaluators/consistency.py b/src/vuln_analysis/eval/evaluators/consistency.py index 4bf47d45e..5fa6f0ab4 100644 --- a/src/vuln_analysis/eval/evaluators/consistency.py +++ b/src/vuln_analysis/eval/evaluators/consistency.py @@ -20,6 +20,10 @@ from tqdm import tqdm +from nat.builder.builder import EvalBuilder +from nat.builder.evaluator import EvaluatorInfo +from nat.cli.register_workflow import register_evaluator +from nat.data_models.evaluator import EvaluatorBaseConfig from nat.eval.evaluator.evaluator_model import EvalInput from nat.eval.evaluator.evaluator_model import EvalOutput from nat.eval.evaluator.evaluator_model import EvalOutputItem @@ -132,3 +136,16 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: ] return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) + + +# Evaluator registration +class ConsistencyEvaluatorConfig(EvaluatorBaseConfig, name="consistency"): + """Configuration for consistency evaluator""" + pass + + +@register_evaluator(config_type=ConsistencyEvaluatorConfig) +async def consistency_evaluator(config: ConsistencyEvaluatorConfig, builder: EvalBuilder): + """Register consistency evaluator""" + evaluator = ConsistencyEvaluator(builder.get_max_concurrency()) + yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Consistency Evaluator") diff --git a/src/vuln_analysis/eval/evaluators/label_accuracy.py b/src/vuln_analysis/eval/evaluators/label_accuracy.py index 0e659d3c4..5980d6708 100644 --- a/src/vuln_analysis/eval/evaluators/label_accuracy.py +++ b/src/vuln_analysis/eval/evaluators/label_accuracy.py @@ -18,6 +18,10 @@ from tqdm import tqdm +from nat.builder.builder import EvalBuilder +from nat.builder.evaluator import EvaluatorInfo +from nat.cli.register_workflow import register_evaluator +from nat.data_models.evaluator import EvaluatorBaseConfig from nat.eval.evaluator.evaluator_model import EvalInput from nat.eval.evaluator.evaluator_model import EvalOutput from nat.eval.evaluator.evaluator_model import EvalOutputItem @@ -92,3 +96,16 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: ] return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) + + +# Evaluator registration +class LabelAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="label_accuracy"): + """Configuration for label accuracy evaluator""" + pass + + +@register_evaluator(config_type=LabelAccuracyEvaluatorConfig) +async def label_accuracy_evaluator(config: LabelAccuracyEvaluatorConfig, builder: EvalBuilder): + """Register label accuracy evaluator""" + evaluator = LabelAccuracyEvaluator(builder.get_max_concurrency()) + yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Label Accuracy Evaluator") diff --git a/src/vuln_analysis/eval/evaluators/response_spread.py b/src/vuln_analysis/eval/evaluators/response_spread.py index da8218d7d..3c331603c 100644 --- a/src/vuln_analysis/eval/evaluators/response_spread.py +++ b/src/vuln_analysis/eval/evaluators/response_spread.py @@ -21,6 +21,10 @@ from tqdm import tqdm +from nat.builder.builder import EvalBuilder +from nat.builder.evaluator import EvaluatorInfo +from nat.cli.register_workflow import register_evaluator +from nat.data_models.evaluator import EvaluatorBaseConfig from nat.eval.evaluator.evaluator_model import EvalInput from nat.eval.evaluator.evaluator_model import EvalOutput from nat.eval.evaluator.evaluator_model import EvalOutputItem @@ -136,3 +140,19 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: ] return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) + + +# Evaluator registration +class ResponseSpreadEvaluatorConfig(EvaluatorBaseConfig, name="response_spread"): + """Configuration for response spread evaluator""" + aggregation: str = "average" # "average", "min", or "max" + stopwords: str | None = None + + +@register_evaluator(config_type=ResponseSpreadEvaluatorConfig) +async def response_spread_evaluator(config: ResponseSpreadEvaluatorConfig, builder: EvalBuilder): + """Register response spread evaluator""" + evaluator = ResponseSpreadEvaluator(max_concurrency=builder.get_max_concurrency(), + aggregation=config.aggregation, + stopwords=config.stopwords) + yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Response Spread Evaluator") diff --git a/src/vuln_analysis/eval/evaluators/status_accuracy.py b/src/vuln_analysis/eval/evaluators/status_accuracy.py index 1f4741e12..7fb0bbcb8 100644 --- a/src/vuln_analysis/eval/evaluators/status_accuracy.py +++ b/src/vuln_analysis/eval/evaluators/status_accuracy.py @@ -18,6 +18,10 @@ from tqdm import tqdm +from nat.builder.builder import EvalBuilder +from nat.builder.evaluator import EvaluatorInfo +from nat.cli.register_workflow import register_evaluator +from nat.data_models.evaluator import EvaluatorBaseConfig from nat.eval.evaluator.evaluator_model import EvalInput from nat.eval.evaluator.evaluator_model import EvalOutput from nat.eval.evaluator.evaluator_model import EvalOutputItem @@ -100,3 +104,16 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: ] return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) + + +# Evaluator registration +class StatusAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="status_accuracy"): + """Configuration for status accuracy evaluator""" + pass + + +@register_evaluator(config_type=StatusAccuracyEvaluatorConfig) +async def status_accuracy_evaluator(config: StatusAccuracyEvaluatorConfig, builder: EvalBuilder): + """Register status accuracy evaluator""" + evaluator = StatusAccuracyEvaluator(builder.get_max_concurrency()) + yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Status Accuracy Evaluator") diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index 55e190085..d7d161d19 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -47,7 +47,10 @@ from vuln_analysis.utils.llm_engine_utils import preprocess_engine_input # Import evaluator registrations -from vuln_analysis.eval import evaluator_register +from vuln_analysis.eval.evaluators import status_accuracy +from vuln_analysis.eval.evaluators import label_accuracy +from vuln_analysis.eval.evaluators import response_spread +from vuln_analysis.eval.evaluators import consistency logger = logging.getLogger(__name__) From 99ad1520cb1afbe380e447d29544fab10759eb3c Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Fri, 19 Sep 2025 15:30:32 -0700 Subject: [PATCH 030/104] change eval test set file structure so each test set has its own .json --- src/vuln_analysis/configs/config-eval.yml | 28 ++- .../data/eval_datasets/common-test-set.json | 106 ++++++++++ .../data/eval_datasets/eval_dataset.json | 181 ----------------- .../morpheus-23.11-test-set.json | 71 +++++++ .../morpheus-24.03-test-set.json | 76 ++++++++ .../eval/evaluators/consistency.py | 15 +- .../eval/evaluators/label_accuracy.py | 21 +- .../eval/evaluators/response_spread.py | 15 +- .../eval/evaluators/status_accuracy.py | 22 +-- src/vuln_analysis/eval/parse_eval_input.py | 182 ++++++------------ 10 files changed, 364 insertions(+), 353 deletions(-) create mode 100644 src/vuln_analysis/data/eval_datasets/common-test-set.json delete mode 100644 src/vuln_analysis/data/eval_datasets/eval_dataset.json create mode 100644 src/vuln_analysis/data/eval_datasets/morpheus-23.11-test-set.json create mode 100644 src/vuln_analysis/data/eval_datasets/morpheus-24.03-test-set.json diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml index 84b489bcc..c460f7e2d 100644 --- a/src/vuln_analysis/configs/config-eval.yml +++ b/src/vuln_analysis/configs/config-eval.yml @@ -15,11 +15,11 @@ general: use_uvloop: true - telemetry: - tracing: - weave: - _type: weave - project: "cve-agent" + # telemetry: + # tracing: + # weave: #WIP + # _type: weave + # project: "cve-agent" functions: cve_generate_vdbs: @@ -150,16 +150,12 @@ workflow: eval: general: - workflow_alias: "first_w&b_run" output_dir: ./.tmp/evaluators dataset: _type: custom - file_path: data/eval_datasets/eval_dataset.json + file_path: data/eval_datasets/common-test-set.json + container_ids: ["morpheus-23.11", "morpheus-24.03"] function: vuln_analysis.eval.parse_eval_input.parse_input - kwargs: - test_set_id: "test-set-common" - container_ids: ["morpheus-23.11"] #["morpheus-23.11", "morpheus-24.03"] - # comment out container_ids to use all available containers evaluators: status_accuracy: @@ -167,8 +163,8 @@ eval: label_accuracy: _type: label_accuracy # Uncomment to enable response spread and consistency evaluators - response_spread: - _type: response_spread - aggregation: "average" #Options: average, min, max - consistency: - _type: consistency + # response_spread: + # _type: response_spread + # aggregation: "average" #Options: average, min, max + # consistency: + # _type: consistency diff --git a/src/vuln_analysis/data/eval_datasets/common-test-set.json b/src/vuln_analysis/data/eval_datasets/common-test-set.json new file mode 100644 index 000000000..adea2fb1a --- /dev/null +++ b/src/vuln_analysis/data/eval_datasets/common-test-set.json @@ -0,0 +1,106 @@ +{ "test_set_description": "Common test set for all containers", + "containers": { + "morpheus-23.11": { + "container_image": { + "name": "nvcr.io/nvidia/morpheus/morpheus", + "tag": "23.11-runtime", + "source_info": [ + { + "type": "code", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "ref": "branch-23.11", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ], + "exclude": [ + "tests/**/*" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "ref": "branch-23.11", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + } + ], + "sbom_info": { + "_type": "file", + "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom" + } + }, + "ground_truth": [ + { + "vuln_id": "GHSA-3f63-hfp8-52jq", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + }, + { + "vuln_id": "CVE-2023-36632", + "status": "AFFECTED", + "label": "vulnerable" + } + ] + }, + "morpheus-24.03": { + "container_image": { + "name": "nvcr.io/nvidia/morpheus/morpheus", + "tag": "v24.03.02-runtime", + "source_info": [ + { + "type": "code", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "ref": "v24.03.02", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ], + "exclude": [ + "tests/**/*" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "ref": "v24.03.02", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + } + ], + "sbom_info": { + "_type": "file", + "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v24.03.02-runtime.sbom" + } + }, + "ground_truth": [ + { + "vuln_id": "GHSA-3f63-hfp8-52jq", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + }, + { + "vuln_id": "CVE-2023-36632", + "status": "AFFECTED", + "label": "vulnerable" + } + ] + } + } +} diff --git a/src/vuln_analysis/data/eval_datasets/eval_dataset.json b/src/vuln_analysis/data/eval_datasets/eval_dataset.json deleted file mode 100644 index 5142acdee..000000000 --- a/src/vuln_analysis/data/eval_datasets/eval_dataset.json +++ /dev/null @@ -1,181 +0,0 @@ -{ - "containers": { - "morpheus-23.11": { - "container_image": { - "name": "nvcr.io/nvidia/morpheus/morpheus", - "tag": "23.11-runtime", - "source_info": [ - { - "type": "code", - "git_repo": "https://github.com/nv-morpheus/Morpheus.git", - "ref": "branch-23.11", - "include": [ - "**/*.cpp", - "**/*.cu", - "**/*.cuh", - "**/*.h", - "**/*.hpp", - "**/*.ipynb", - "**/*.py", - "**/*Dockerfile" - ], - "exclude": [ - "tests/**/*" - ] - }, - { - "type": "doc", - "git_repo": "https://github.com/nv-morpheus/Morpheus.git", - "ref": "branch-23.11", - "include": [ - "**/*.md", - "docs/**/*.rst" - ] - } - ], - "sbom_info": { - "_type": "file", - "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom" - } - }, - "test_sets": { - "morpheus-23.11-test-set": { - "description": "morpheus-23.11 only test set", - "ground_truth": [ - { - "vuln_id": "GHSA-3f63-hfp8-52jq", - "status": "NOT AFFECTED", - "label": "code_not_reachable" - }, - { - "vuln_id": "CVE-2023-36632", - "status": "AFFECTED", - "label": "vulnerable" - }, - { - "vuln_id": "GHSA-cxfr-5q3r-2rc2", - "status": "AFFECTED", - "label": "vulnerable" - }, - { - "vuln_id": "GHSA-554w-xh4j-8w64", - "status": "AFFECTED", - "label": "vulnerable" - }, - { - "vuln_id": "CVE-2023-51767", - "status": "AFFECTED", - "label": "vulnerable" - } - ] - }, - "test-set-common": { - "description": "Common test set for both containers", - "ground_truth": [ - { - "vuln_id": "GHSA-3f63-hfp8-52jq", - "status": "NOT AFFECTED", - "label": "code_not_reachable" - }, - { - "vuln_id": "CVE-2023-36632", - "status": "AFFECTED", - "label": "vulnerable" - } - ] - } - } - }, - "morpheus-24.03": { - "container_image": { - "name": "nvcr.io/nvidia/morpheus/morpheus", - "tag": "v24.03.02-runtime", - "source_info": [ - { - "type": "code", - "git_repo": "https://github.com/nv-morpheus/Morpheus.git", - "ref": "v24.03.02", - "include": [ - "**/*.cpp", - "**/*.cu", - "**/*.cuh", - "**/*.h", - "**/*.hpp", - "**/*.ipynb", - "**/*.py", - "**/*Dockerfile" - ], - "exclude": [ - "tests/**/*" - ] - }, - { - "type": "doc", - "git_repo": "https://github.com/nv-morpheus/Morpheus.git", - "ref": "v24.03.02", - "include": [ - "**/*.md", - "docs/**/*.rst" - ] - } - ], - "sbom_info": { - "_type": "file", - "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v24.03.02-runtime.sbom" - } - }, - "test_sets": { - "morpheus-24.03-test-set": { - "description": "morpheus-24.03 only test set (toy data, not real ground truth)", - "ground_truth": [ - { - "vuln_id": "GHSA-3f63-hfp8-52jq", - "status": "NOT AFFECTED", - "label": "code_not_reachable" - }, - { - "vuln_id": "CVE-2023-50782", - "status": "AFFECTED", - "label": "vulnerable" - }, - { - "vuln_id": "CVE-2023-36632", - "status": "AFFECTED", - "label": "vulnerable" - }, - { - "vuln_id": "CVE-2023-43804", - "status": "NOT AFFECTED", - "label": "code_not_reachable" - }, - { - "vuln_id": "GHSA-cxfr-5q3r-2rc2", - "status": "AFFECTED", - "label": "vulnerable" - }, - { - "vuln_id": "GHSA-554w-xh4j-8w64", - "status": "AFFECTED", - "label": "vulnerable" - } - ] - }, - "test-set-common": { - "description": "Common test set for both containers", - "ground_truth": [ - { - "vuln_id": "GHSA-3f63-hfp8-52jq", - "status": "NOT AFFECTED", - "label": "code_not_reachable" - }, - { - "vuln_id": "CVE-2023-36632", - "status": "AFFECTED", - "label": "vulnerable" - } - ] - } - } - } - } -} diff --git a/src/vuln_analysis/data/eval_datasets/morpheus-23.11-test-set.json b/src/vuln_analysis/data/eval_datasets/morpheus-23.11-test-set.json new file mode 100644 index 000000000..0eb118a56 --- /dev/null +++ b/src/vuln_analysis/data/eval_datasets/morpheus-23.11-test-set.json @@ -0,0 +1,71 @@ +{ + "test_set_description": "Test set for morpheus-23.11 container", + "containers": { + "morpheus-23.11": { + "container_image": { + "name": "nvcr.io/nvidia/morpheus/morpheus", + "tag": "23.11-runtime", + "source_info": [ + { + "type": "code", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "ref": "branch-23.11", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ], + "exclude": [ + "tests/**/*" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "ref": "branch-23.11", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + } + ], + "sbom_info": { + "_type": "file", + "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom" + } + }, + "ground_truth": [ + { + "vuln_id": "GHSA-3f63-hfp8-52jq", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + }, + { + "vuln_id": "CVE-2023-36632", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "GHSA-cxfr-5q3r-2rc2", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "GHSA-554w-xh4j-8w64", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "CVE-2023-51767", + "status": "AFFECTED", + "label": "vulnerable" + } + ] + } + } +} diff --git a/src/vuln_analysis/data/eval_datasets/morpheus-24.03-test-set.json b/src/vuln_analysis/data/eval_datasets/morpheus-24.03-test-set.json new file mode 100644 index 000000000..dbdef0b0e --- /dev/null +++ b/src/vuln_analysis/data/eval_datasets/morpheus-24.03-test-set.json @@ -0,0 +1,76 @@ +{ + "test_set_description": "Test set for morpheus-24.03 container (toy data)", + "containers": { + "morpheus-24.03": { + "container_image": { + "name": "nvcr.io/nvidia/morpheus/morpheus", + "tag": "v24.03.02-runtime", + "source_info": [ + { + "type": "code", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "ref": "v24.03.02", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ], + "exclude": [ + "tests/**/*" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "ref": "v24.03.02", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + } + ], + "sbom_info": { + "_type": "file", + "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v24.03.02-runtime.sbom" + } + }, + "ground_truth": [ + { + "vuln_id": "GHSA-3f63-hfp8-52jq", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + }, + { + "vuln_id": "CVE-2023-50782", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "CVE-2023-36632", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "CVE-2023-43804", + "status": "NOT AFFECTED", + "label": "code_not_reachable" + }, + { + "vuln_id": "GHSA-cxfr-5q3r-2rc2", + "status": "AFFECTED", + "label": "vulnerable" + }, + { + "vuln_id": "GHSA-554w-xh4j-8w64", + "status": "AFFECTED", + "label": "vulnerable" + } + ] + } + } +} diff --git a/src/vuln_analysis/eval/evaluators/consistency.py b/src/vuln_analysis/eval/evaluators/consistency.py index 5fa6f0ab4..719dac2d7 100644 --- a/src/vuln_analysis/eval/evaluators/consistency.py +++ b/src/vuln_analysis/eval/evaluators/consistency.py @@ -15,20 +15,19 @@ import asyncio import json -import numpy as np from collections import Counter -from tqdm import tqdm - +import numpy as np from nat.builder.builder import EvalBuilder from nat.builder.evaluator import EvaluatorInfo from nat.cli.register_workflow import register_evaluator from nat.data_models.evaluator import EvaluatorBaseConfig from nat.eval.evaluator.evaluator_model import EvalInput +from nat.eval.evaluator.evaluator_model import EvalInputItem from nat.eval.evaluator.evaluator_model import EvalOutput from nat.eval.evaluator.evaluator_model import EvalOutputItem -from nat.eval.evaluator.evaluator_model import EvalInputItem from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry +from tqdm import tqdm class ConsistencyEvaluator: @@ -96,8 +95,10 @@ async def process_item(item): all_status_consistencies.append(status_consistency) # Calculate overall average consistency - overall_label_consistency = sum(all_label_consistencies) / len(all_label_consistencies) if all_label_consistencies else 1.0 - overall_status_consistency = sum(all_status_consistencies) / len(all_status_consistencies) if all_status_consistencies else 1.0 + overall_label_consistency = sum(all_label_consistencies) / len( + all_label_consistencies) if all_label_consistencies else 1.0 + overall_status_consistency = sum(all_status_consistencies) / len( + all_status_consistencies) if all_status_consistencies else 1.0 reasoning = { "label_consistencies": label_consistencies, "status_consistencies": status_consistencies, @@ -105,7 +106,7 @@ async def process_item(item): "overall_status_consistency": overall_status_consistency, "num_cves": len(vuln_responses), "total_responses": sum(len(responses['labels']) for responses in vuln_responses.values()), - "test_set_id": item.full_dataset_entry.get('test_set_id', 'unknown'), + "test_set_file": item.full_dataset_entry.get('test_set_file', 'unknown'), } # Return average of label and status consistency diff --git a/src/vuln_analysis/eval/evaluators/label_accuracy.py b/src/vuln_analysis/eval/evaluators/label_accuracy.py index 5980d6708..f27917a40 100644 --- a/src/vuln_analysis/eval/evaluators/label_accuracy.py +++ b/src/vuln_analysis/eval/evaluators/label_accuracy.py @@ -16,17 +16,16 @@ import asyncio import json -from tqdm import tqdm - from nat.builder.builder import EvalBuilder from nat.builder.evaluator import EvaluatorInfo from nat.cli.register_workflow import register_evaluator from nat.data_models.evaluator import EvaluatorBaseConfig from nat.eval.evaluator.evaluator_model import EvalInput +from nat.eval.evaluator.evaluator_model import EvalInputItem from nat.eval.evaluator.evaluator_model import EvalOutput from nat.eval.evaluator.evaluator_model import EvalOutputItem -from nat.eval.evaluator.evaluator_model import EvalInputItem from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry +from tqdm import tqdm class LabelAccuracyEvaluator: @@ -47,15 +46,15 @@ async def process_item(item): label_accuracies = [] generated_answers = [] question = [] - + for result in pipeline_result: vuln_id = result['vuln_id'] question.append(vuln_id) - + # Extract label from pipeline result justification_label = result.get('justification', {}).get('label', 'MISSING') generated_answers.append(justification_label) - + # Calculate label accuracy # Ground truth labels are stored as "{vuln_id}_label" in the ground truth map # This is populated by parse_eval_input.py from the eval dataset JSON files @@ -67,11 +66,11 @@ async def process_item(item): "question": question, "answer": [ground_truth_map.get(f"{q}_label", 'MISSING') for q in question], "generated_answer": generated_answers, - "test_set_id": item.full_dataset_entry.get('test_set_id', 'unknown'), + "test_set_file": item.full_dataset_entry.get('test_set_file', 'unknown'), } - return sum(label_accuracies)/len(label_accuracies) if label_accuracies else 0.0, reasoning - + return sum(label_accuracies) / len(label_accuracies) if label_accuracies else 0.0, reasoning + async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: async with self.semaphore: result = await process_item(item) @@ -80,7 +79,9 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: try: tqdm_position = TqdmPositionRegistry.claim() - pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Label Accuracy", position=tqdm_position) + pbar = tqdm(total=len(eval_input.eval_input_items), + desc="Evaluating Label Accuracy", + position=tqdm_position) results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) finally: pbar.close() diff --git a/src/vuln_analysis/eval/evaluators/response_spread.py b/src/vuln_analysis/eval/evaluators/response_spread.py index 3c331603c..af7b9a637 100644 --- a/src/vuln_analysis/eval/evaluators/response_spread.py +++ b/src/vuln_analysis/eval/evaluators/response_spread.py @@ -15,21 +15,20 @@ import asyncio import json -import numpy as np from collections import Counter -from sklearn.feature_extraction.text import CountVectorizer - -from tqdm import tqdm +import numpy as np from nat.builder.builder import EvalBuilder from nat.builder.evaluator import EvaluatorInfo from nat.cli.register_workflow import register_evaluator from nat.data_models.evaluator import EvaluatorBaseConfig from nat.eval.evaluator.evaluator_model import EvalInput +from nat.eval.evaluator.evaluator_model import EvalInputItem from nat.eval.evaluator.evaluator_model import EvalOutput from nat.eval.evaluator.evaluator_model import EvalOutputItem -from nat.eval.evaluator.evaluator_model import EvalInputItem from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry +from sklearn.feature_extraction.text import CountVectorizer +from tqdm import tqdm class ResponseSpreadEvaluator: @@ -111,7 +110,7 @@ async def process_item(item): "aggregation_method": self.aggregation, "num_cves": len(vuln_responses), "total_responses": sum(len(responses) for responses in vuln_responses.values()), - "test_set_id": item.full_dataset_entry.get('test_set_id', 'unknown'), + "test_set_file": item.full_dataset_entry.get('test_set_file', 'unknown'), } return overall_spread, reasoning @@ -124,7 +123,9 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: try: tqdm_position = TqdmPositionRegistry.claim() - pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Response Spread", position=tqdm_position) + pbar = tqdm(total=len(eval_input.eval_input_items), + desc="Evaluating Response Spread", + position=tqdm_position) results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) finally: pbar.close() diff --git a/src/vuln_analysis/eval/evaluators/status_accuracy.py b/src/vuln_analysis/eval/evaluators/status_accuracy.py index 7fb0bbcb8..4053c6519 100644 --- a/src/vuln_analysis/eval/evaluators/status_accuracy.py +++ b/src/vuln_analysis/eval/evaluators/status_accuracy.py @@ -16,18 +16,16 @@ import asyncio import json -from tqdm import tqdm - from nat.builder.builder import EvalBuilder from nat.builder.evaluator import EvaluatorInfo from nat.cli.register_workflow import register_evaluator from nat.data_models.evaluator import EvaluatorBaseConfig from nat.eval.evaluator.evaluator_model import EvalInput +from nat.eval.evaluator.evaluator_model import EvalInputItem from nat.eval.evaluator.evaluator_model import EvalOutput from nat.eval.evaluator.evaluator_model import EvalOutputItem -from nat.eval.evaluator.evaluator_model import EvalInputItem from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry - +from tqdm import tqdm _LABEL_MAP = {'TRUE': 'AFFECTED', 'FALSE': 'NOT AFFECTED', 'UNKNOWN': 'UNKNOWN'} @@ -50,36 +48,36 @@ async def process_item(item): status_accuracies = [] generated_status_answers = [] question = [] - + for result in pipeline_result: vuln_id = result['vuln_id'] question.append(vuln_id) - + # Extract status from pipeline result affected_status = result.get('justification', {}).get('status', 'MISSING') - + status_result = _LABEL_MAP.get(affected_status, 'MISSING') generated_status_answers.append(status_result) - + # ORIGINAL LOGIC - ground truth MISSING factors into accuracy calculation # accuracies.append(float(status_result == ground_truth_map.get(vuln_id, 'MISSING'))) - + # NEW LOGIC - Only include in accuracy calculation if ground truth exists (not MISSING) ground_truth_status = ground_truth_map.get(vuln_id) if ground_truth_status is not None: status_accuracies.append(float(status_result == ground_truth_status)) # Calculate average accuracy - status_avg = sum(status_accuracies)/len(status_accuracies) if status_accuracies else 0.0 + status_avg = sum(status_accuracies) / len(status_accuracies) if status_accuracies else 0.0 reasoning = { "question": question, "status_answer": [ground_truth_map.get(q, 'MISSING') for q in question], "generated_status_answer": generated_status_answers, - "test_set_id": item.full_dataset_entry.get('test_set_id', 'unknown'), + "test_set_file": item.full_dataset_entry.get('test_set_file', 'unknown'), } return status_avg, reasoning - + async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: async with self.semaphore: result = await process_item(item) diff --git a/src/vuln_analysis/eval/parse_eval_input.py b/src/vuln_analysis/eval/parse_eval_input.py index 5ff8d1009..e2f976860 100644 --- a/src/vuln_analysis/eval/parse_eval_input.py +++ b/src/vuln_analysis/eval/parse_eval_input.py @@ -5,186 +5,129 @@ from nat.eval.evaluator.evaluator_model import EvalInputItem -def parse_input(file_path: Path, test_set_id: str | None = None, container_ids: list[str] | None = None, max_test_sets: int | None = None) -> EvalInput: +def parse_input(file_path: Path, container_ids: list[str] | None = None) -> EvalInput: """ - To run just this script outside of the nat pipeline: - python3 parse_eval_input.py eval-dataset.json --test-set-id test-set-small --container-ids morpheus-23.11 --preview -o my_eval.json - Note: need to run inside vuln_analysis docker container where nat is installed + To run just this script outside of the full nat pipeline: + python3 parse_eval_input.py [test_set_file].json --container-ids [containers] --preview -o [your output file].json + Ex: python3 parse_eval_input.py ../data/eval_datasets/common-test-set.json --container-ids morpheus-23.11 morpheus-24.03 --preview -o my_eval.json + + Note: Run this inside the vuln_analysis container, from the eval directory """ - # Load the container image config JSON + # Load the test set file (validation should be done by Pydantic models before this function) with open(file_path, 'r', encoding='utf-8') as f: config_data = json.load(f) - + # Extract containers containers = config_data.get('containers', {}) - - if not containers: - print("No containers found in config file") - return EvalInput(eval_input_items=[]) - - # Filter containers if specific IDs provided + + # Filter containers if specific container IDs provided if container_ids is not None: available_containers = {k: v for k, v in containers.items() if k in container_ids} - if not available_containers: - print(f"No containers found with IDs: {container_ids}") - print(f"Available containers: {list(containers.keys())}") - return EvalInput(eval_input_items=[]) + print(f"Using specified containers: {list(available_containers.keys())}") else: available_containers = containers print(f"No container IDs specified, using all available containers: {list(containers.keys())}") - - # Find test set across all containers (test set must exist in all containers) - if test_set_id is not None: - # Check if test set exists in ALL specified containers - missing_containers = [] - for container_id, container_data in available_containers.items(): - test_sets = container_data.get('test_sets', {}) - if test_set_id not in test_sets: - missing_containers.append(container_id) - - if missing_containers: - print(f"Test set '{test_set_id}' not found in containers: {missing_containers}") - print("Available test sets across all containers:") - for container_id, container_data in available_containers.items(): - test_sets = container_data.get('test_sets', {}) - print(f" {container_id}: {list(test_sets.keys())}") - return EvalInput(eval_input_items=[]) - - print(f"Processing test set: {test_set_id}") - else: - print("Error: No test set ID specified") - print("Available test sets per container:") - for container_id, container_data in available_containers.items(): - test_sets = container_data.get('test_sets', {}) - print(f" {container_id}: {list(test_sets.keys())}") - return EvalInput(eval_input_items=[]) - + + # Get test set description from the file + test_set_description = config_data.get('test_set_description', 'Unknown test set') + test_set_file = file_path.name # Extract filename from the file path + print(f"Processing test set: {test_set_description} (file: {test_set_file})") + # Create evaluation items for each container eval_items = [] - + for container_id, container_data in available_containers.items(): container_image = container_data.get('container_image', {}) - test_sets = container_data.get('test_sets', {}) - - if not container_image: - print(f"No container_image found for container: {container_id}") - continue - - # Get the test set (we already verified it exists in all containers) - target_test_set = test_sets[test_set_id] - ground_truth = target_test_set.get('ground_truth', []) - + ground_truth = container_data.get('ground_truth', []) + # Create scan structure with vulnerability IDs from ground truth scan_vulns = [] ground_truth_mapping = {} - + for gt_item in ground_truth: vuln_id = gt_item.get('vuln_id') status = gt_item.get('status') label = gt_item.get('label') - + if vuln_id and status: # Add to scan structure scan_vulns.append({"vuln_id": vuln_id}) - + # Add to ground truth mapping ground_truth_mapping[vuln_id] = status - + # Add label to ground truth mapping if available if label: ground_truth_mapping[f"{vuln_id}_label"] = label - + # Create the question structure (container_image + scan) - question_structure = { - "image": container_image, - "scan": { - "vulns": scan_vulns - } - } - - # Convert to stringified format + question_structure = {"image": container_image, "scan": {"vulns": scan_vulns}} + + # Convert to stringified format compatible with NAT EvalInputItem format question_string = json.dumps(question_structure, separators=(',', ':')) answer_string = json.dumps(ground_truth_mapping, separators=(',', ':')) - + # Create evaluation item - eval_item = EvalInputItem( - id=container_id, - input_obj=question_string, - expected_output_obj=answer_string, - full_dataset_entry={ - "id": container_id, - "question": question_string, - "answer": answer_string, - "test_set_id": test_set_id, - "container_id": container_id, - "original_test_set": target_test_set - } - ) - + eval_item = EvalInputItem(id=container_id, + input_obj=question_string, + expected_output_obj=answer_string, + full_dataset_entry={ + "id": container_id, + "question": question_string, + "answer": answer_string, + "test_set_file": test_set_file, + "test_set_description": test_set_description, + "container_id": container_id, + "ground_truth": ground_truth + }) + eval_items.append(eval_item) - + if not eval_items: print("No valid evaluation items created") return EvalInput(eval_input_items=[]) - - print(f"Created {len(eval_items)} evaluation items for test set '{test_set_id}'") + + print(f"Created {len(eval_items)} evaluation items for test set '{test_set_description}'") return EvalInput(eval_input_items=eval_items) -# For if you want to look at the eval input string +# For debugging - saves the EvalInputItem string to a file def save_as_baseline_string_format(eval_input: EvalInput, output_path: Path) -> None: baseline_data = [] - + for item in eval_input.eval_input_items: - baseline_item = { - "id": item.id, - "question": item.input_obj, - "answer": item.expected_output_obj - } + baseline_item = {"id": item.id, "question": item.input_obj, "answer": item.expected_output_obj} baseline_data.append(baseline_item) - + with open(output_path, 'w', encoding='utf-8') as f: json.dump(baseline_data, f, indent=4) - + print(f"Saved baseline string format to: {output_path}") def main(): import argparse - + parser = argparse.ArgumentParser( description='Parse consolidated evaluation dataset with multiple containers and test sets', formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument('input_file', help='Path to eval-dataset.json file') parser.add_argument('-o', '--output', help='Output path for baseline string format (optional)') - parser.add_argument('--test-set-id', help='Specific test set ID to process (e.g., --test-set-id test-set-small)') - parser.add_argument('--container-ids', nargs='+', help='Specific container IDs to process (e.g., --container-ids morpheus-23.11 morpheus-24.03)') - parser.add_argument('--max-test-sets', type=int, help='Maximum number of test sets to process') + parser.add_argument('--container-ids', + nargs='+', + help='Specific container IDs to process (e.g., --container-ids morpheus-23.11 morpheus-24.03)') parser.add_argument('--preview', action='store_true', help='Show preview of first converted item') - + args = parser.parse_args() - + try: input_path = Path(args.input_file) - + # Convert to EvalInput format - eval_input = parse_input(input_path, args.test_set_id, args.container_ids, args.max_test_sets) - - if args.preview and eval_input.eval_input_items: - print(f"Total evaluation items: {len(eval_input.eval_input_items)}") - for i, item in enumerate(eval_input.eval_input_items[:3]): # Show first 3 items - print(f"\n--- Item {i} ---") - print(f"ID: {item.id}") - print(f"Container ID: {item.full_dataset_entry.get('container_id', 'unknown')}") - print(f"Test Set ID: {item.full_dataset_entry.get('test_set_id', 'unknown')}") - print(f"Question (first 200 chars):") - print(f"{item.input_obj[:200]}...") - print(f"Answer:") - print(f"{item.expected_output_obj}") - print("="*80) - - # Save as baseline string format if output specified + eval_input = parse_input(input_path, args.container_ids) + + # Save as baseline string format if output is specified if args.output: output_path = Path(args.output) save_as_baseline_string_format(eval_input, output_path) @@ -193,8 +136,7 @@ def main(): eval_dir = Path(__file__).parent default_output = eval_dir / "eval_input_string.json" save_as_baseline_string_format(eval_input, default_output) - - + except FileNotFoundError as e: print(f"Error: Input file not found - {e}") return 1 @@ -204,7 +146,7 @@ def main(): except Exception as e: print(f"Unexpected error: {e}") return 1 - + return 0 From f91f1bcadae156ace2d9b45530c5d34f10d37fd1 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Fri, 19 Sep 2025 15:44:28 -0700 Subject: [PATCH 031/104] update readme to match new eval test set input file structure. moved profiler test data outside of eval_datasets dir --- README.md | 36 +++++++++---------- .../{eval_datasets => }/profiler_dataset.json | 0 src/vuln_analysis/eval/parse_eval_input.py | 2 +- 3 files changed, 19 insertions(+), 19 deletions(-) rename src/vuln_analysis/data/{eval_datasets => }/profiler_dataset.json (100%) diff --git a/README.md b/README.md index 5ca19d87f..75663faf8 100644 --- a/README.md +++ b/README.md @@ -765,42 +765,42 @@ Each CVE has one ground truth status and one corresponding label. Valid statuses include: `AFFECTED` or `NOT AFFECTED`. Valid labels include: `vulnerable` if the CVE status is `AFFECTED`, or one of 10 VEX statuses if the status is `NOT AFFECTED` (see **Justification Assignment** under the [Key Components section](#key-components)). -To run evaluaton, run `nat eval` using the `config-eval.yml` configuration. +To run evaluaton, run `nat eval` using the `config-eval.yml` configuration. ``` nat eval --config_file=configs/config-eval.yml ``` -This will write the files `status_accuracy_eval_output.json` and `label_accuracy_eval_output.json` to the configured output directory (`.tmp/eval/my_eval` in this example), containing accuracy scores for status and label respectively. Scores range from 0 to 1, with 0 indicating no CVEs were categorized correctly, and 1 indicating all CVEs were categorized correctly. +This will write the files `status_accuracy_output.json` and `label_accuracy_output.json` to the configured output directory (`.tmp/evaluators` in this example), containing accuracy scores for status and label respectively. Scores range from 0 to 1, with 0 indicating no CVEs were categorized correctly, and 1 indicating all CVEs were categorized correctly. ### Inputting Evaluation Test Sets -Ground truth datasets should be put in `data/eval_datasets/eval_dataset.json`. The file contains all containers and CVEs to be tested, and has the following structure: +Ground truth datasets should be put in the `data/eval_datasets/` directory. Each eval dataset should be a separate json, with the following structure: ``` +test-set-description containers: - container1 - container1 metadata - test-setA: {list of CVEs} - test-setB: {list of CVEs} - container2 - container2 metadata - test-setB: {list of CVEs} - test-setC: {list of CVEs} + containerA + metadata + ground-truth: [list of CVEs] + containerB + metadata + ground-truth: [list of CVEs] + ... ``` -In `config-eval.yml`, an example evaluation experiment has been set up on the morpheus_23.11-runtime and morpheus_24.03-runtime images, using the `test-set-common` test set. +In `config-eval.yml`, the following should be configured: +- `file_path`: Specifies the test set to use +- `container_ids`: Can be used to filter the evaluation run to only run on specific containers listed in that test set. If left as an empty list, evaluation will be run on all containers listed in the test set. + +An example evaluation experiment has been set up to run the `test-set-common` test set on the morpheus_23.11-runtime and morpheus_24.03-runtime images. -In `config-eval.yml`, the `test_set_id` and `container_ids` should be configured. For example: ```yml -test_set_id: "test-setB" -container_ids: ["container1", "container2"] +file_path: data/eval_datasets/common-test-set.json +container_ids: ["morpheus-23.11", "morpheus-24.03"] ``` -Note that in order to run a particular test set on a set of images, that test set ID must be defined for all images (eg. here `test-setB` is valid, but `test-setA` would be invalid because `container2` does not define a ground truth for that test set). - You can also configure the evaluation output directory using the `output_dir` variable. - ### Running Multiple Evaluations Use the `--reps` flag to run evaluation multiple times on the same image and workflow. Note that **caching must be disabled first** (see [Disable Caching](#disable-caching) below). diff --git a/src/vuln_analysis/data/eval_datasets/profiler_dataset.json b/src/vuln_analysis/data/profiler_dataset.json similarity index 100% rename from src/vuln_analysis/data/eval_datasets/profiler_dataset.json rename to src/vuln_analysis/data/profiler_dataset.json diff --git a/src/vuln_analysis/eval/parse_eval_input.py b/src/vuln_analysis/eval/parse_eval_input.py index e2f976860..6b661ea69 100644 --- a/src/vuln_analysis/eval/parse_eval_input.py +++ b/src/vuln_analysis/eval/parse_eval_input.py @@ -9,7 +9,7 @@ def parse_input(file_path: Path, container_ids: list[str] | None = None) -> Eval """ To run just this script outside of the full nat pipeline: python3 parse_eval_input.py [test_set_file].json --container-ids [containers] --preview -o [your output file].json - Ex: python3 parse_eval_input.py ../data/eval_datasets/common-test-set.json --container-ids morpheus-23.11 morpheus-24.03 --preview -o my_eval.json + Ex: python3 parse_eval_input.py ../data/eval_datasets/common-test-set.json --container-ids morpheus-23.11 morpheus-24.03 --preview -o my_eval_string.json Note: Run this inside the vuln_analysis container, from the eval directory """ From 68fbde55d0af8fa8683fe3d73889a4f4387f51c9 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Fri, 19 Sep 2025 18:47:32 -0700 Subject: [PATCH 032/104] renamed response spread evaluator to response similarity, updated comments and readme with better explanations for the custom evaluators --- README.md | 6 +- src/vuln_analysis/configs/config-eval.yml | 6 +- .../eval/evaluators/consistency.py | 15 ++-- .../eval/evaluators/label_accuracy.py | 10 +-- ...ponse_spread.py => response_similarity.py} | 75 +++++++++---------- .../eval/evaluators/status_accuracy.py | 16 +--- src/vuln_analysis/register.py | 2 +- 7 files changed, 55 insertions(+), 75 deletions(-) rename src/vuln_analysis/eval/evaluators/{response_spread.py => response_similarity.py} (66%) diff --git a/README.md b/README.md index 75663faf8..8ac625763 100644 --- a/README.md +++ b/README.md @@ -834,14 +834,14 @@ nat eval --config_file=configs/config-eval.yml --reps=3 --override llms.checklis ``` #### Evaluation Metrics for Multiple Runs -Below are two example evaluation metrics for evaluating response spread and consistency across multiple evaluation runs. Uncomment the last section in `config-eval.yml` to enable these evaluators. You can view their source code in the `eval` directory. +Below are two example evaluation metrics for evaluating response similarity and consistency across multiple evaluation runs. Uncomment the last section in `config-eval.yml` to enable these evaluators. You can view their source code in the `eval` directory. You can write similar evaluators to calculate any desired custom metrics; read more at the NeMo Agent Toolkit documentation for [custom evaluators](https://docs.nvidia.com/nemo/agent-toolkit/1.2/reference/evaluate.html#adding-custom-evaluators). -- **Measure Response Spread:** Measures how similar the response summary field is across multiple runs for the same CVE. The summary field is in natural language; this evaluator converts each response into a bag-of-words frequency vector, then computes cosine similarity between each run and the centroid of all runs. Results are aggregated via a selected aggregation method (average, min, or max) configured in `config-eval.yml`. The final output is a spread score from 0 to 1 for each CVE, with 1 indicating no variation across runs and 0 indicating maximum variation. +- **Measure Response Similarity:** Measures how similar the summary field is across multiple runs on the same CVE. This evaluator converts the natural language summary field of each response into a bag-of-words frequency vector, then computes cosine similarity between each run and the centroid of all runs. Results are aggregated via a selected aggregation method (average, min, or max) configured in `config-eval.yml`. The final output is a similarity score from 0 to 1 for each CVE, with 1 indicating perfect similarity across runs and 0 indicating no similarity (maximum variation). -- **Measure Consistency:** Measures how consistent the model is in categorical outputs (status and label) across multiple runs for the same CVE. Computes a custom score to capture consistency, which we define as the sum of p², where p is the probability of each unique value. For example, if labels for one CVE across 3 runs are `{vulnerable, code_not_reachable, code_not_reachable}`, then the probabilities are `{1/3, 2/3}`, and the consistency score is `(1/3)² + (2/3)² = 0.56`. A score of 1.0 means perfect consistency (all runs yielded exact same values). The final output is a per-CVE consistency score for status and a per-CVE consistency score for labels. +- **Measure Consistency:** Measures how consistent the status or label field is, across multiple runs on the same CVE. Computes ∑p² per CVE, where p is the probability of each unique status or label. For example, if the labels for a CVE across 3 runs are `{vulnerable, code_not_reachable, code_not_reachable}`, then the probabilities become `{1/3, 2/3}`, so the consistency score is `(1/3)² + (2/3)² = 0.56`. A score of 1.0 means perfect consistency (all runs yielded exact same values). The final output is a per-CVE consistency score for status, and a separate per-CVE consistency score for labels. ## Troubleshooting diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml index c460f7e2d..bbd6ed62e 100644 --- a/src/vuln_analysis/configs/config-eval.yml +++ b/src/vuln_analysis/configs/config-eval.yml @@ -162,9 +162,9 @@ eval: _type: status_accuracy label_accuracy: _type: label_accuracy - # Uncomment to enable response spread and consistency evaluators - # response_spread: - # _type: response_spread + # Uncomment to enable response similarity and consistency evaluators + # response_similarity: + # _type: response_similarity # aggregation: "average" #Options: average, min, max # consistency: # _type: consistency diff --git a/src/vuln_analysis/eval/evaluators/consistency.py b/src/vuln_analysis/eval/evaluators/consistency.py index 719dac2d7..f9cbef364 100644 --- a/src/vuln_analysis/eval/evaluators/consistency.py +++ b/src/vuln_analysis/eval/evaluators/consistency.py @@ -31,7 +31,7 @@ class ConsistencyEvaluator: - '''Evaluator class for measuring consistency using custom metric''' + '''Evaluator class for measuring consistency of the label and status fields via the agreement probability metric ∑p², where where p is the probability of each unique value''' def __init__(self, max_concurrency: int): self.max_concurrency = max_concurrency @@ -41,7 +41,6 @@ async def evaluate(self, eval_input: EvalInput) -> EvalOutput: '''Evaluate consistency across multiple runs''' async def process_item(item): - """Calculate consistency for an individual item across multiple runs""" pipeline_result = json.loads(item.output_obj)['output'] # Group responses by vuln_id to get multiple responses per CVE @@ -59,7 +58,6 @@ async def process_item(item): vuln_responses[vuln_id]['labels'].append(label) vuln_responses[vuln_id]['statuses'].append(status) - # Calculate consistency for each CVE label_consistencies = {} status_consistencies = {} all_label_consistencies = [] @@ -69,13 +67,13 @@ async def process_item(item): # Calculate label consistency labels = responses['labels'] if len(labels) > 1: - # Consistency calculation: sum of squared probabilities + # Consistency metric is defined as the sum of squared probabilities label_counts = Counter(labels) n = len(labels) probs = [count / n for count in label_counts.values()] label_consistency = sum(p * p for p in probs) else: - label_consistency = 1.0 # Perfect consistency if only one response + label_consistency = 1.0 # Perfect consistency if there is only one response label_consistencies[vuln_id] = label_consistency all_label_consistencies.append(label_consistency) @@ -83,13 +81,12 @@ async def process_item(item): # Calculate status consistency statuses = responses['statuses'] if len(statuses) > 1: - # Consistency calculation: sum of squared probabilities status_counts = Counter(statuses) n = len(statuses) probs = [count / n for count in status_counts.values()] status_consistency = sum(p * p for p in probs) else: - status_consistency = 1.0 # Perfect consistency if only one response + status_consistency = 1.0 status_consistencies[vuln_id] = status_consistency all_status_consistencies.append(status_consistency) @@ -109,7 +106,7 @@ async def process_item(item): "test_set_file": item.full_dataset_entry.get('test_set_file', 'unknown'), } - # Return average of label and status consistency + # Return average of label consistencyand status consistency overall_consistency = (overall_label_consistency + overall_status_consistency) / 2 return overall_consistency, reasoning @@ -139,7 +136,7 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) -# Evaluator registration +# Register custom evaluator in NAT class ConsistencyEvaluatorConfig(EvaluatorBaseConfig, name="consistency"): """Configuration for consistency evaluator""" pass diff --git a/src/vuln_analysis/eval/evaluators/label_accuracy.py b/src/vuln_analysis/eval/evaluators/label_accuracy.py index f27917a40..0f53602ac 100644 --- a/src/vuln_analysis/eval/evaluators/label_accuracy.py +++ b/src/vuln_analysis/eval/evaluators/label_accuracy.py @@ -29,17 +29,15 @@ class LabelAccuracyEvaluator: - '''Evaluator class for justification labels accuracy''' + '''Evaluator class for accuracy of the label field''' def __init__(self, max_concurrency: int): self.max_concurrency = max_concurrency self.semaphore = asyncio.Semaphore(self.max_concurrency) async def evaluate(self, eval_input: EvalInput) -> EvalOutput: - '''Evaluate label accuracy''' async def process_item(item): - """Compute simple accuracy for an individual item""" ground_truth_map = eval(item.expected_output_obj) pipeline_result = json.loads(item.output_obj)['output'] @@ -55,9 +53,7 @@ async def process_item(item): justification_label = result.get('justification', {}).get('label', 'MISSING') generated_answers.append(justification_label) - # Calculate label accuracy - # Ground truth labels are stored as "{vuln_id}_label" in the ground truth map - # This is populated by parse_eval_input.py from the eval dataset JSON files + # Calculate label accuracy. Ground truth labels are stored as "{vuln_id}_label" in the EvalInputItem, as populated by parse_eval_input.py ground_truth_label = ground_truth_map.get(f"{vuln_id}_label") if ground_truth_label is not None: label_accuracies.append(float(justification_label == ground_truth_label)) @@ -99,7 +95,7 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) -# Evaluator registration +# Register custom evaluator in NAT class LabelAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="label_accuracy"): """Configuration for label accuracy evaluator""" pass diff --git a/src/vuln_analysis/eval/evaluators/response_spread.py b/src/vuln_analysis/eval/evaluators/response_similarity.py similarity index 66% rename from src/vuln_analysis/eval/evaluators/response_spread.py rename to src/vuln_analysis/eval/evaluators/response_similarity.py index af7b9a637..67f33575a 100644 --- a/src/vuln_analysis/eval/evaluators/response_spread.py +++ b/src/vuln_analysis/eval/evaluators/response_similarity.py @@ -31,25 +31,24 @@ from tqdm import tqdm -class ResponseSpreadEvaluator: - '''Evaluator class for measuring response spread using cosine similarity''' +class ResponseSimilarityEvaluator: + '''Evaluator class for measuring similarity of the summary field across multiple runs on the same CVE, by calculating cosine similarity from the centroid, on Bag-of-Words vectors''' def __init__(self, max_concurrency: int, aggregation: str = "average", stopwords: str = None): self.max_concurrency = max_concurrency self.semaphore = asyncio.Semaphore(self.max_concurrency) - self.aggregation = aggregation # "average", "min", or "max" + self.aggregation = aggregation # Accepts key words: "average", "min", "max" self.stopwords = stopwords async def evaluate(self, eval_input: EvalInput) -> EvalOutput: - '''Evaluate response spread across multiple runs''' async def process_item(item): - """Calculate response spread for an individual item across multiple runs""" pipeline_result = json.loads(item.output_obj)['output'] - # Group responses by vuln_id to get multiple responses per CVE + # Group responses by vuln_id to group multiple responses for a given CVE vuln_responses = {} + # Get the summary field from each response for result in pipeline_result: vuln_id = result['vuln_id'] summary = result.get('summary', '') @@ -58,24 +57,22 @@ async def process_item(item): vuln_responses[vuln_id] = [] vuln_responses[vuln_id].append(summary) - # Calculate response spread for each CVE - response_spreads = {} - all_spreads = [] + response_similarities = {} + all_similarities = [] for vuln_id, responses in vuln_responses.items(): - if len(responses) < 2: - # Need at least 2 responses to calculate spread - response_spreads[vuln_id] = 1.0 # Perfect similarity if only one response - all_spreads.append(1.0) + if len(responses) < 2: # Perfect similarity if there is only one or less responses + response_similarities[vuln_id] = 1.0 + all_similarities.append(1.0) continue - # Calculate cosine similarity spread using bag-of-words + # Apply bag-of-words on the summary field of each response vectorizer = CountVectorizer(stop_words=self.stopwords) X = vectorizer.fit_transform(responses).toarray() if X.shape[0] < 2: - response_spreads[vuln_id] = 1.0 - all_spreads.append(1.0) + response_similarities[vuln_id] = 1.0 + all_similarities.append(1.0) continue # Calculate centroid (average of all response vectors) @@ -84,36 +81,34 @@ async def process_item(item): # Calculate cosine similarities between each response and centroid dot_products = np.dot(X, centroid) norms = np.linalg.norm(X, axis=1) * np.linalg.norm(centroid) - - # Avoid division by zero - norms = np.where(norms == 0, 1, norms) + norms = np.where(norms == 0, 1, norms) # Avoid division by zero cos_similarities = dot_products / norms - # Apply aggregation + # Apply aggregation on the cosine similarities if self.aggregation == "average": - spread_score = float(np.average(cos_similarities)) + similarity_score = float(np.average(cos_similarities)) elif self.aggregation == "max": - spread_score = float(np.max(cos_similarities)) + similarity_score = float(np.max(cos_similarities)) elif self.aggregation == "min": - spread_score = float(np.min(cos_similarities)) + similarity_score = float(np.min(cos_similarities)) else: - spread_score = float(np.average(cos_similarities)) + similarity_score = float(np.average(cos_similarities)) - response_spreads[vuln_id] = spread_score - all_spreads.append(spread_score) + response_similarities[vuln_id] = similarity_score + all_similarities.append(similarity_score) - # Calculate overall average spread - overall_spread = sum(all_spreads) / len(all_spreads) if all_spreads else 1.0 + # Calculate average similarity across all CVEs + overall_similarity = sum(all_similarities) / len(all_similarities) if all_similarities else 1.0 reasoning = { - "response_spreads": response_spreads, - "overall_spread": overall_spread, + "response_similarities": response_similarities, + "overall_similarity": overall_similarity, "aggregation_method": self.aggregation, "num_cves": len(vuln_responses), "total_responses": sum(len(responses) for responses in vuln_responses.values()), "test_set_file": item.full_dataset_entry.get('test_set_file', 'unknown'), } - return overall_spread, reasoning + return overall_similarity, reasoning async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: async with self.semaphore: @@ -124,7 +119,7 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: try: tqdm_position = TqdmPositionRegistry.claim() pbar = tqdm(total=len(eval_input.eval_input_items), - desc="Evaluating Response Spread", + desc="Evaluating Response Similarity", position=tqdm_position) results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) finally: @@ -143,17 +138,17 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) -# Evaluator registration -class ResponseSpreadEvaluatorConfig(EvaluatorBaseConfig, name="response_spread"): - """Configuration for response spread evaluator""" +# Register custom evaluator in NAT +class ResponseSimilarityEvaluatorConfig(EvaluatorBaseConfig, name="response_similarity"): + """Configuration for response similarity evaluator""" aggregation: str = "average" # "average", "min", or "max" stopwords: str | None = None -@register_evaluator(config_type=ResponseSpreadEvaluatorConfig) -async def response_spread_evaluator(config: ResponseSpreadEvaluatorConfig, builder: EvalBuilder): - """Register response spread evaluator""" - evaluator = ResponseSpreadEvaluator(max_concurrency=builder.get_max_concurrency(), +@register_evaluator(config_type=ResponseSimilarityEvaluatorConfig) +async def response_similarity_evaluator(config: ResponseSimilarityEvaluatorConfig, builder: EvalBuilder): + """Register response similarity evaluator""" + evaluator = ResponseSimilarityEvaluator(max_concurrency=builder.get_max_concurrency(), aggregation=config.aggregation, stopwords=config.stopwords) - yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Response Spread Evaluator") + yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Response Similarity Evaluator") diff --git a/src/vuln_analysis/eval/evaluators/status_accuracy.py b/src/vuln_analysis/eval/evaluators/status_accuracy.py index 4053c6519..a8d9455ec 100644 --- a/src/vuln_analysis/eval/evaluators/status_accuracy.py +++ b/src/vuln_analysis/eval/evaluators/status_accuracy.py @@ -29,19 +29,16 @@ _LABEL_MAP = {'TRUE': 'AFFECTED', 'FALSE': 'NOT AFFECTED', 'UNKNOWN': 'UNKNOWN'} - class StatusAccuracyEvaluator: - '''Status accuracy evaluator class that calculates status accuracy (AFFECTED/NOT AFFECTED)''' + '''Evaluator class that calculates accuracy of the status field (AFFECTED/NOT AFFECTED)''' def __init__(self, max_concurrency: int): self.max_concurrency = max_concurrency self.semaphore = asyncio.Semaphore(self.max_concurrency) async def evaluate(self, eval_input: EvalInput) -> EvalOutput: - '''Evaluate accuracy funciont''' async def process_item(item): - """Compute simple accuracy for an individual item""" ground_truth_map = eval(item.expected_output_obj) pipeline_result = json.loads(item.output_obj)['output'] @@ -59,12 +56,8 @@ async def process_item(item): status_result = _LABEL_MAP.get(affected_status, 'MISSING') generated_status_answers.append(status_result) - # ORIGINAL LOGIC - ground truth MISSING factors into accuracy calculation - # accuracies.append(float(status_result == ground_truth_map.get(vuln_id, 'MISSING'))) - - # NEW LOGIC - Only include in accuracy calculation if ground truth exists (not MISSING) ground_truth_status = ground_truth_map.get(vuln_id) - if ground_truth_status is not None: + if ground_truth_status is not None: # If ground truth status is missing, do not factor this entry into the final accuracy score status_accuracies.append(float(status_result == ground_truth_status)) # Calculate average accuracy @@ -86,7 +79,7 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: try: tqdm_position = TqdmPositionRegistry.claim() - pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Accuracy", position=tqdm_position) + pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Status Accuracy", position=tqdm_position) results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) finally: pbar.close() @@ -104,12 +97,11 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) -# Evaluator registration +# Register custom evaluator in NAT class StatusAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="status_accuracy"): """Configuration for status accuracy evaluator""" pass - @register_evaluator(config_type=StatusAccuracyEvaluatorConfig) async def status_accuracy_evaluator(config: StatusAccuracyEvaluatorConfig, builder: EvalBuilder): """Register status accuracy evaluator""" diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index d7d161d19..85ecfc44b 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -49,7 +49,7 @@ # Import evaluator registrations from vuln_analysis.eval.evaluators import status_accuracy from vuln_analysis.eval.evaluators import label_accuracy -from vuln_analysis.eval.evaluators import response_spread +from vuln_analysis.eval.evaluators import response_similarity from vuln_analysis.eval.evaluators import consistency logger = logging.getLogger(__name__) From e91b6103e592039751b27d4fcdd37325ed344271 Mon Sep 17 00:00:00 2001 From: Katherine H Date: Fri, 10 Oct 2025 06:51:41 +0000 Subject: [PATCH 033/104] removed response similarity custom evaluator --- .../eval/evaluators/response_similarity.py | 154 ------------------ 1 file changed, 154 deletions(-) delete mode 100644 src/vuln_analysis/eval/evaluators/response_similarity.py diff --git a/src/vuln_analysis/eval/evaluators/response_similarity.py b/src/vuln_analysis/eval/evaluators/response_similarity.py deleted file mode 100644 index 67f33575a..000000000 --- a/src/vuln_analysis/eval/evaluators/response_similarity.py +++ /dev/null @@ -1,154 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -import json -from collections import Counter - -import numpy as np -from nat.builder.builder import EvalBuilder -from nat.builder.evaluator import EvaluatorInfo -from nat.cli.register_workflow import register_evaluator -from nat.data_models.evaluator import EvaluatorBaseConfig -from nat.eval.evaluator.evaluator_model import EvalInput -from nat.eval.evaluator.evaluator_model import EvalInputItem -from nat.eval.evaluator.evaluator_model import EvalOutput -from nat.eval.evaluator.evaluator_model import EvalOutputItem -from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry -from sklearn.feature_extraction.text import CountVectorizer -from tqdm import tqdm - - -class ResponseSimilarityEvaluator: - '''Evaluator class for measuring similarity of the summary field across multiple runs on the same CVE, by calculating cosine similarity from the centroid, on Bag-of-Words vectors''' - - def __init__(self, max_concurrency: int, aggregation: str = "average", stopwords: str = None): - self.max_concurrency = max_concurrency - self.semaphore = asyncio.Semaphore(self.max_concurrency) - self.aggregation = aggregation # Accepts key words: "average", "min", "max" - self.stopwords = stopwords - - async def evaluate(self, eval_input: EvalInput) -> EvalOutput: - - async def process_item(item): - pipeline_result = json.loads(item.output_obj)['output'] - - # Group responses by vuln_id to group multiple responses for a given CVE - vuln_responses = {} - - # Get the summary field from each response - for result in pipeline_result: - vuln_id = result['vuln_id'] - summary = result.get('summary', '') - - if vuln_id not in vuln_responses: - vuln_responses[vuln_id] = [] - vuln_responses[vuln_id].append(summary) - - response_similarities = {} - all_similarities = [] - - for vuln_id, responses in vuln_responses.items(): - if len(responses) < 2: # Perfect similarity if there is only one or less responses - response_similarities[vuln_id] = 1.0 - all_similarities.append(1.0) - continue - - # Apply bag-of-words on the summary field of each response - vectorizer = CountVectorizer(stop_words=self.stopwords) - X = vectorizer.fit_transform(responses).toarray() - - if X.shape[0] < 2: - response_similarities[vuln_id] = 1.0 - all_similarities.append(1.0) - continue - - # Calculate centroid (average of all response vectors) - centroid = np.sum(X, axis=0) - - # Calculate cosine similarities between each response and centroid - dot_products = np.dot(X, centroid) - norms = np.linalg.norm(X, axis=1) * np.linalg.norm(centroid) - norms = np.where(norms == 0, 1, norms) # Avoid division by zero - cos_similarities = dot_products / norms - - # Apply aggregation on the cosine similarities - if self.aggregation == "average": - similarity_score = float(np.average(cos_similarities)) - elif self.aggregation == "max": - similarity_score = float(np.max(cos_similarities)) - elif self.aggregation == "min": - similarity_score = float(np.min(cos_similarities)) - else: - similarity_score = float(np.average(cos_similarities)) - - response_similarities[vuln_id] = similarity_score - all_similarities.append(similarity_score) - - # Calculate average similarity across all CVEs - overall_similarity = sum(all_similarities) / len(all_similarities) if all_similarities else 1.0 - reasoning = { - "response_similarities": response_similarities, - "overall_similarity": overall_similarity, - "aggregation_method": self.aggregation, - "num_cves": len(vuln_responses), - "total_responses": sum(len(responses) for responses in vuln_responses.values()), - "test_set_file": item.full_dataset_entry.get('test_set_file', 'unknown'), - } - - return overall_similarity, reasoning - - async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: - async with self.semaphore: - result = await process_item(item) - pbar.update(1) - return result - - try: - tqdm_position = TqdmPositionRegistry.claim() - pbar = tqdm(total=len(eval_input.eval_input_items), - desc="Evaluating Response Similarity", - position=tqdm_position) - results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) - finally: - pbar.close() - TqdmPositionRegistry.release(tqdm_position) - - sample_scores, sample_reasonings = zip(*results) if results else ([], []) - - avg_score = round(sum(sample_scores) / len(sample_scores), 4) if sample_scores else 1.0 - - eval_output_items = [ - EvalOutputItem(id=item.id, score=score, reasoning=reasoning) - for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings) - ] - - return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) - - -# Register custom evaluator in NAT -class ResponseSimilarityEvaluatorConfig(EvaluatorBaseConfig, name="response_similarity"): - """Configuration for response similarity evaluator""" - aggregation: str = "average" # "average", "min", or "max" - stopwords: str | None = None - - -@register_evaluator(config_type=ResponseSimilarityEvaluatorConfig) -async def response_similarity_evaluator(config: ResponseSimilarityEvaluatorConfig, builder: EvalBuilder): - """Register response similarity evaluator""" - evaluator = ResponseSimilarityEvaluator(max_concurrency=builder.get_max_concurrency(), - aggregation=config.aggregation, - stopwords=config.stopwords) - yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Response Similarity Evaluator") From 5d8a9b34b495983735ea6beed517a4b8641b9f91 Mon Sep 17 00:00:00 2001 From: Katherine H Date: Fri, 10 Oct 2025 06:58:22 +0000 Subject: [PATCH 034/104] fixed evaluator average accuracy calculation bug. refactored consistency evaluator by field and fixed calculation bug. --- .../eval/evaluators/consistency.py | 144 +++++++----------- .../eval/evaluators/label_accuracy.py | 17 ++- .../eval/evaluators/status_accuracy.py | 25 ++- src/vuln_analysis/eval/parse_eval_input.py | 21 +-- 4 files changed, 94 insertions(+), 113 deletions(-) diff --git a/src/vuln_analysis/eval/evaluators/consistency.py b/src/vuln_analysis/eval/evaluators/consistency.py index f9cbef364..dcf26305c 100644 --- a/src/vuln_analysis/eval/evaluators/consistency.py +++ b/src/vuln_analysis/eval/evaluators/consistency.py @@ -17,13 +17,11 @@ import json from collections import Counter -import numpy as np from nat.builder.builder import EvalBuilder from nat.builder.evaluator import EvaluatorInfo from nat.cli.register_workflow import register_evaluator from nat.data_models.evaluator import EvaluatorBaseConfig from nat.eval.evaluator.evaluator_model import EvalInput -from nat.eval.evaluator.evaluator_model import EvalInputItem from nat.eval.evaluator.evaluator_model import EvalOutput from nat.eval.evaluator.evaluator_model import EvalOutputItem from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry @@ -31,106 +29,78 @@ class ConsistencyEvaluator: - '''Evaluator class for measuring consistency of the label and status fields via the agreement probability metric ∑p², where where p is the probability of each unique value''' + '''Evaluator class for measuring consistency of the specified field via the agreement probability metric ∑p², where p is the probability of each unique value''' - def __init__(self, max_concurrency: int): + def __init__(self, max_concurrency: int, field: str = "label"): self.max_concurrency = max_concurrency self.semaphore = asyncio.Semaphore(self.max_concurrency) + self.field = field # "label" or "status" async def evaluate(self, eval_input: EvalInput) -> EvalOutput: '''Evaluate consistency across multiple runs''' - async def process_item(item): - pipeline_result = json.loads(item.output_obj)['output'] - - # Group responses by vuln_id to get multiple responses per CVE - vuln_responses = {} - - for result in pipeline_result: - vuln_id = result['vuln_id'] - # Measure consistency on label and status fields - label = result.get('justification', {}).get('label', 'MISSING') - status = result.get('justification', {}).get('status', 'MISSING') - - if vuln_id not in vuln_responses: - vuln_responses[vuln_id] = {'labels': [], 'statuses': []} - - vuln_responses[vuln_id]['labels'].append(label) - vuln_responses[vuln_id]['statuses'].append(status) - - label_consistencies = {} - status_consistencies = {} - all_label_consistencies = [] - all_status_consistencies = [] - - for vuln_id, responses in vuln_responses.items(): - # Calculate label consistency - labels = responses['labels'] - if len(labels) > 1: - # Consistency metric is defined as the sum of squared probabilities - label_counts = Counter(labels) - n = len(labels) - probs = [count / n for count in label_counts.values()] - label_consistency = sum(p * p for p in probs) - else: - label_consistency = 1.0 # Perfect consistency if there is only one response - - label_consistencies[vuln_id] = label_consistency - all_label_consistencies.append(label_consistency) - - # Calculate status consistency - statuses = responses['statuses'] - if len(statuses) > 1: - status_counts = Counter(statuses) - n = len(statuses) - probs = [count / n for count in status_counts.values()] - status_consistency = sum(p * p for p in probs) - else: - status_consistency = 1.0 - - status_consistencies[vuln_id] = status_consistency - all_status_consistencies.append(status_consistency) - - # Calculate overall average consistency - overall_label_consistency = sum(all_label_consistencies) / len( - all_label_consistencies) if all_label_consistencies else 1.0 - overall_status_consistency = sum(all_status_consistencies) / len( - all_status_consistencies) if all_status_consistencies else 1.0 - reasoning = { - "label_consistencies": label_consistencies, - "status_consistencies": status_consistencies, - "overall_label_consistency": overall_label_consistency, - "overall_status_consistency": overall_status_consistency, - "num_cves": len(vuln_responses), - "total_responses": sum(len(responses['labels']) for responses in vuln_responses.values()), - "test_set_file": item.full_dataset_entry.get('test_set_file', 'unknown'), - } - - # Return average of label consistencyand status consistency - overall_consistency = (overall_label_consistency + overall_status_consistency) / 2 - return overall_consistency, reasoning - - async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: + async def extract_run_data(item): + """Extract data from a single evaluation run""" async with self.semaphore: - result = await process_item(item) + evaluation_run = json.loads(item.output_obj)['output'] pbar.update(1) - return result + return evaluation_run try: tqdm_position = TqdmPositionRegistry.claim() pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Consistency", position=tqdm_position) - results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) + + # Extract data from all evaluation runs in parallel + all_runs = await asyncio.gather(*[extract_run_data(item) for item in eval_input.eval_input_items]) finally: pbar.close() TqdmPositionRegistry.release(tqdm_position) - sample_scores, sample_reasonings = zip(*results) if results else ([], []) - - avg_score = round(sum(sample_scores) / len(sample_scores), 4) if sample_scores else 1.0 + # Now aggregate ALL results across ALL runs to calculate consistency per CVE + vuln_responses = {} + for evaluation_run in all_runs: + for cve_result in evaluation_run: + vuln_id = cve_result['vuln_id'] + # Measure consistency on the specified field + field_value = cve_result.get('justification', {}).get(self.field, 'MISSING') + if vuln_id not in vuln_responses: + vuln_responses[vuln_id] = {'values': []} + + vuln_responses[vuln_id]['values'].append(field_value) + + # Calculate consistency score per CVE + consistencies_dict = {} # Stores consistency scores mapped to CVE + consistencies = [] # Stores all consistency scores in an array + + for vuln_id, responses in vuln_responses.items(): + # Calculate consistency for the specified field + values = responses['values'] + if len(values) > 1: + # Consistency metric is defined as the sum of squared probabilities + value_counts = Counter(values) + n = len(values) + probs = [count / n for count in value_counts.values()] + consistency = sum(p * p for p in probs) + else: + consistency = 1.0 # Perfect consistency if there is only one response + + consistencies_dict[vuln_id] = round(consistency, 2) + consistencies.append(round(consistency, 2)) + + # Calculate average consistency + avg_consistency = sum(consistencies) / len(consistencies) if consistencies else 1.0 + avg_score = round(avg_consistency, 2) + + # Create output with per-CVE scores + output = { + f"{self.field}_consistencies": consistencies_dict, + "total_cves": len(vuln_responses), + } + + # Create a single dummy EvalOutputItem to store reasoning eval_output_items = [ - EvalOutputItem(id=item.id, score=score, reasoning=reasoning) - for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings) + EvalOutputItem(id=f"{self.field.title()} Consistency per CVE", score=avg_score, reasoning=output) ] return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) @@ -139,11 +109,13 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: # Register custom evaluator in NAT class ConsistencyEvaluatorConfig(EvaluatorBaseConfig, name="consistency"): """Configuration for consistency evaluator""" - pass + field: str = "label" # "label" or "status" @register_evaluator(config_type=ConsistencyEvaluatorConfig) async def consistency_evaluator(config: ConsistencyEvaluatorConfig, builder: EvalBuilder): """Register consistency evaluator""" - evaluator = ConsistencyEvaluator(builder.get_max_concurrency()) - yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Consistency Evaluator") + evaluator = ConsistencyEvaluator(builder.get_max_concurrency(), field=config.field) + yield EvaluatorInfo(config=config, + evaluate_fn=evaluator.evaluate, + description=f"{config.field.title()} Consistency Evaluator") diff --git a/src/vuln_analysis/eval/evaluators/label_accuracy.py b/src/vuln_analysis/eval/evaluators/label_accuracy.py index 0f53602ac..1aad059b3 100644 --- a/src/vuln_analysis/eval/evaluators/label_accuracy.py +++ b/src/vuln_analysis/eval/evaluators/label_accuracy.py @@ -58,6 +58,11 @@ async def process_item(item): if ground_truth_label is not None: label_accuracies.append(float(justification_label == ground_truth_label)) + total_correct = sum(label_accuracies) + total_items = len(label_accuracies) + + # Calculate average accuracy + label_avg = total_correct / total_items if total_items > 0 else 0.0 reasoning = { "question": question, "answer": [ground_truth_map.get(f"{q}_label", 'MISSING') for q in question], @@ -65,9 +70,9 @@ async def process_item(item): "test_set_file": item.full_dataset_entry.get('test_set_file', 'unknown'), } - return sum(label_accuracies) / len(label_accuracies) if label_accuracies else 0.0, reasoning + return label_avg, reasoning, total_items, total_correct - async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: + async def wrapped_process(item: EvalInputItem) -> tuple[float, dict, int, int]: async with self.semaphore: result = await process_item(item) pbar.update(1) @@ -83,9 +88,13 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: pbar.close() TqdmPositionRegistry.release(tqdm_position) - sample_scores, sample_reasonings = zip(*results) if results else ([], []) + sample_scores, sample_reasonings, total_items, total_correct = zip(*results) if results else ([], [], [], []) + + # Calculate weighted average (each EvalInputItem is weighted by the number of CVEs in it) + total_correct_sum = sum(total_correct) + total_items_sum = sum(total_items) - avg_score = round(sum(sample_scores) / len(sample_scores), 2) if sample_scores else 0.0 + avg_score = round(total_correct_sum / total_items_sum, 2) if total_items_sum > 0 else 0.0 eval_output_items = [ EvalOutputItem(id=item.id, score=score, reasoning=reasoning) diff --git a/src/vuln_analysis/eval/evaluators/status_accuracy.py b/src/vuln_analysis/eval/evaluators/status_accuracy.py index a8d9455ec..0cfa3c429 100644 --- a/src/vuln_analysis/eval/evaluators/status_accuracy.py +++ b/src/vuln_analysis/eval/evaluators/status_accuracy.py @@ -29,6 +29,7 @@ _LABEL_MAP = {'TRUE': 'AFFECTED', 'FALSE': 'NOT AFFECTED', 'UNKNOWN': 'UNKNOWN'} + class StatusAccuracyEvaluator: '''Evaluator class that calculates accuracy of the status field (AFFECTED/NOT AFFECTED)''' @@ -57,11 +58,14 @@ async def process_item(item): generated_status_answers.append(status_result) ground_truth_status = ground_truth_map.get(vuln_id) - if ground_truth_status is not None: # If ground truth status is missing, do not factor this entry into the final accuracy score + if ground_truth_status is not None: # If ground truth status is missing, do not factor this entry into the final accuracy score status_accuracies.append(float(status_result == ground_truth_status)) + total_correct = sum(status_accuracies) + total_items = len(status_accuracies) + # Calculate average accuracy - status_avg = sum(status_accuracies) / len(status_accuracies) if status_accuracies else 0.0 + status_avg = total_correct / total_items if total_items > 0 else 0.0 reasoning = { "question": question, "status_answer": [ground_truth_map.get(q, 'MISSING') for q in question], @@ -69,9 +73,9 @@ async def process_item(item): "test_set_file": item.full_dataset_entry.get('test_set_file', 'unknown'), } - return status_avg, reasoning + return status_avg, reasoning, total_items, total_correct - async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: + async def wrapped_process(item: EvalInputItem) -> tuple[float, dict, int, int]: async with self.semaphore: result = await process_item(item) pbar.update(1) @@ -79,15 +83,21 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]: try: tqdm_position = TqdmPositionRegistry.claim() - pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Status Accuracy", position=tqdm_position) + pbar = tqdm(total=len(eval_input.eval_input_items), + desc="Evaluating Status Accuracy", + position=tqdm_position) results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) finally: pbar.close() TqdmPositionRegistry.release(tqdm_position) - sample_scores, sample_reasonings = zip(*results) if results else ([], []) + sample_scores, sample_reasonings, total_items, total_correct = zip(*results) if results else ([], [], [], []) - avg_score = round(sum(sample_scores) / len(sample_scores), 2) if sample_scores else 0.0 + # Calculate weighted average (each EvalInputItem is weighted by the number of CVEs in it) + total_correct_sum = sum(total_correct) + total_items_sum = sum(total_items) + + avg_score = round(total_correct_sum / total_items_sum, 2) if total_items_sum > 0 else 0.0 eval_output_items = [ EvalOutputItem(id=item.id, score=score, reasoning=reasoning) @@ -102,6 +112,7 @@ class StatusAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="status_accuracy") """Configuration for status accuracy evaluator""" pass + @register_evaluator(config_type=StatusAccuracyEvaluatorConfig) async def status_accuracy_evaluator(config: StatusAccuracyEvaluatorConfig, builder: EvalBuilder): """Register status accuracy evaluator""" diff --git a/src/vuln_analysis/eval/parse_eval_input.py b/src/vuln_analysis/eval/parse_eval_input.py index 6b661ea69..93dc0b862 100644 --- a/src/vuln_analysis/eval/parse_eval_input.py +++ b/src/vuln_analysis/eval/parse_eval_input.py @@ -5,11 +5,11 @@ from nat.eval.evaluator.evaluator_model import EvalInputItem -def parse_input(file_path: Path, container_ids: list[str] | None = None) -> EvalInput: +def parse_input(file_path: Path) -> EvalInput: """ To run just this script outside of the full nat pipeline: - python3 parse_eval_input.py [test_set_file].json --container-ids [containers] --preview -o [your output file].json - Ex: python3 parse_eval_input.py ../data/eval_datasets/common-test-set.json --container-ids morpheus-23.11 morpheus-24.03 --preview -o my_eval_string.json + python3 parse_eval_input.py [test_set_file].json --preview -o [your output file].json + Ex: python3 parse_eval_input.py ../data/eval_datasets/common-test-set.json --preview -o my_eval_string.json Note: Run this inside the vuln_analysis container, from the eval directory """ @@ -20,14 +20,6 @@ def parse_input(file_path: Path, container_ids: list[str] | None = None) -> Eval # Extract containers containers = config_data.get('containers', {}) - # Filter containers if specific container IDs provided - if container_ids is not None: - available_containers = {k: v for k, v in containers.items() if k in container_ids} - print(f"Using specified containers: {list(available_containers.keys())}") - else: - available_containers = containers - print(f"No container IDs specified, using all available containers: {list(containers.keys())}") - # Get test set description from the file test_set_description = config_data.get('test_set_description', 'Unknown test set') test_set_file = file_path.name # Extract filename from the file path @@ -36,7 +28,7 @@ def parse_input(file_path: Path, container_ids: list[str] | None = None) -> Eval # Create evaluation items for each container eval_items = [] - for container_id, container_data in available_containers.items(): + for container_id, container_data in containers.items(): container_image = container_data.get('container_image', {}) ground_truth = container_data.get('ground_truth', []) @@ -114,9 +106,6 @@ def main(): ) parser.add_argument('input_file', help='Path to eval-dataset.json file') parser.add_argument('-o', '--output', help='Output path for baseline string format (optional)') - parser.add_argument('--container-ids', - nargs='+', - help='Specific container IDs to process (e.g., --container-ids morpheus-23.11 morpheus-24.03)') parser.add_argument('--preview', action='store_true', help='Show preview of first converted item') args = parser.parse_args() @@ -125,7 +114,7 @@ def main(): input_path = Path(args.input_file) # Convert to EvalInput format - eval_input = parse_input(input_path, args.container_ids) + eval_input = parse_input(input_path) # Save as baseline string format if output is specified if args.output: From 625dae6c05d5d9639cd7e4184616af33d1ff5ab7 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Fri, 10 Oct 2025 01:07:39 -0700 Subject: [PATCH 035/104] refactored accuracy evaluator to accept field variable, updated readme --- README.md | 24 ++-- .../{label_accuracy.py => accuracy.py} | 82 +++++++----- .../eval/evaluators/status_accuracy.py | 120 ------------------ src/vuln_analysis/register.py | 10 +- 4 files changed, 64 insertions(+), 172 deletions(-) rename src/vuln_analysis/eval/evaluators/{label_accuracy.py => accuracy.py} (53%) delete mode 100644 src/vuln_analysis/eval/evaluators/status_accuracy.py diff --git a/README.md b/README.md index 8ac625763..b5558da9a 100644 --- a/README.md +++ b/README.md @@ -773,8 +773,12 @@ nat eval --config_file=configs/config-eval.yml This will write the files `status_accuracy_output.json` and `label_accuracy_output.json` to the configured output directory (`.tmp/evaluators` in this example), containing accuracy scores for status and label respectively. Scores range from 0 to 1, with 0 indicating no CVEs were categorized correctly, and 1 indicating all CVEs were categorized correctly. +Under the hood, the source code of the evaluator being run can be found at `eval/evaluators/accuracy.py`; this custom evaluator is registered in the NeMo Agent Toolkit as an evaluator of type `_accuracy`. In `config-eval.yml`, under the `evaluators` section, we have defined two evaluations to be run: `status_accuracy` and `label_accuracy`. However, the underlying code for these evaluations is the same. In other words, they use the same evaluator, which accepts either `status` or `label` as a configurable field input. + +You can write similar evaluators to calculate any desired custom metrics; read more at the NeMo Agent Toolkit documentation for [custom evaluators](https://docs.nvidia.com/nemo/agent-toolkit/1.2/reference/evaluate.html#adding-custom-evaluators). + ### Inputting Evaluation Test Sets -Ground truth datasets should be put in the `data/eval_datasets/` directory. Each eval dataset should be a separate json, with the following structure: +Ground truth datasets should be put in the `data/eval_datasets/` directory. Each eval test set should be a separate json, with the following structure: ``` test-set-description @@ -788,15 +792,12 @@ containers: ... ``` -In `config-eval.yml`, the following should be configured: -- `file_path`: Specifies the test set to use -- `container_ids`: Can be used to filter the evaluation run to only run on specific containers listed in that test set. If left as an empty list, evaluation will be run on all containers listed in the test set. +In `config-eval.yml`, the `file_path` should be configured to specify the specific test set to use. -An example evaluation experiment has been set up to run the `test-set-common` test set on the morpheus_23.11-runtime and morpheus_24.03-runtime images. +An example evaluation experiment has been set up to run the `test-set-common` test set on the morpheus_23.11-runtime and morpheus_24.03-runtime images. Note that in order to run a test set across multiple containers, each container must contain exactly the same test set of CVEs present in every other container (ie. a container cannot contain only a subset of the full test set). ```yml file_path: data/eval_datasets/common-test-set.json -container_ids: ["morpheus-23.11", "morpheus-24.03"] ``` You can also configure the evaluation output directory using the `output_dir` variable. @@ -833,15 +834,12 @@ For quick testing, you can also change configuration at runtime using the `--ove nat eval --config_file=configs/config-eval.yml --reps=3 --override llms.checklist_llm.base_url https://integrate.api.nvidia.com/v1 ``` -#### Evaluation Metrics for Multiple Runs -Below are two example evaluation metrics for evaluating response similarity and consistency across multiple evaluation runs. Uncomment the last section in `config-eval.yml` to enable these evaluators. You can view their source code in the `eval` directory. - -You can write similar evaluators to calculate any desired custom metrics; read more at the NeMo Agent Toolkit documentation for [custom evaluators](https://docs.nvidia.com/nemo/agent-toolkit/1.2/reference/evaluate.html#adding-custom-evaluators). - +#### Evaluation Across Multiple Runs +Below is an example evaluation metric suitable for evaluating across multiple runs. Specifically, it measures consistency of results across repeated runs of the same input source code and CVE test set. It can be configured to measure consistency of the `status` or `label` field by configuring the `evaluators:consistency:field` field in the `config-eval.yml` file. -- **Measure Response Similarity:** Measures how similar the summary field is across multiple runs on the same CVE. This evaluator converts the natural language summary field of each response into a bag-of-words frequency vector, then computes cosine similarity between each run and the centroid of all runs. Results are aggregated via a selected aggregation method (average, min, or max) configured in `config-eval.yml`. The final output is a similarity score from 0 to 1 for each CVE, with 1 indicating perfect similarity across runs and 0 indicating no similarity (maximum variation). +Consistency is defined as: ∑p² per CVE, where p is the probability of each unique status or label. For example, if the labels for a CVE across 3 runs are `{vulnerable, code_not_reachable, code_not_reachable}`, then the probabilities become `{1/3, 2/3}`, so the consistency score is `(1/3)² + (2/3)² = 0.56`. A score of 1.0 means perfect consistency (all runs yielded exact same values). The final output is a per-CVE consistency score and an average consistency score. -- **Measure Consistency:** Measures how consistent the status or label field is, across multiple runs on the same CVE. Computes ∑p² per CVE, where p is the probability of each unique status or label. For example, if the labels for a CVE across 3 runs are `{vulnerable, code_not_reachable, code_not_reachable}`, then the probabilities become `{1/3, 2/3}`, so the consistency score is `(1/3)² + (2/3)² = 0.56`. A score of 1.0 means perfect consistency (all runs yielded exact same values). The final output is a per-CVE consistency score for status, and a separate per-CVE consistency score for labels. +Uncomment the last section in `config-eval.yml` to enable this evaluator. You can view the source code at `eval/evaluators/consistency.py`. ## Troubleshooting diff --git a/src/vuln_analysis/eval/evaluators/label_accuracy.py b/src/vuln_analysis/eval/evaluators/accuracy.py similarity index 53% rename from src/vuln_analysis/eval/evaluators/label_accuracy.py rename to src/vuln_analysis/eval/evaluators/accuracy.py index 1aad059b3..dae3bc24a 100644 --- a/src/vuln_analysis/eval/evaluators/label_accuracy.py +++ b/src/vuln_analysis/eval/evaluators/accuracy.py @@ -27,50 +27,68 @@ from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry from tqdm import tqdm +_STATUS_MAP = {"TRUE": "AFFECTED", "FALSE": "NOT AFFECTED", "UNKNOWN": "UNKNOWN"} -class LabelAccuracyEvaluator: - '''Evaluator class for accuracy of the label field''' - def __init__(self, max_concurrency: int): +class AccuracyEvaluator: + '''Configurable accuracy evaluator for either the status or label field.''' + + def __init__(self, max_concurrency: int, field: str = "status"): self.max_concurrency = max_concurrency self.semaphore = asyncio.Semaphore(self.max_concurrency) + # field must be either "status" or "label" + self.field = field async def evaluate(self, eval_input: EvalInput) -> EvalOutput: async def process_item(item): ground_truth_map = eval(item.expected_output_obj) - pipeline_result = json.loads(item.output_obj)['output'] + pipeline_result = json.loads(item.output_obj)["output"] - label_accuracies = [] + per_item_accuracies = [] generated_answers = [] question = [] for result in pipeline_result: - vuln_id = result['vuln_id'] + vuln_id = result["vuln_id"] question.append(vuln_id) - # Extract label from pipeline result - justification_label = result.get('justification', {}).get('label', 'MISSING') - generated_answers.append(justification_label) + if self.field == "status": + justification_status = result.get("justification", {}).get("status", "MISSING") + pred = _STATUS_MAP.get(justification_status, "MISSING") + gt = ground_truth_map.get(vuln_id) + else: # label + justification_label = result.get("justification", {}).get("label", "MISSING") + pred = justification_label + gt = ground_truth_map.get(f"{vuln_id}_label") + + generated_answers.append(pred) + + if gt is not None: + per_item_accuracies.append(float(pred == gt)) + + total_correct = sum(per_item_accuracies) + total_items = len(per_item_accuracies) - # Calculate label accuracy. Ground truth labels are stored as "{vuln_id}_label" in the EvalInputItem, as populated by parse_eval_input.py - ground_truth_label = ground_truth_map.get(f"{vuln_id}_label") - if ground_truth_label is not None: - label_accuracies.append(float(justification_label == ground_truth_label)) + avg = total_correct / total_items if total_items > 0 else 0.0 - total_correct = sum(label_accuracies) - total_items = len(label_accuracies) + if self.field == "status": + answer_key = "status_answer" + gen_key = "generated_status_answer" + else: + answer_key = "label_answer" + gen_key = "generated_label_answer" - # Calculate average accuracy - label_avg = total_correct / total_items if total_items > 0 else 0.0 reasoning = { "question": question, - "answer": [ground_truth_map.get(f"{q}_label", 'MISSING') for q in question], - "generated_answer": generated_answers, - "test_set_file": item.full_dataset_entry.get('test_set_file', 'unknown'), + answer_key: [ + ground_truth_map.get(q if self.field == "status" else f"{q}_label", "MISSING") for q in question + ], + gen_key: generated_answers, + "test_set_file": item.full_dataset_entry.get("test_set_file", "unknown"), } - return label_avg, reasoning, total_items, total_correct + return avg, reasoning, total_items, total_correct async def wrapped_process(item: EvalInputItem) -> tuple[float, dict, int, int]: async with self.semaphore: @@ -81,7 +99,7 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict, int, int]: try: tqdm_position = TqdmPositionRegistry.claim() pbar = tqdm(total=len(eval_input.eval_input_items), - desc="Evaluating Label Accuracy", + desc=f"Evaluating {self.field.title()} Accuracy", position=tqdm_position) results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) finally: @@ -90,7 +108,6 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict, int, int]: sample_scores, sample_reasonings, total_items, total_correct = zip(*results) if results else ([], [], [], []) - # Calculate weighted average (each EvalInputItem is weighted by the number of CVEs in it) total_correct_sum = sum(total_correct) total_items_sum = sum(total_items) @@ -104,14 +121,15 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict, int, int]: return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) -# Register custom evaluator in NAT -class LabelAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="label_accuracy"): - """Configuration for label accuracy evaluator""" - pass +class AccuracyEvaluatorConfig(EvaluatorBaseConfig, name="accuracy"): + """Configuration for unified accuracy evaluator""" + field: str = "status" # "status" or "label" -@register_evaluator(config_type=LabelAccuracyEvaluatorConfig) -async def label_accuracy_evaluator(config: LabelAccuracyEvaluatorConfig, builder: EvalBuilder): - """Register label accuracy evaluator""" - evaluator = LabelAccuracyEvaluator(builder.get_max_concurrency()) - yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Label Accuracy Evaluator") +@register_evaluator(config_type=AccuracyEvaluatorConfig) +async def accuracy_evaluator(config: AccuracyEvaluatorConfig, builder: EvalBuilder): + """Register unified accuracy evaluator""" + evaluator = AccuracyEvaluator(builder.get_max_concurrency(), field=config.field) + yield EvaluatorInfo(config=config, + evaluate_fn=evaluator.evaluate, + description=f"{config.field.title()} Accuracy Evaluator") diff --git a/src/vuln_analysis/eval/evaluators/status_accuracy.py b/src/vuln_analysis/eval/evaluators/status_accuracy.py deleted file mode 100644 index 0cfa3c429..000000000 --- a/src/vuln_analysis/eval/evaluators/status_accuracy.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -import json - -from nat.builder.builder import EvalBuilder -from nat.builder.evaluator import EvaluatorInfo -from nat.cli.register_workflow import register_evaluator -from nat.data_models.evaluator import EvaluatorBaseConfig -from nat.eval.evaluator.evaluator_model import EvalInput -from nat.eval.evaluator.evaluator_model import EvalInputItem -from nat.eval.evaluator.evaluator_model import EvalOutput -from nat.eval.evaluator.evaluator_model import EvalOutputItem -from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry -from tqdm import tqdm - -_LABEL_MAP = {'TRUE': 'AFFECTED', 'FALSE': 'NOT AFFECTED', 'UNKNOWN': 'UNKNOWN'} - - -class StatusAccuracyEvaluator: - '''Evaluator class that calculates accuracy of the status field (AFFECTED/NOT AFFECTED)''' - - def __init__(self, max_concurrency: int): - self.max_concurrency = max_concurrency - self.semaphore = asyncio.Semaphore(self.max_concurrency) - - async def evaluate(self, eval_input: EvalInput) -> EvalOutput: - - async def process_item(item): - ground_truth_map = eval(item.expected_output_obj) - pipeline_result = json.loads(item.output_obj)['output'] - - status_accuracies = [] - generated_status_answers = [] - question = [] - - for result in pipeline_result: - vuln_id = result['vuln_id'] - question.append(vuln_id) - - # Extract status from pipeline result - affected_status = result.get('justification', {}).get('status', 'MISSING') - - status_result = _LABEL_MAP.get(affected_status, 'MISSING') - generated_status_answers.append(status_result) - - ground_truth_status = ground_truth_map.get(vuln_id) - if ground_truth_status is not None: # If ground truth status is missing, do not factor this entry into the final accuracy score - status_accuracies.append(float(status_result == ground_truth_status)) - - total_correct = sum(status_accuracies) - total_items = len(status_accuracies) - - # Calculate average accuracy - status_avg = total_correct / total_items if total_items > 0 else 0.0 - reasoning = { - "question": question, - "status_answer": [ground_truth_map.get(q, 'MISSING') for q in question], - "generated_status_answer": generated_status_answers, - "test_set_file": item.full_dataset_entry.get('test_set_file', 'unknown'), - } - - return status_avg, reasoning, total_items, total_correct - - async def wrapped_process(item: EvalInputItem) -> tuple[float, dict, int, int]: - async with self.semaphore: - result = await process_item(item) - pbar.update(1) - return result - - try: - tqdm_position = TqdmPositionRegistry.claim() - pbar = tqdm(total=len(eval_input.eval_input_items), - desc="Evaluating Status Accuracy", - position=tqdm_position) - results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items]) - finally: - pbar.close() - TqdmPositionRegistry.release(tqdm_position) - - sample_scores, sample_reasonings, total_items, total_correct = zip(*results) if results else ([], [], [], []) - - # Calculate weighted average (each EvalInputItem is weighted by the number of CVEs in it) - total_correct_sum = sum(total_correct) - total_items_sum = sum(total_items) - - avg_score = round(total_correct_sum / total_items_sum, 2) if total_items_sum > 0 else 0.0 - - eval_output_items = [ - EvalOutputItem(id=item.id, score=score, reasoning=reasoning) - for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings) - ] - - return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) - - -# Register custom evaluator in NAT -class StatusAccuracyEvaluatorConfig(EvaluatorBaseConfig, name="status_accuracy"): - """Configuration for status accuracy evaluator""" - pass - - -@register_evaluator(config_type=StatusAccuracyEvaluatorConfig) -async def status_accuracy_evaluator(config: StatusAccuracyEvaluatorConfig, builder: EvalBuilder): - """Register status accuracy evaluator""" - evaluator = StatusAccuracyEvaluator(builder.get_max_concurrency()) - yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description="Status Accuracy Evaluator") diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index 85ecfc44b..8322067c4 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -28,6 +28,9 @@ from vuln_analysis.data_models.input import AgentMorpheusInput from vuln_analysis.data_models.output import AgentMorpheusOutput from vuln_analysis.data_models.state import AgentMorpheusEngineState +# Import evaluator registrations +from vuln_analysis.eval.evaluators import accuracy +from vuln_analysis.eval.evaluators import consistency # pylint: disable=unused-import from vuln_analysis.functions import cve_agent from vuln_analysis.functions import cve_check_vuln_deps @@ -46,12 +49,6 @@ from vuln_analysis.utils.llm_engine_utils import postprocess_engine_output from vuln_analysis.utils.llm_engine_utils import preprocess_engine_input -# Import evaluator registrations -from vuln_analysis.eval.evaluators import status_accuracy -from vuln_analysis.eval.evaluators import label_accuracy -from vuln_analysis.eval.evaluators import response_similarity -from vuln_analysis.eval.evaluators import consistency - logger = logging.getLogger(__name__) @@ -75,7 +72,6 @@ class CVEAgentWorkflowConfig(FunctionBaseConfig, name="cve_agent"): description="Workflow function description") - @register_function(config_type=CVEAgentWorkflowConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def cve_agent_workflow(config: CVEAgentWorkflowConfig, builder: Builder): From f3e3c39705b19ff5cd6384624fccd4843e48f958 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Fri, 10 Oct 2025 02:47:07 -0700 Subject: [PATCH 036/104] refactored accuracy evaluator to take in multiple_reps=True/False parameter, so runs with multiple reps will have box and whisker plot metrics in the output. Added script to construct plot visualization based on this output --- src/vuln_analysis/eval/evaluators/accuracy.py | 46 +++++-- .../visualizations/box_and_whisker_plot.py | 121 ++++++++++++++++++ .../eval/visualizations/example_plot.png | Bin 0 -> 35057 bytes 3 files changed, 157 insertions(+), 10 deletions(-) create mode 100644 src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py create mode 100644 src/vuln_analysis/eval/visualizations/example_plot.png diff --git a/src/vuln_analysis/eval/evaluators/accuracy.py b/src/vuln_analysis/eval/evaluators/accuracy.py index dae3bc24a..3f1d7693f 100644 --- a/src/vuln_analysis/eval/evaluators/accuracy.py +++ b/src/vuln_analysis/eval/evaluators/accuracy.py @@ -15,6 +15,7 @@ import asyncio import json +import statistics as stats from nat.builder.builder import EvalBuilder from nat.builder.evaluator import EvaluatorInfo @@ -33,11 +34,12 @@ class AccuracyEvaluator: '''Configurable accuracy evaluator for either the status or label field.''' - def __init__(self, max_concurrency: int, field: str = "status"): + def __init__(self, max_concurrency: int, field: str = "status", multiple_reps: bool = False): self.max_concurrency = max_concurrency self.semaphore = asyncio.Semaphore(self.max_concurrency) # field must be either "status" or "label" self.field = field + self.multiple_reps = multiple_reps async def evaluate(self, eval_input: EvalInput) -> EvalOutput: @@ -106,30 +108,54 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict, int, int]: pbar.close() TqdmPositionRegistry.release(tqdm_position) - sample_scores, sample_reasonings, total_items, total_correct = zip(*results) if results else ([], [], [], []) - - total_correct_sum = sum(total_correct) - total_items_sum = sum(total_items) - - avg_score = round(total_correct_sum / total_items_sum, 2) if total_items_sum > 0 else 0.0 + all_averages, all_reasonings, total_items, total_correct = zip(*results) if results else ([], [], [], []) + + # Choose output based on config: return box & whiskers values if multiple_reps=True, else return a single avg + if self.multiple_reps: + scores_sequential = list(all_averages) + if scores_sequential: + scores_sequential.sort() + # Calculating average by averaging each EvalOutputItem's score since for multiple reps, each EvalOutputItem has the same number of CVEs + mean_value = round(sum(scores_sequential) / len(scores_sequential), 2) + min_value = round(scores_sequential[0], 2) + max_value = round(scores_sequential[-1], 2) + try: + q1, med, q3 = stats.quantiles(scores_sequential, n=4, method="inclusive") + q1 = round(q1, 2) + med = round(med, 2) + q3 = round(q3, 2) + except Exception: # Fallback for very small samples that may fail to compute quantiles + med = round(stats.median(scores_sequential), 2) + q1 = med + q3 = med + box_and_whiskers_values = [mean_value, min_value, q1, med, q3, max_value] + else: + box_and_whiskers_values = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + final_output = box_and_whiskers_values + else: # Single rep + # Calculating weighted average (each EvalOutputItem's score is weighted by its number of CVEs) + total_correct_sum = sum(total_correct) + total_items_sum = sum(total_items) + final_output = round(total_correct_sum / total_items_sum, 2) if total_items_sum > 0 else 0.0 eval_output_items = [ EvalOutputItem(id=item.id, score=score, reasoning=reasoning) - for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings) + for item, score, reasoning in zip(eval_input.eval_input_items, all_averages, all_reasonings) ] - return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items) + return EvalOutput(average_score=final_output, eval_output_items=eval_output_items) class AccuracyEvaluatorConfig(EvaluatorBaseConfig, name="accuracy"): """Configuration for unified accuracy evaluator""" field: str = "status" # "status" or "label" + multiple_reps: bool = False @register_evaluator(config_type=AccuracyEvaluatorConfig) async def accuracy_evaluator(config: AccuracyEvaluatorConfig, builder: EvalBuilder): """Register unified accuracy evaluator""" - evaluator = AccuracyEvaluator(builder.get_max_concurrency(), field=config.field) + evaluator = AccuracyEvaluator(builder.get_max_concurrency(), field=config.field, multiple_reps=config.multiple_reps) yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description=f"{config.field.title()} Accuracy Evaluator") diff --git a/src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py b/src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py new file mode 100644 index 000000000..b59f4b0e1 --- /dev/null +++ b/src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +import os +from typing import List +from typing import Tuple + +import matplotlib.pyplot as plt + +EXAMPLE_USAGE = """ +python3 src/vuln_analysis/eval/box_and_whisker_plot.py \ + .tmp/evaluators/llama8b-experiment/status_accuracy_output.json \ + .tmp/evaluators/llama70b-experiment/status_accuracy_output.json \ + --title "Status Accuracy: llama8b vs llama70b" \ + --save src/vuln_analysis/eval/box_plots/plot.png \ + --labels llama8b llama70b +""" + + +def load_box_values(path: str) -> Tuple[List[float], str]: + """Load average_score as [mean, min, q1, median, q3, max] from accuracy evaluator output JSON. + Returns (values, label) where label is the input file. + """ + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + + avg = data.get("average_score") + if not isinstance(avg, list) or len(avg) != 6: + raise ValueError( + f"File '{path}' does not contain box/whisker values. Expecting list of 6 numbers in 'average_score'.") + + label = os.path.splitext(os.path.basename(path))[0] + return [float(x) for x in avg], label + + +def main() -> int: + parser = argparse.ArgumentParser( + description= + "Plot box-and-whisker charts from evaluator output JSON files (average_score = [mean, min, q1, median, q3, max])." + ) + parser.add_argument("files", nargs="+", help="Paths to output files") + parser.add_argument("--title", default="Accuracy Distribution Across Runs", help="Chart title.") + parser.add_argument("--save", + metavar="OUTPUT_PATH", + help="If provided, save the figure to this path instead of displaying it.") + parser.add_argument("--dpi", type=int, default=160, help="Figure DPI when saving (default: 160).") + parser.add_argument("--labels", + nargs='*', + help="Optional custom x-axis labels (one per file, in order). If omitted, file stems are used.") + + args = parser.parse_args() + + # Load all files + stats_values: List[dict] = [] + means_overlay: List[Tuple[float, float]] = [] # (x_position, mean) + labels: List[str] = [] + + for idx, path in enumerate(args.files): + values, default_label = load_box_values(path) + mean_value, min_value, q1, median, q3, max_value = values + + # Matplotlib boxplot stats dictionary + stats_values.append({'whislo': min_value, 'q1': q1, 'med': median, 'q3': q3, 'whishi': max_value, 'fliers': []}) + if args.labels and len(args.labels) == len(args.files): + labels.append(args.labels[idx]) + else: + labels.append(default_label) + means_overlay.append((len(labels), mean_value)) + + if not stats_values: + print("No valid files provided.") + return 1 + + # Plot + fig, ax = plt.subplots(figsize=(max(6, 1.8 * len(stats_values)), 5)) + + # Create boxplot from precomputed stats + bp = ax.bxp(stats_values, showfliers=False) + + # Overlay means as black diamonds + for x_pos, mean_value in means_overlay: + ax.plot(x_pos, mean_value, marker='D', color='black', linestyle='None', label='_nolegend_') + + ax.set_title(args.title) + ax.set_ylabel('Accuracy') + ax.set_xticks(range(1, len(labels) + 1)) + ax.set_xticklabels(labels, rotation=20, ha='right') + ax.set_ylim(0.0, 1.0) + ax.grid(axis='y', linestyle='--', alpha=0.3) + + # Legend entry for mean marker + ax.plot([], [], marker='D', color='black', linestyle='None', label='Mean') + ax.legend() + + fig.tight_layout() + + if args.save: + fig.savefig(args.save, dpi=args.dpi, bbox_inches='tight') + print(f"Saved figure to {args.save}") + else: + plt.show() + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/vuln_analysis/eval/visualizations/example_plot.png b/src/vuln_analysis/eval/visualizations/example_plot.png new file mode 100644 index 0000000000000000000000000000000000000000..b11a33da5b462b931ca16ecb59ac973f14f49aaa GIT binary patch literal 35057 zcmeFad05VS+b$lKWy#R8NQ$yhG$~4@s4NM0Xr4z!^E_*2p%x+C8cGpB`~B@d-gp20*xRw51Nq+fclvy;>%7kMysqb(qWp<>MQo1+D)%LcrF+R4_T^YB|1TwcSZAM+Sy1VdQ_yD9)FI z7#RN2qa6D~^+Nb?m+OU1O&^xNyp8*<{^T#5fwi(qe_W4ZjnV#ZxZk}B+~q#gAKLW9%VXtFKfUVe(sLeataEPBi`Or4AIxddZzy!81>L!$ z(sm{_WV3)Sw|~V>{3j?V_~4LhT4vZ4xonpa?T#;Y!>t)dtG9|+aQas~66^8Ide&ib z+<)`ldavoO_@CFYB^flmaN9fGLmzUew4w&5@YDrIL6S4Sooi9zX=ZTm9 zVfn9l(K>lHH6D&fn2(sPJS%Mf;h3JsNY3oL%S_qMG<7a6E}lb&bXKok-CpFHuAZV9 z<=W$=(N*Z&aEtvAW2!;mo`xa!jKziNo{mMY_s>=?%zu3w+{zYd@#@v9TFL3o=%TSF zI$uA&QBhS55)u+}!XfO3Tux`1yircqFF9aV`@6BRv6_pcJ!}5rER9`YtF(9V&Ye3a z8Wf~?BqX$1Sy``NyH@w?>YAox4Z)jiB1y}27**F@QD>mWunL=)H6|!ky?(8_Fgq#Y zJ%3?gW+3AF&6{ecPu~!=ZjbrW`!d;apeDGh(SmRPex*|({5+qv7k^z-_Jqa2Q)$g+ z{@a{gP7P4Gg8T!C;JViqBk>X5 z$|fdpoP(o9W6ikp9Q)qm>{6bZ8v1U=TzTvF#WyypdM!-0P-AeDW=%K z5Qw!FM@ui&-=pxbQNzG1Hm}E9M?hfSCg8)@iHV^2c+)`RlwIeRuULTv>dLJB@ZP;d z7X1rDjaQe`D$QjoB&q-OBn-=_4GZk0RROD()aoffAp|_VW`)0zT%V=j}>Ei4u>H#e8qL=RlOXnZ; zz6>};?e?Ct);XKwU~gZSZ{N#FsjRG=eebuus-Wpe(;sFl72hqdX=%xOPE&Xwp|;t- zVsl}0s*Z-4@^$|)M@i?QAoA|W5?r@cR29odHOu6Thv88`!VhUL9ywx}A)vJ>Wb3IGVH^sUW+H)+c5sQ{B zTZY&6p!-7aIR;U4D&q7>QN*(Ow3L+8_~c}hNQ-XtDvk?B5Mwh|MxJ|l ztFod(cC){%f!Aac7tOjk&aSKCgcHW zYTo7W{$|K7;gnG3zJ^7hF*UC{>D{}tZfMB584|A-pOMZcQ_W|IeyKiXhakn zT7iRuWBkh(CBz32tJZxZ9eHox7m8RkK3?c(SNG7>)8nGp*w`?xUF)PNPpxlgnC$i# z8=R%3ho+~eTQnub3g{J@%}xx;rVYG$dU=)goWpourEG}E+VUx{iLw<;o3y78MiO1e zx*6B43-P-G7epv|MOlVAQpZdw35Y z*3GeKqT=!jT}F~{-{PIOgiB?HD}DI2zQ(3HN96i|1-6t zE8jjEyIo4pzJfVPDcTnyI%dPJNfi1;mC+PG>{ z$Ly^|8s79JXFLnF^KSkXMmN|G<<{o@%*5eYX4j_`>yl1(r8K38%eX>vT{ijt0nmT(2(}ouo;^E zdUQ^n{43+!%k_TN6a@t_EROf5n(Qsgos^ylTvBh4GOY%v?|F6nsn=8eel8(ZGn z>B#u4&nbEk6BBbPx1p=6>#nHv{jNzt-TbrPW@p)`H`cRDY-ww2yIK`~$h9Un-FrF(~k&~9i@nY`MKRorsYFzrV%CQ!z5gW&B7%;nn!cL`Y_FZE@$#BDk|!du*rY_ z^Dc4mwv(q%A3#nU4$08ue=*(TW9xdT=G%IEPCmZs1f9IJ9hLbX?^|vQv8;P^6@SKS z^G~sTS9Zylv;6Jb^xDk%Ia-+Q6P}3@^M$wg)tYaA?pn{9v+5Js%N4P6)YA1ODQ=}d zEzt~?>MyNWb-iR!hDj0$Ym?-+cbBO#H4hH4+>iY5!JsU?WJikB~G zpX#E?U&-QQ^^Mwp?uFURc>l`gEP3BL1R_d3UwW~!na;TvH+ZT?x!=8edYMhqJ)4sU z3my5hwl4aF^VF~w$>^s-1)36+q7mU#hphYh`aZqf?i0j0IozhAp=*CNra08TGei~EjH{hok1Qxx#H)wiWU}0 zM`dJ0fJ;VtN)1eF!s~EW5!>#=l6cZ9-Lt~2mlhY?X9wd62-~ZZdpaW{L&E;U6{&>@ z6&+SFF|oi4KV-ZB^L#lsp+Al$F?V{EXT9}Kb!#; zLd<;*`U_9pYXa`sWov8u=I(wOl8Of#;ziuX&(D1O_UZLKVfUfL_yf2buP+rWZ2OI_ zxD2;O37giK4b(iytu)@P@GwO!S-mZ_6en|ETA0pppR_=zPyhmxaQbw6Grwje&dbZk zcPc=7G2dmZThB~SK|!Hpyo!%H?oOMV7>s@sCdMZ&u0DTG$ zmU97;arcBxEojZD*RNk!wz5iDod2p&0-y%~RJ+}0KI(nBAB)yoz=?@(y#WNQX_vU? z024UlMRoK{;Yu6*wtFYMew^!od40^QSvD~nejeMu?0^lsg#4JcH>)T}#**!yyWq1p za{}o$G%qiYdgl(oMs4jYRo#Lg++TLL{06Hq zJb7|lTN^P7mtmvh)10EUKJQL&u-dz&_}Ez1KmPc`mopmB=|!SS{PmJFVGnICR#Xe3 zKv36HlDT3{CO5WTvgFZ|Xnk=~40XRl->c=-qb1Y&2(*As zzgg!Utg3Ht=kZyX3`PF1(B$9Ti=ygk$iNauJ}~LXl#w*TSnj#kZ{ECV1q6!{0)7?- zB-v4U_kg+bwGOw$w=zlqQ>G_Zo~XjL{Pia=ZPcsT-KX!K<(q3U@L|>r@$tzqt9y2m zLgC3?tP19{o-=y>*9I(rWpghtrLL~-zO>Ji#f34SWXqPc z!VlN^CfY62dG_v2?d&+WkmHVLFX6N3Sz!A%aCq2?LdB{f`B$Lh@zf)C4-XnJ%HGGr zfhRZuIb&mD%({!+t~Q^Mm(QIXU&nSJ;nf^ZS0|(VmZjbRY2F<>{-93uhZ^u6IFQ{+ zljy)M9g){n)&(-l^~!VTuR2;SJ=3?5;x;YdgQz3hJ$HAh#QnmB*2SNt7r#Y!nR;SB zy_Gw4>Q04GU37qyg+~cqp3qtuUqiE@#>7}Ro&F)^IsQ^Sgs#y(ORT=Uk!rrQ{G&_L zK1&)2HvoyO58WD7E8RrviA->Lf*3N3FH}RCK3ZL0U+*;DNAW9BpFwd`V`FkK?^OZ!NxjsG!E^*c4QV zDt|PZc^ekWRzUUnGY%z>Uw^mHk{5#LCijJpVYbaA$nx0A|3u4Jp=ykRP!uUTVsU0* zJ1OD6KPj;85JfF;8jqv|aIhasE-a<5kd}5Q$Tbj) zGf>DjIa@o=W-pK6HYwCMsO^GOXbO0Lk_O{=wZVCLu9y1rx?I8lM!4Fx?<_Vf|GDG& zAGdZd**$k(riEnOP#T2xl2*3TuI#devpWcjB0&&1gmK$?Du_T^Pf>c9j2G@51DPt`^lt-f1cK7Y>5#T8pyxHSJY zpgr3>8cA6fh%0NIprqGq_GUr-l&v=Te~?suMy%_6*xHwY{yqn*(t%tp0Q_T$SKG-| zNgVq|@+EM@n!R~Hzqd>)I4;&{Cv(ZTdF&AmG)K5&9`7}@Cs_aoJ|R^oLH-vmY&uwz zvayNTIh>SS8hXybUB+j^L~|AnE!`gU*Bd(M*o?Q11kwKFGW)u9Pc)UJ?=zbg1WoTU zRhs+oWN!3M(DZ`ogQhJBRp50PA5UFB{qV5Xs`c9xm6fR>f(B_w3$L3u=9<3ckeune zW87YY5<3Z*&9Xj*(q7_0qIqywm}Q>TqSh`xZJ-Y8w;f@J-7=p&d*)QWPK>Yz_r+`< z;;?GfCFGivi3umlRS)%;;_e5?V{%nR3DQX1m-=T`luSO#%}=4_VyhpciR;+^sgE3k zgdDj;#@7ss7Wpl6B+p;Zvv}Jfm)D|>)|BbMAeP^4ioYDE@E-U|BNg*|<4?6wTxmsz zgN-BfUMr0SPL%3LeF)0PTL=%|wvJWk7+8SDZ1V=EcV)|B&x`?~@J zE2r*0rsmVqimY~t3XEDyI)1z{((PlJ|7NUU;GqhD3wc0A0Z&c@aJU26XPdmaFGo8w zhv-B|ftwtX+DP6cpLT6ntz7p~%l=lxhmb7SaPhBSZ%0f9o?l#C%??!1?)krw;49!4BKfCM*)0u)cFo|6F;MO>3K{ID)`f z0Yr6yjbdECzWK3Xxg2{+YDv7exA*whuMJ2oj0-K9#-}K58{KpdbME&s6Bz8hor%l^OrBmAztT=N38XpcoxV< zNLhPyu^BBaELcl!HBk!ja>;lkaO;%vi-%yDI;2*)b~%TBta)G=#!%T_gO}7=WzgAd zTe{FZb*=_CH)2zcdVH+->#dmLhw-SH#jOIOZ`*xw@Q{2G`eAS|ac->CfJ#^|=5J9p zKm=7)Ra69ZAbC}@F6|r(bG!zE995Im_v+=_{bAXhQio@m$|32V0IG5JQ*^+##cvuM zcxFv`wg!Oq$>5V_-})>PI7kPwa&nmq3n}Gp{t`9UJ?1yXhnH+W96sarSmMIx$i;8H zOq7*-YO-Fow6si>kHzjj0U|AD0idhhrFzr!!C*Ppl;tKWPsDhZC8)RO*`||765+F0 z5K}xb*pzNi`g~Her4+z12YhBh{-(u+*L8xETx#Kl>dJL2JSgfhEi=lMm7_vYfloFz859EDPc{Y1{nvuDrC z=4v6kcV){<{s>M@nM>KJRm#M1;oY(~;o=Q8thCRkJW?W}5bhHU%YD~Zlax0#_Q>Q< zfgz4(^VO&3R=h&IPO*J5a~$xu9&grU>wVjWKax-gtS{BLeEISuYC-d7S5~)Yobw}( zGZi7ZI#f^tduMTMX;I;2!1k8s*EXJN%B9{HHoXP@r2oT*N2JuXq2+5D`^AmV9_}kO z?p2n#tf#Bnn4%>-(v~$q%_~M#tg77Y8XEnu{{u(7a{to4)dyEdrZUKJF3D2N1uvvd zJ4}xC^w^vYK0jTzIo5EUz3jS|SUWLg&TAbIeJ&2zBDeb< zb#xaWiW`8LlhgyCAt?3frHd9GJ^_+bnu$Yu1yQau_XCdSZrB=Y9Y48+ztZbZ+2AfL z%D~Ctt|ATPZhJL0;Mv?%OZJ_f!1>nQ#ah&bsHZHB&dzEVF62M-o;AmsRlwx}#N3G~ z-oEVmoR_+KbMG*?AJ;9K!otGN;362!19tU)aoY#FXByk#URPz5z-A5(0iH3>z@4;1$!Gz$v{_px*UCK1xVg>;OE>pj@KVbH zsj+EVX;YdW7vvuvcB|Bq6g25d|MKOFgvV4$mk?6cSvWulRqf4hj6lHraxEED> zdU-WvoI62rlMy=A;cv^nv)E_J8&9K6^uRaVX`bGV`?6ZoTCzOyha4Ro4uDGZbJi^N z`|#nz^u|GWhbk) z=AK`-TpgthRiLZL1tlJ~K{xMWoA^NiUY!j>|Anu(2#MOe^x$oEz<)Zjh1 zfg0@*FsxPTU1II9){;qIX{b3h8Rf2ARak2MEMEzTwbW4C7dKdQ*D!ksa3mc98Q1YT zeb@0D+;-cWVw-bDTjt--eHec$H8;%Ft@foaBEkIqv#VD_OpT2PA%O^vKWl3b0fXD_ z#ed5CmhS-|;Evs?v!HCOvuZ#3yuPV*@LYu26+oEw(UPndd5ZhM1J~=}Z$y?}x1>Lr zP3jWT($aDt$!W1#D%D8WUr+PDal`4t=>RqnIpuC9^@_m_&d&>NKv{fz8{}gq0 zYJ_ITJrL`?iEqSJNlP;z3x$2#$Qenu@fdyYnSMs|Q%Y66U(YTC)*595ac67$>~jOT zzvk*pXQyt*A{g3|zWt)$!ius^rfcU^P4!kI5b1tkZCIv~^lsqU@_kWVMXp<^Ye51; z`z$TKXiQYOS-P~?@mD~A*l!y*-UKV7cr=Q?t&j~_kyXNF~qKrhyfFv!@OmgyzAuU557Lmctt+_i6C zt!9?V&A|KjEjrZYcMi4MaR7-8EG(v0*jE?u zGH*ahV@`6!dd&5!ZAsR6`wiLSCa^P`>Dz}Qx3){$@ayK+ZV@{F*Ozpkh2+{ovF;1e zBl$DrC{P`{q@=pP;Z|;fd*1T!;ltbo;R~PN9i=gG9B#dR-f5&g=hgIs2M?@4Iy}F9 zo)C~5cc0$3e!G;_^NI>qltseW0_RU=vI`@{W$P%EdM{YbjWz4oiiGfM8HGzYR~-aI z%kY@C>8%zlJwOhgzK}JGvs*i#6gqtP@QrWZE)*FDa7bG4nT{>6yhJf!<%|@OT{{kf zt6kFOLB`v-)Ax)zT$HxuMCChc1+1XFKt+9%tP<2h$IqXSvDR7~yX-$Lqpq%gsh=R1 z4DhnMcI+mS0-T)#gRopthlnUm=D!-^`E9k@f_JF43K+Qax2Y6>pI`^P6 z<-GDz-UcWIY6BWaW>b;Vpt4^-#LV+fpK70O!F$RuZ`i$S_wE-bPoCTd%F4PJmAx$Z@VWg^BhM~g-m1Go#@9E1h(T^_ zaJdmQh&Bd>M5JJpS3`)DnM##14vgmXhcp&@m z+}F~VpnzF{NRH#&H&MZ`B2d2oReb|hcqP%h!y}vozjX{5n1^gaBbMyzn3-#N<0mO;U(QNnhXj^6Kj9+vmYb z#+PM+mD-B(@5yH%R4J@UX=H4cXwotX-R}(+xJ}E-0Wn02U!X_40i;ARe`K z7Xc~*bfmvpV~Zc5tT|U3`A2By30b{O5t1IcM-G878F*}E?5keRJtvi%Q0tCu8g1YE zVtq$j>?xLWgY~hpqka&BSV6<)_w`KWcGw6jtEj9=jaaf30sRq;=eC}9;oYTGjPt)} zXtOLafJIb3U(&pw+vtN;AhdqlLB$WbL894gzJ{KkpRQWpp%Q(3-MN}@apOe}Zc8(f zwF91hn*B@5zF?sVW~NAP_xyb6J#yRQy+|qi%^veef$Yt_0|Ta3fc(#`@iyK+WgxhS z@YbM@=?osoVkO*hFiK}^4%IvxJ?GI*E%4QK-6bBA;0fAOUp6$T06jMdzdam{00F?$Mr6dv7h7!;kSxccj)Ko31u~dm1&Fu=)&*KW0WVL;X}~K3LqkK^^*%%0B}D|> zP10Hv$Z4^LNkzXW!c+N>>!>Ou4feD-6l63om9-Gy8w;H+SFTz`iVU!R77cMX^*z7H z5+WN}U~gv041msC(J&Kx-=B46W@p<+uW9F6-2(w?T}{5%x+B*d7^o22g?ZaHepgr5 zrbLw!SP{**nR8*rlSW|3YeAw;R`I1Lptc^Lo(_$Vk7pCLN`QPd2u`TpW2{8OtOOOH zJeGR{ev*XcUx{jj@baXjgm8!(ENvYdQ7&{Ee1yNQ+p@3B#me`}l}|N?#u|wf%OpMj znd%HkEz=OM&@$ANJPCh55|A=ct4M8!iqov@;xbZBkK9X{Y6S6I7H$1nbeW2GO{*#M zj)Mm^31tonH69Wq5hpKQxRpHatX8Gzbd37TQQ6xx4P}Zd86MpwTChAk99$z8C9Vtl*6Y#QnKM z1i*hMO%F=PZu#(pCth z9kI4=U|1%hOYCa*@1N>&|>7qF6u+#ai6-+i#Dg zyk?1>a(|!U(TpLt57+h(x5wm2hsKT_J8)}?P|@hZXIobMt1+R-q>L@Tm*ZVmFkC$9 z%d|ZWfI|+;PQrUWznt~)GKPzec5y2iBJT=7+VTkbgyr>13Il^GLrpFN!={EH9u?F_ zoDlSgE+5>rje$X6jZJVD>@TkcgoRaTE$J<#OAAolr+0~()kacvRx*_JU-Hmi#=!k& zg%Qu^r4=hyAc_b;-BeTdXJE)LRvmngjk~QDj)Wxe90}O-(6kBJNF?<*ca2o-SZp&s zZtj!%`uY~##Rd9alWK&zNbG00n6w)|M}3|rYSnt=EG~Sx8om!DpK(U~O2IU|J58cQ zfXXFxKXwTX+E@#UWf5d^>(<9WkAKK_?Pe$|-g5gVh9mt+^^&QGMzu=|Jxk`0eNlx* z5Lr>EDGxGZ$>(Ql9Gsn-s9dLrm_d9f5)OTT5hZ8Es`X}|j^!;Ri5KXda*~q*4Fre& zyQR>{x)n<^Z+*^h@#XNEQ0nrY8_8_}8cqcCkUw+(KwW2nV-7$$*QfVtWger#!4e7M4))t$fBizK2kPd_wkD^sE7+O6=mW1v zJmZl|Q@(}=J^PL>R>luE^$r*Ex_k7ig+>d_(1a{IlFPkcRaPpYRxP*I2)cdy*Mh>r zaj}$`+^p^2^i8BUrO_OuLfV+rN=N3`t!Z%fd9 z3eDtbmk!EpHN3}3yLayy6y>!*O^;v|su=z04EgxQal`%FBt2L4-#XFF zz{SGOo{B|wsw%85tnD&x{&i|fLA=~F-7aS*MPD+zX7kP(XK3~f4e|O{D8KL6P5;N+ zoY>dk86D-$oKYk1LTuIU-VYykZr{FrW^S(f<;(bIxu?ul;*61~F1P`pGQ*Z_MB-C( zaL5EJh5VILR#wK?CJ@DpgYsYF>)MwmhlYkOU%tGrzh8|<<_|-8Uqfz5Nj)HOHFx(y zLXo_Gue3ul4fH-XE>M1_IK}N!&L5Y5h94-gZ}Za|2(*1PS~8p*VCoD zx-Zt26Y-dn=l9<)mR;Xjke|=}`|n4T^_je6Pd8U=SWQ0$i`eRp09-0+4wj0{ngMrE zb@li41zKUVRhjQUWv*e<#3S0o!gAu@KjZZ|o0G)*GzVl#rY1lB!*W**G)8`v(a{nUdnl@{ z{r9I|=+lpEvOuhZ$;~O>6ZRJ7*84->O^uO9H!*9bCdt{YV`9=UGI|7ep_%Ipd`sB~ zScv0kf9j|&{EFz#zz|gQ;V1Rvq+fNxYw%Xu+YS*AU*tBq0J=LeQa70MvGzU5d}7=} zoSH>>HPWRbF)YGw#D37OY*a$VcWoe&2VxhYT8VzoxO#OUzU98CbrMmdt6sh0 zUZ~+V{r&{{;|^SEsv3Uk4BEWv&6_5CZcXtbxgZe{HB@XN{{9QEzz{!zUL>n<_{$Zd zF6%)_Bti?UR6ywli2y}liyZ=|ch zF&^SiDqw&f*_NRKdYd+Gq=Q0&BO0D?EBt+FkN9zEub(;tR>S_pyGa{DKR>ohU+p`f zB6HM%gY{biO7hHubJ zea)r|rrLR|WP0fH`*PgT1J1#+ODo!8Ga|+$nBrRXY~lZeYa+s_f?2~1lnQyNm-@k+ zq_|IyrUK@ew`Cfy43Yd?wqks0O5pQ-yt%(N?7~af17h+7RM@PdE}|Y0RvYwybviWG zqX3GJInAMma&;_YzT&r%EWaZ>>)0h!32=mB-IQag(eeDW>|`b|m#VMfC!!*P*Xa^M z3~Z>6J(Ytu(S=fNWe71nfr%f$QSIkBRG|oLCGD z5-We~9AbEa!0)A+%E~&>@u?t5#J4TyC%!`$1KZljOI9T*kpk2kTol-kdFyU>BATD1qM*<;YXcLP4UfhLu|mcj z^jOI4%C$O8jpwm8;cG!IRJdE6hNA1#<2B90%PWUsDjhaq^@P({&2b75HetGy*`6yD zq8nKMmBCr@{P`ycCyND!hG>*`2`HU_mdN{WjG=5K9!$vk-dv@^1JUs$ z0Uf!@lk{@ZVRZ_XSO!wOiq}*U)H@V-=GBzI`rM z7zKuHRa*A3Z1^cI^}n=JpgJ{kEBVTxyuT4|Bkje@m#3(xa(9X4)f*i3yK*JS-0Pjg z^F3q}HgzL9sDb~hvhjWN=+UOlo7LOiFeGjAT}8kxS{EJ?(4*>s&;W6nx2bUd{{2|( zC7e7$LdqzOjV)GPq|&!xWV<7NRXy$mH1?fd&U^zr3HkMZs0)Ie>c6?s;{T>oB)4B_ zZp$)FCK&6NUv9%*gz_*JZto+I9=Ou@0P{)7x(Q0xccU^N`UqUR-Qt0|RM)4jBG0O| zSG{eWov&$xmpZV+Z5VglCqC@}R41Y&_?%4Q{2(z8V5lOAKoY$|C&HNZV;hnj1Q&BX z(r(WA%HK$tO9XOw2difENIqvs@>HPxxpK_`!)HIik7EW_n|Meee=3(?sB?p|#3PgpcR+>BIdD?t#`s3$N_Aab-6&(cd1y<)LDN!TkVl%8 zz?@Rt3_h?-ucZ@_CJOSXz)eg{qJ&WF<$7EY782Uly+RlE0~mWwQo;5Hg@x%P+yHvt z+ZZA3qo0_V=t4_^eH>U2UJih$k&r|<9WIr=zq*EnF&O6OB$fDEoIJ3uqQJ}1(O6ut zLoO)_kf|!&4o>Uc=ls=J@L+N|U(Q95hNrJz;Q-nao1J~2$Qf-Zq%i>*NABdw$2u?Y z2wcLUM#`f_=xP<9_i$HWt73aXU~ai*RFxD30RqY<;qVTU(DbWO9DVT z#XGAx^_^mkDb^~7D$5xB1w3A=b`Gc7e1>l4TPLl6eyVre12_Id?MyQP{g0#?Y7U2d z^@-yf(oECJii^qgiTk>}`qq@p;<5t-PgX}VIGJ-%&X?Yr?{c zhG7nzz>x1&n_3;)l2~k-Lx&$@fxYttO%%+b$=8AH&Au4qNCJ`IlW#_sj1i9Ll@qioV(BIAQdN!KmV#DJ zPEPlRmI>^El`_|G2{OLyEfZw2?K7oIyWsL*W79V5sHv&Rom6dx#r2urClEue?ia+T zqj-5OM-y>Nq!Es1tep0j^SvA^B}nRt!F(EW)SuUH(=Ny{I1 zL2YdW-ZPV7n8u{H>jHAN&`&J~}V28oZXU>8{iC z#wQsrepMZ;W!t z#|7}{B*Iq ztkw!`z{np?6_Q|Zp0{S9<0*!ak;l<~(3|uZ?a^#p&f=9@4^$nm8?Fyp&8IIgL2Ko*j z;Agp^_#z0r6lF99@)pMn@7^7!>~R4}S3nFv+4wg$wxq|8A2W8DKw#u`^1uZo9A3*r zfAW9B{Qu87;Qt-)y&n4h9Mo!ndUDVrR%P=u7{+G>s}c_5Nvl=i#{^M0k99xrUxjM% zUFQl_funpEdCT4Feq~{a&(A+7z=8LF%aKWO23%p%m3fwR>bi#+7^*f7aFlJsk50es zmc7i(!UPVstYFnXCM_FoX`*40Q}bVG@ES58ZSEhYZ~3XY#}!cKC+ za@N1E;reAqrnBDPxhi@=c=rXGk&{Vb3|-JTjWoAu!@ZG)mY_Y^YZ(;v$mM_!EK?1) z|57Hqf+3QVTn!vPXC;gaF1&k$i($jXd?gnXTd#ZAt`66-2G=4^u0@?9z>#mpz;JK{ zF2$2vO6oxtCfx%J4B^*sB?7pTIE}`N6)fxi{V*=_KJEVdC(rSCR*{FhYw7z>uH%6v zc+%A??$xx4%ac90O|cruBmFpk0*r0QCkM!14KM7&Cip+z`AyFGUm0lr6%CWlm_2|# zPb}1kuMmC+=|Ot25t!A6BtSYPKvU!f;)kpO#x9K*j7vb*$XkiipF^O<25q2ZEh1TO z6|+$VO-DMX9~`tlN^n2QfT)RyiQk$+(fbQVG8rr&%8Ovq!b@^JiS>cIP9XI^cW~cB zzVYIGCvVX*%iW&p<>NH4vK6S6s zg{d%PqBs&(!OjW%s>wz0AKXk%$~5}d$Rhhrf>CTQbWY|p2N8QU1OkB;ZbRevao-RD zJ?#_;c$HM(ue*h|zy~dwkHvH*;_p=c1iDeL8z# z4vJF)>E|V#?C>cNK@(^r8nw$1s)lAHnx+)3tW(Xbn>T;7(*g`cQvx#!i-y@f=qj)m za`l%eoDlxK1ufCkAfy(QF0FfeX*50zAfPD2MK2^Qye8NlLL>P98K4y_YN_9jx#* z#1vPi;k~U1{_@=&!AUHISN|+kLtz0p%v-*1)AsGQd8fgyu9;tlc1y*C8<;>%&p>py{+1uTpa?q%)ItgsL|PDRFnMWM1w^C~6Vm{d z=u&;b!=vNYSMZIvu4qM8@KGI-t@rNT10|TL6B-c_VbPedgCYUD6l4k91+-6mHz$vz zWY?5dR%S~L5-F>%(x=J!xg8J;V3f@3_&36jFEiaQjhBcAs6y>M*Vfde2GJx$bs}SU zY)lZykKz^qUw-&eW8y18*+W@66!Nun8ub7QprGpi71%!*k~$8wKdUJ~GnH7%7&epc zlMeB3XM)|q!LhpNz++b%>OpBOd+#chz=3%G4#m!M1rp&A0#ei8(Apv)-Bv!s!-R4m z80Em5H^Kg=2FFgP)=y_k=qz52L@h^(yJzL)q)=B+)h3<#7U&y=QX^v1nJ;I>Lsq0N z2fuGgzP_v{_!Q+0BY=i_f+y1rq~AQ0EP6Fdx?o^5O(Ebs;562E=Y4THJh~lvKoR=i<%ujm>ugMT}Ja~O7HWOI)?w%h(`g3r2 zL<8lVn;!1>dBA#wY?H`&wf`KX7il`9J!pX%Lz*Y`o;dW&QIrw;PBok-Cp)ql7@wB{ zTTme9&8=ec9eK8SB@gf3O+b_NNh&cMcf2;D9{2&OtMMk`XO=vAPhK8iCEzQ1B1Ji_-{0E;HD8!poFNAfh#(p9I~ffk!9F zd1b90( zX?;vePOFEyQ{7F^s7DgjzB7;vM<$Q`2O{}D0gxdoGL{&KA)HKqM!Fxv*|do@f|#mM zcGje9CO2hw&%uID%;@_pIHR5Z2+@p4kTg|iBLANM`k+X*4ViEtIdL1#1qcUesCTeC zmb<-!4ptNobnuDbV#kbtAF+tIApqyr{X#@X_h6T z90X!BG9m+`F`T=cTL$5VIoE7R&T>(Yc|5!swmlkYbb(sVMS-LZ^Anaz14O4pBAB5r zX0F>r4nM-IoE8PS)kh67jT0G70PB(}#1e83Age$fs}MPsRDk65zLlKTbrYrUAH|gg z;ziHk7d&%MNEW7=SV$9`<@;38fVOVClrHJ3RxNU|)@dX6hBMl-kF`TA0Y~&!tb+}-8AQ0638pw3zLSVCqDt}^W*-)VswHo!52$1fc;P$jDWM$e&Y9lV^qSift=!^#jC_9 z5p+&NR0!oC7#MhoE(`MWQ&_O6;I5{|dVDG!;D{n}#E;(y*J6UKBvg$S&0#DS0h94Th1-VA3gp$}$IqUKHjT9M{(Y7D|XH@f=q=KC`!Qm-!~lM))#o9%JM7u=Zh9)gHe;j zq(_!M7?6Yf=eGFrZ~m@2F(`84sUTvJc^pbugJ9ocT3a>Q0i}qo5f*T$jq=!RpRg|j zk?{Zh$lVoaWY2;bB)F02+QdUfsfTSAKr<21I05!d(lC!s0O22(2S~W%@C0C*OZl)s zDO|e~AZI_3IZk-c;RWX8x{rwTHP*60_y_oGT`Zlvmjc+4+k!C!M7<`K&3k_b2pRcx zsbEqa9OrthGvV;0;s#om_rnnh{;2<-cZ&Oj>2i?JPfI@7=5kab7ka$ex7NDn2^rJ! zUf5Fv1|pCR{P^`KS813PL`+AV6moy$iQm1u zg$zF+eo@4jy-Z9@C2&#x`&>hg+PGT(N?H|P$dd*W1u<=Ht7Lt`#Mt5r?PZ{!);u{v6qk+D<6>S3x*kHPsju!mtQ z1;=_YOO6~>X8bxaYLd)jfh%;)nHN8iA09DB0(gQ!6|jTUZd5oNg;^G+>h%4azgFCN z>K%_t9Qp$E;TpM59t(+ZupCoELp7o@F}a2q)rH0Axg@~^`yFM-nGVTRP`dC>t49gp z5MRUZ&KQPJylE1vguk!oGV}262F7>eN^e_swq!Td7gP_tfX-84Sih4DPzY$edI>ZSBTa@Z%v_-=(~Fr{zb?z2-=JXvnp@7n1~s zOL^K0MUZ8}IF8_+K5T-=;qS;~H^UR!c6nX&=L>W58xQRF*rzBNetLgU zFZ<4!mgM0$)w|)FE_+;Oi2MyDyMQznpl$IN->w!&fX+ldi`$DNo^jVc(i?z;YYt;`Y|?x0|JvN&sLeE$3=B%JhDe+RBe#t6xLP?U^JOf4uN zmFxaq`1^y$Uu^{>hkHfsMI1dYsGYp(*9?ea4p%F}e%hzMzhX?_Q7YJH#aV;1ZCE zN2}4Ll}a4asJsZ5{Lk%lhHJH9YKsUO^@u&0LVP6MLapwbP58A!_;p2GHFsRaCQasn=FKAP@U{f-WEY}8~4-y8JlHYD(Raf%mVO= zpE1Cz7@19ii62fMzc00>+bS|N7@pBLp9tDsoWB7n${LUn%g_18W6%doEcme2OYo!a zO@=DKEhJR`&&tQSh^$^4`IM`g;79!B-{H~1Y!cdzoLm)GH46EvimT$0QWHxprX$1YHRbS{q&K@b|0|z`{bjtA8)2kFTqa z-`bXlK1ywZaPeFOiDL?hCX|!(3f$SF&Oq? z#6gNPcKyW;9zzl#&Y%as$aPF0;LUevbIoCODvR)18DhS{!iXn5y;z0J8Cv979bwi0 z!iAE1AAR?6@(&;Z%D0(pvO!}eMv4TE{|mv`xtqN~v=0||oPdO74j@GzV}VTZ3H^cf z2o;aee%9O;Mn|uzGV*Iu3D(DufJ3A(Lxpw6a0_unLkT8;dJuO;G?^?>x*(pvHi#!1 z_9bnn* z^xgn0*86CQg7g;zHjj7)5UVU&A4dVFj@`(mzeNFI2xRJjC}f5oLDZnJ$P*>g12FW^ zy8P-cD2F&F4Xd)Ios02p{5 zJGCeoyUSaeJCC#nn-+e*hs0;@-z`ntJpT&ipqR<8e=IKlscXVG%>#k%yWN&#jt8~c zvL1uemeh((uWgm0i!^uf{ z9tr{eCt<-j6k9?*d68 z5SBuv9fiZi`bsFa3x_}iJ8Gi3& z(Afscu@SrafJGxWbjF~>o9Jj+9}4FmsftlzUJZfy8r=^dD5=yBc0!5ZzU8Xsge1Z1 zbqN7d(}z-$F{Hx6S&C$y7ie2KDp-JR>*LeVOA309vw!mVz{ii9d3y&3d3=0)DD}u# z!Dvj)O9M`>#~C1=`!P9&jl?W&10i7{7H+P`{8&LiI{SkVCZ{I$fcLRQ% z;*t_B3L=uq@#F9p2nYy}V>n?}{I@w_yAkrK%DbIqRp^8Q{-?$O?ITxg(zHo`fObTZ zE@(eE6UbOuSf`cHet@3dkq{iBM7kctr9$-d^wd!3Q)IVSqi!XwY@8Tk!MkTqydokmZnhUkSLf}30EMWpcO=6j#_hmt z-W!OAkG>#3$osZ!Fb9X#La#Q{K96yqU;KMfp^+g3m~*1iv5<*1dBnH~OpUelFUvwV z`Wp(SX>5QCwF&ik5=bAk0X?-LJ*H~phh(NCvX*XU4MdPnp$6V{=tM){7VDrPtuzs+ zf*4PxG0B^Qwak6wm(O6#%+7ATwc=N1X0^V*{v7(~;628mcn{0RD$n=ol`E>Be(ge; z)DaVou0M=OOMPrTpiwur)h;CS<3nDrMjgLB;%tn+SBmOBYd6*p!Y~SbSZjP`z$HM{ba5qicxF0 zf}j*AY1m*!$@Nh=h?msy-Adx%r@C0L8 z^=W~0Hq@FCjW#PD426IYWlK7DU~nS#?Bz6!T$Bf2a`u8Cx#GLJ!DHsZF{?N3CN_4` z{Rcc21BR6s9R@I4HQ;qxGl_ra-u9}^il?fvzcrUta~79Qo**yIuHZw_cJ45|8Vd~W zB43&C@PEz(|Gguh$vOW!gunkw?t*BE3`q}RknJc1ZOiA`fk;M;LH+USO{!7p3nL?= z@3w}}Ly&`iI|S`*qz9E!((@2J_{hR@kb8rkp|2wqtyDy^XE>5`LIFowqO}&B3O@xM zC5g2={%5l+bMDAI@2nS4n{HbEhvkY!Xx+!UJ!dQ?11l}2^CqME#tF84_n`kL6!(wj z0es?r<+lER-|+v(gHj8th3;qt>hBDaD%w`mlhsovp03)?jj46VCK0A(ho;yR5*F^(80mL| z^$L*mUzjp?k}9W+iPBHLeON=OhY>gqLaJV-0Pqt!BhH}HwiF#wYxpfm1L3!KnDmON z8?3wO9gWP9a(>OIImz5kY7Cm~n6PTC?(Asj*!{5-F&VRtOH4svdUp3;! zCNvm=n><=wTwIKn4%ibY6eV0>mjs5$@bL)#1Qhzhh84_9$1zom3PDR9zv>H2mE8QC zcw7GsCMi&#A-(QZeStIzNJWi7k&l+MCL2AX5M#CoDmqLR=@NpSCMX~vfRhKB0qF-) zePNVA#<){sQd0Qg<7l?go0%?0VK%bw|q%K=p-$Wp0pI=I>lw6*i>DKDU;It{E@}SC(#vOzaG<0s*y3)FFx=^^QT@ zBpPf<_4@qzF;MNBv`62+Hw<%1eH76D!8@+noj-I!c;xlDwmR=uKKv|`_gp?^gf9u} zQO~tKBvJbZw=DGQ1Tclvs>OTo$mTt`vu(t0{-uMxbi-85ncoKX@YN`|C3VbHa?pBe zG5YeUONv(i(&4xx#gVJiDUsa$Z>gvi#zsi$iV#-~#PhE9aaz^@!@fXZW zF-OZA**p36y;Z6#WbH>@_^xt0|gFNh4@bI`elU zBA%Hu=EsD_wuv_Dx1wn|zHyZ*voR}2I3~Ev<>cd48nbm*e?E&+@z{wI9mmJCP7E?| zKcPdz92543eP-MjIXmoDTZXz(ps15c)WMuya9zuKcCP0{d&Ka z=kl?7)_Cu}@`15`0F{r7NB!A8j6VDUYR%G_6A(UTLS&>w)i=%eYagY^qRPOG6#Mh! zg%s2>y%I(!$FcTQsZ3+rJ3Za7()mHnptAgp@nTp`;d4bZY0I+VLNCs3%s<+60(bW(3gYE-!o`zuWld2?safcBBzv2l0Yx3G(Jc~*wF0wGLxS`LEKI4pJ|9NxgK5IB-rDjaz z8Cy?mzj7`wm{^1SfGcxX%kPG;vMq1DV?IL4*YmqU!B<=5{@GJW966yUuT9QI`+y26 zazPn6Dnc`A`qq@h``MgJ2{SvKoUZG!?r5vh{^vFNTu8c7r-IZV&)h9V9ekKti;B*9N<3|~loA&hCdYChzd- zTGwJ>ebUOY52LH>^6Px{JslH!W&QunB3noFqOq`15G%8@j}@w$`lhGMI=eC^GE*|Z zS(DpGw)n&hQ?W(*SUPE+agzG_CI$c_4!-$UN-C>H7EKF#my*)gz2rNjt^ps4F_LMT zP+i62L@U?N`fYgV)yANuJlJ%%yj}iIb+4}+v>5GPd)vJjaTM#Qx1B#n9HqRi)6X@} zemv(tod)I{~%!q+|6}silbV8n$G4rFO1tUk7%J&Q`Hwl3Tot6+RI@AnQ1ab?*K1!@@2_#nvFMwTee!X1 z)oJ~WZZ1yW4tjs~t-o-0%gR26Rdz`_7FVS1q(((OI^rPO7ngC6xKlr^y6(*z%PFgz z2d~xjvx#1$rBHe94Gg|A#r=hsFTO!WkhQPO>4XqW$$l%JJNd6)03M6@`Ok8w#Hl3z zS9P|pPDi%?JsqhEp%2P(y`otinebmP{YOk10qH@+{Jw(7bQ@OW+nJ7dQ!XzyNxGiJ6C3Gz{{mBgUM3lVS z%9q37og<~=9%kg?;!<<>=P5kXDd-;*QXoknMcXYcb@Z}k*Ve+2=poYMKXzMTkGOk2 z?g+6c2H6S5%%~*j@@We9B9d_a&0ZAYc$|Xrv)f~$Fc6R|3zf(M@{ zj=T`;4q%yjEES%j(sh`*uKS(=XIDIbNuA?+qRd1oKu#ksAUQNicL*RI2Fgdue|Z>% zOsYP<$!Ux<3bJnIaDMK~zU_l2_tqN019Uy<7gxUO%?!CJnWog&JI$Y;NY(xsTp}U! z@n5WOwuF08;}UBI*;6vM0oHg>guVcpiu!v!eH4*Z%VgLrTSdC(zYS`p*m<2@*$*Ds zX8oZvh1q$1`xi=vmp8$6#TS_}|3X{$=^5O6J6@L^d1yjc`RPKaW7L;bP zdv0M4lEASi`fDq|h}FxSkH?^v;81@CDKPA zt?$J<$#W?Jf6;5!>FFZs0dLy9=W0|_p2&Jb=OAyZz9mo?5n>mmDFKVaxtJiGrn*ap zTTNjE@;m8UI9@lrXlE<59OL<2`$(f5T^w6t!z}~^_X8OwlDW#A0ghILYQ;Wg=4aX+ zWESEIKYJ3lk64nR(WJTkyNXqZIo+kK!pFo|DZxc@;Il81ghjj=Awo>(zVuQGW)2mjL!yK9=+TwlT5gs#^YC#I zi9Lx`oYcP6O;pplh=j#2?+s&UC0aiM7elcm6heV<<;Ubz&+dH1uO|cP$*2bnSBTmP zBV%#=0frUU(BX&FH#y~tE_M~Mc51{WtgDSHy%oV@#37I=$T*nMvQ>Q^E4-IY2jM_u zC}NeP1vE^+c%u(cM|uL~4U3ntJAj0YeY4O`OkzPPC$svHuS(_`MDt>vpD^eFg96bU zFR$#?V&cyP&@ty%6l`O1PSr`+?;&m+j*t?7_X`Js?H`6HB%TjLN)3He!sp(_5DQC* zgYp`AZZ{jF_jqJmpX9GNRSX3*$LjBO#6}VOZW_-6T#mQ^iLn2k0SlcO2_bW3{B-qW z{pw3ys<+0|19npal#Q(ukp@eU zXV1J*05h;6ERSra){*g~WSu^_Yg4P6D3X7qRPttmEX$E{U38R^=pG1+K%Fjp?wa~I zn7$+-M$|vf9M4W@+;?2i(aM_~?Zo(AN`{dra+WsU>j&|}eO)C4B@LlQulqsr!!yNE;GBoK;n|52#4l}!%H zBQiP~$#0k6dAlBrl*Jc_DV>FqRC8Ozo2yIFv=iI|%_qByA6pDXp{Uw2alM(cEc}4h049VY?xPIjmRk0(+*{1cgyUfJ6y2|4GJD_0 zS%N`8GlFK7Y41fk!B-v!JT)MhC~o~dQL!3`L%w(|(4~=cMovToC*sH|bpI$SVX4cJ z4BmGj%*z>Kg2k-wh^dm92F`E(ZKoHcqo%C zy=SM}1dIw3FoP1sGIRvQ$ENk**Lh#tExWYs%dmLVI|Tm{P~8^A1TuWBT+^3XFX>^* z^jH8^SVAm8ELe_B(W(`y;>gg?y}%{k2s)*>68yaNy*_igS~wME4;217La?;$hYFsJ zI3*F)Nca-B1lLQBZO@>@g=|Z_5ZG(jA?61x+P=D_3%iXg@!6`?vUTfQc*Hp71g%mo z->R>gm2+AWFflHr-RiV*<>1^k8nJp`Y5jERV&21{meYRulAtEP&$%PoPHe=|-ZPbY z_?69E@zhwNUQg!~dCCt_7gTMbB`@NmRDMUYpYwWw5YSs3GhxDQX<3S-39)6}iXe@h z*e=-i^Y~c*E87=bN5Fv>CmI%a1&$?AgT(GQC~so`D1x40o-N^lFCh}bzyk|1z6A!! z8IZtqfI9nnz??A5SE_x%+Z>@YuS?a<`CkG`?8G4vhZHEUx}&VwQ{mWT$L{=Jw7sWQ zYn?9cHrab>`c!j<6e3|8z3jJE%H_YzE5lSHKFcG^b;ObSMPAH@ngyltD)o4a)WjP} zzp!`;5_+{dXke)xrF2k8XX12YL!hLqoVZn`C&anZHTD@f$AVM6@nBT(l@d^D@g&>= zhKXWHpz9RHK>4x{d1=Xr$Zc|N7e!1IP1|{T37y zFW5XHTyNakrUjn6@7z%t2V@-ecW{e0_|!A>lUDohHP!T29alBg9%&x&neGp-R(|1O z`0M?e_k-_gAFdd%^hSxtLYtGf(_d#l9~ltPpK~ZLH@At0$K#xjrz=Zl_jp@hSND+A zYjbgmt3!;qys- z9J9vMq@+2(iBnwDH*#RzyLayl@?>{1d^@wsWTU;DP zmcqzNb@1RkW|H5yyD$3vw~>ZvH+J_6rKJ)^*wg9M=`!0e>CmA>l`4cKOo+4f@iAvr zHEAeIITaEzcwocfHXj={8rRvJ9KDQGVN>nB7OQ`nl@ny{maz$1&v(d(FoWT=FQ0hdWz7vc=6&ZnmfG-T(B~&mp<>p z!tM|F%6vv<;hpJaeQZyzeA|bGVq|30u4~up#9Tb?9^!)xYp;cB;hl&bJ3Ky&q64F} zx1UW`WIb@;_Pu+rK~(KwVKE;-S(kx}e_;4HE3j-sNSa?`MqEX%dHC)qWdYwIHg51A z!vItN@%}v~2>~FChc90~kSjB=vFS#FU3_T9SS)dM?k0p-4aoM=U9LIgZs9qf{^5KA zV$El%3Z4`u=`h+C;v+}fdZ;HS3)T(-q@A2>M&n}C zV8Rzya|an3uEl+fjSt3q|B)lxh-hp0j$ylYUB*@Rq~Co_%j~-L6K9!e<-0enC@KDSV1z38RjS^)lZ5JcF1q%pn(PJnjJf&RUq!I*3T;|o1Gzo z$~x)azd!3CJR;&eEyS;OIcRn49UKtqmM z$BYV8Y4Y&fBhl*U_!+rl(|c&kU~abvr8z=3)%ShkF}@6+FgYhTGPSVC^XU(iy!8C) zHrm==7LAE<71XvMoPR9He=&!ihPYjKso=l1#HNjh)eja>7D2~pQ?F_L{T%RNSm@zkP0)kQcSuE!u@u8k~6}bZ`wJZTB!=-(Fp~NQ<&V1fLUmu@M zYs5TLK@HJF5fHFB@GxWV-ZY+h{`O1Xz_)ijmkdD<%F)H8FWm|P+#h(9?5Un$VBqW!=M>-S z1L(DssYkyj{5GjSew+MW!dH0s2&M9}Va33p57tu>9lkU-7e573w_Kh%cf@eOfE|Q$ z+GD}%``KU+Wiqj``J1!TJjZ(d4q610MUcJCaD~%~Ezol$>@XH>d4?uV=T)~03ber^ zIGEfroa16=7wuDAod4pi$Ii8Dn}~x!L$bl#iJrP%n&3Wk*oWVskec^Z_+>jV?gL3J ze&3d%N=Y&5-#-ejAN@;=EW9^mMpOf@+M{7e8Z#1fQ)NDbT={4+#1#L{kwOIKfd!9YV&i%ta{n{`lf0(ZS*m? z*(YfrtW*U#-xgLIZ$=nVD2T~WzrO$JCqt6G$DA*=56WATRJjAIvCZw<{ENYZyHI>n zoLvVcfKzCD*cE66T#KU?^7%Dw11Zf*%Bos)6dK$g4Aa(by?Nru-NB)ut;w9GXD-=5 zSKz_P%io;4aA6(Ko5F6>14OH?#SD!>$?e+wzC6^GaJnRCptT?(++DhKQOLQU-#f5d zr%tVGIc%`){BhBEE}dUsK;4B797-NX!Qzxsg~CH zii(&IA66oJ<<;fH(W7UJigX7J8e}mu>k?hUT|)x<)59>aw%$qAPD3F0UB0W6lY7=z zw^qxt){T54A^}Poz^giLib%O zc+Z~B)0X|U!P?roHzrnQW@h%|#`W8>$H7G8 zvN^4GSkB^oR?&txH8=kTT&VffBf$jeDO0AbE%A9oJ7Gz6aU0KNf!>{F6H=N7b1>}^ zrcBB6Np^O-*XzC85Mm`eWjH|n=2aEJuEiQ{x>#?6mAd%RwN4<&d+)iP*Q5c|F6Skf zq(uP%j{rq*($AYWZv;62sDrIlb`ljZtXe3EJ~?pj${S6hMyy!zHDR{5%~0Q8 ztvD?2oLH!jA{hdH5fs?@Cl*#*^lxooP}@-z7ZYMXgjGLZmR9VX?*deo7lD)LgD?9~ zd(|>;(bx^Q>CtooMRK&clF(=)%Vb%n=ut(=ZJ>cL!22l-kc3a!=<3D92 zk(imenQq>6Sy~ zS$XRT3ZC=!1CeD9PAf>wwcmOGk9xrJRL7RCB+Bx)+S&&b`l-K)fx+DLKi`M Date: Fri, 10 Oct 2025 03:27:01 -0700 Subject: [PATCH 037/104] readme updates about plotting box and whisker --- README.md | 21 ++++++++++++++++-- .../visualizations/box_and_whisker_plot.py | 21 +++++++++--------- .../eval/visualizations/example_plot.png | Bin 35057 -> 38224 bytes 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index b5558da9a..0e7b28b38 100644 --- a/README.md +++ b/README.md @@ -792,7 +792,7 @@ containers: ... ``` -In `config-eval.yml`, the `file_path` should be configured to specify the specific test set to use. +In `config-eval.yml`, the `file_path` field should specify the specific test set to use. An example evaluation experiment has been set up to run the `test-set-common` test set on the morpheus_23.11-runtime and morpheus_24.03-runtime images. Note that in order to run a test set across multiple containers, each container must contain exactly the same test set of CVEs present in every other container (ie. a container cannot contain only a subset of the full test set). @@ -810,7 +810,24 @@ Use the `--reps` flag to run evaluation multiple times on the same image and wor nat eval --config_file=configs/config-eval.yml --reps=3 ``` -The output file will show individual accuracy scores for each run, as well as the average accuracy across all runs. +#### Visualizing Score Distribution Across Multiple Runs +NeMo Agent Toolkit enables flexibility in defining evaluator output, and the ensuing consumption of this output (eg. uploading to observability platforms, or running a custom visualization script). In this repo, we have implemented an example to visualize the distribution of accuracy outputs after running multiple repetitions. + +To enable this, set the `multiple_reps` field to True under both the `status_accuracy` and `label_accuracy` evaluators. This configuration option will indicate to the evaluator that it should calculate and output the minimum, 1st quartile, median, 3rd quartile, and maximum accuracy values across all evaluation runs (instead of a single accuracy score). We have also implemented a script in `eval/visualizations/box_and_whisker_plot.py` that consumes this output and draws the box & whisker plot. The script can be run like so: + +``` +python3 src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py \ + .tmp/evaluators/your-experiment-output-1.json \ + .tmp/evaluators/your-experiment-output-2.json \ + --title "Status Accuracy: Experiment1 vs Experiment2" \ + --save src/vuln_analysis/eval/visualizations/my-plot.png \ + --labels my-experiment1 my-experiment2 +``` + +You can list an arbitary number of .json files output by the `_accuracy` type of evaluator, in order to compare experiment run distributions side by side. Additionally, you should specify the chart title, the save location, and the X axis labels (the label for each box & whisker plot). The output will look something like the plot below. + +![example plot](src/vuln_analysis/eval/visualizations/example_plot.png) + #### Disable Caching LLM and embedder caching must be disabled for multiple evaluation runs, to prevent getting the same results every time. diff --git a/src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py b/src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py index b59f4b0e1..fb3c700b2 100644 --- a/src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py +++ b/src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py @@ -16,22 +16,20 @@ import argparse import json import os -from typing import List -from typing import Tuple import matplotlib.pyplot as plt EXAMPLE_USAGE = """ -python3 src/vuln_analysis/eval/box_and_whisker_plot.py \ +python3 src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py \ .tmp/evaluators/llama8b-experiment/status_accuracy_output.json \ .tmp/evaluators/llama70b-experiment/status_accuracy_output.json \ --title "Status Accuracy: llama8b vs llama70b" \ - --save src/vuln_analysis/eval/box_plots/plot.png \ + --save src/vuln_analysis/eval/visualizations/plot.png \ --labels llama8b llama70b """ -def load_box_values(path: str) -> Tuple[List[float], str]: +def load_box_values(path: str) -> tuple[list[float], str]: """Load average_score as [mean, min, q1, median, q3, max] from accuracy evaluator output JSON. Returns (values, label) where label is the input file. """ @@ -58,16 +56,17 @@ def main() -> int: metavar="OUTPUT_PATH", help="If provided, save the figure to this path instead of displaying it.") parser.add_argument("--dpi", type=int, default=160, help="Figure DPI when saving (default: 160).") - parser.add_argument("--labels", - nargs='*', - help="Optional custom x-axis labels (one per file, in order). If omitted, file stems are used.") + parser.add_argument( + "--labels", + nargs='*', + help="Optional custom x-axis labels (one per file, in order). If omitted, .json file names are used.") args = parser.parse_args() # Load all files - stats_values: List[dict] = [] - means_overlay: List[Tuple[float, float]] = [] # (x_position, mean) - labels: List[str] = [] + stats_values: list[dict] = [] + means_overlay: list[tuple[float, float]] = [] + labels: list[str] = [] for idx, path in enumerate(args.files): values, default_label = load_box_values(path) diff --git a/src/vuln_analysis/eval/visualizations/example_plot.png b/src/vuln_analysis/eval/visualizations/example_plot.png index b11a33da5b462b931ca16ecb59ac973f14f49aaa..9303512e4c3a055081cff62721336a1e649b7357 100644 GIT binary patch literal 38224 zcmeFZ2UJzrwk?WU79|GCfB^wZ1tlmHA}HBViW^B1C94RCgJg!l!2W5`gugCmRD=cTb~d1juUf8spzliKkWJS%G>?umev z0w0?u=bj!doC$ko(HbhaL+2{v zuTz7K0hLB|gDJ&&pX+$#_1&iYx@OjHc(p;R(*IX!_39WnG-XXy6yzu7wNhf#l}B3?)41|OU|Vf{n(aClByu2k1Fd-o1ObdhtVoMoMDv9j1!% z^0P@=S+$sdHT;=wp%6Delu^x;#lCXHpGMnb;Pvw5%gOs5-%ghMA5@W(^ZMM?m1WT` z)^1(OPK(SkZ5)_xwd}oPy1GlPVLe0QrHVj78h=<&XPUvS6DLkk*E2ISYm|DkPc1EW zF4bWm5s!pT{Z9$;WsVljEA;pG(;~%3a!!&by}5p;?Z@XUge}@~ytHqn_jGkVeg6D; zy2VZUdJc*6HyD}m!c>nsv{~PBo*79JF>iU>*{M&S>| zll^&v$@yD0Zrm*`E$w;pCV6;g?4&IFVXc6!k~eSqryBJ&ut&GBvhwQd>z_J(IvyK? zEVwP3H~S>GUKDTc>nO}O6A7;4(+^m*#K037IUJ8YeE4vDZ0wy{w%}>)XJ+rq1&R|D zLdD`v-(wx0o>o6|=FgTa)0jp*m!z8NFJ?`t$4}l|_ufxKFYfvCW2;xM?jJ} z&=)7dmySLQKWj-Tp7i+Ibzgb_F-iUW`M_@r3w8(&{hr(OlrCMmbbT1X)vLp)m)&mQ z!@IOFUF$b21C1%+{`|TF)Pl2(*@Z4MSDR8V;-3h-9jcwHkG_nYsQ5FjLg%bQhYn4C zy0t4_K3Ifl!v^&$S7ON;_2bj%VYO^b)^+~!f>A9+PjYr{E?%$D*?GFzc&(k^*vA() zSVgIkkB(R-fBbl1c78s=x~qg$#4Pa1lP5af4*gVpM81)N2?ftvx5^tEHP*8puKDum z6?XbNiLR|vF6oJ(k}m3qqUp_M&QtXoYY%j0nQCAjbUhZPoNgY}D>%O}H$}o8eQ+@0 z@#9~}a>9Sr?X)b2z~lJ8KG1Di8%gn|FOHS&%0F;~+I>{v399 zeUkQ(`q*>)V@Ig^awC^GC!y|M=r*(daJMFUQTB)8Eh? zPt8_US?+q6VUTe0)&?F?QO%UX=||Z6(bujq;Bf}4#793MYT4Y&HmYQ8cj^s|SBde) zypnj`fA2^~jpy2JOd-9x`SvE5^t^%ige}{*$KuPnHkHE8Be}h{LrsDKUAlQ~+YX$6 z9Bh$CL5lGW4-bEI*yM?zK}p|GbB0q%mgOt^`x4`4!iH0Gs&n$$CV$qjJg{>=puD>BkJJ}lR-rv$GxYDS^n?2ZA>&B1kd9AeY1X^L^ z0SZgXQv>zoLdorj^!gtUTNX|=>=DQg(R2K?dha_&N5?>eQg3AEprh7Lo-WoWX!MFN zKHMu)Vb`duaX~yZNYo~J&AN3J4h6?={77f*&1%#WqRC;qj+<*_TeOFGTgr_rRT_u8 z8A(V=+Qs`@>zAfeD$ibDpXwe869)qgDuRP8iw5|TGHR1pZ)MM_D{g5YqDlW ziQuB5^ulE?!F^(4E%b4|#1Olq+DF!Q&Gv>nn5jrFeX|MsBOU~DXR!0`DDqkC)Cudj4yX5J}MaoNbIXUCD=xn~z%tn3;c9i_Hc z&AQpfg_PU8_ zl@fjN)y-M|yWZX+BEgks4k3qFCtt)P+FqYn_@JW5l53uzo<#MMHZ1eqbKlZSaHh@X z{=>^yN);pR`4j7wu(a7HrfSa$(07E{?DFee%0^W0>3-+K-#WH1(;0m2+BMgJ_tHzV ziZNYV>EhzWZogI0bKMu5>vF6OVurs=ObGF*C;nQp9T9!>sDE{oG`5b2rlzJsYe;CQ zIeF~?U9yL+U%zfN_W6xMX?1R!j5k|PM7TzkT~F(sR#sMvh-;asm9ha#SicIozhViq0CN z5~I+A@O$K}{{g|`rN#W)ckc?lfB)X-9gVAJB!ARC^9={X5Wh~&1)7hKJWz!}uX}%H zZEYTzVJh}o|(3ZaMgD~25&Y)m;q zcK2;BuL_|NtWI|E2`tCpHJkSZVr>KguLR=Vi$ce#3GTOQEKsbLehui)h53~bML2)-{h_cqhAQ%d7XC^nu!NTv)bk>zj+vT3RyS1m)vRD{zq-V0f&S++&-KhKRL+`0C z^&0w>LPc(FYd35VIez?j50aTkhEe6A()+uPkbk7VeYx?}?BYq&S2x$oA3Ju<3Cq6X z`YJ{_SJwi~XJ$brwUM_UUXE8!x{6$M6-Y)tMX#`;%#TlAM<-+!0JI|Z+*7tGuN^`q z{nVvVkEKCwoB84F;Bb${W;SW}w%4Vl!3aaa+7`KQ(kl7&i`wW^OW^3VX0skF>E-u_ z{o0dKym$zS^o~K0u5)@sY^P5^d1!>)`zNQo=~)+F{TM*+a2`B}2C>mf#)ixD5D8DycKmwbT64PLT{6%44x=K95@UZny?3O|>c+dOswx$k zstavVXSb16w0ZMpgH4+@RbYcv0QU67zG{Bw`^43Fn@FaJSyRo2M9n(6kfZO}q+FX` zzj+hXoM9aN<%?D67wmh{9;do!`F*>0zZh!IZzN0G+qvlL=e05A3vagRAFh4lV^8oF zuDmB|H{eVD(JXLGBcr{?sRBe6(>XRx??}CPa__r!>|!6x+Vkq)-z?qb(f0o4Sw)c$ zafd3-{rkmldLYF+Jv(WDgp3Id?LT+=D62=UB_w1?Ab!ayv z(~9-L>Jt_gX36fk{!h<~#p6xLsB8Wf6roT#1Sh(Jro^bBz>7K7+U7j3` zp%L}&3~C`a`ntO>%UC(sr}_!GN=9&YjQ!N(6%hB@2~Gfp6#?w?^VX)@qQrXJb5K%xT31cUPspqi!C?mfCj%&h5GgNip(AESdUsIJw!Bphq{bEXXs=1Se3B@x zc=gP(bEL_iB;lB@n3$Mge%-w7l%8j;7Ue<`y;P6Er0kFeYwX>f{h!;Vs7ynz#ty<&_>Q|sms;+h6gblWzH&py$r?tHP2 z1v;W9)i1h_YggkL2e@ZmFzz~geF=ki#$T#Y#8H=O(SiLra2!1Nvi&JH$qbYezKpJ%Qw{BhCU{8{+< zX%sjaNzJXTxi-D7t$EHIc?4j7TxjkQuX6iquN~=Tbv&~s~lEk4$#3KdUqY$(O)~mwuL$&I&f03Dr%pcx;wru zT*4_m@@}ZbVbpeK>;j?@@cZ$_0M2_Hk}ZPzq$cTm|8y-|F_u;yGKJ2D;`xz$E(-E( zc|DK6o22Q7*oRjelaqXXx5P-}Ra|XOkK)W7YN{~G==ayQ;CbD)^+2&RtMFA$3MtT% zQ4hu#Qj2c#iin6b{yHx)Gt{iIjbFz~v>H#Ng!+uLaiT5aBxAtnNpqyvu9?mI6>>tt z$3FvD(>Oz=+*(YkV-V=>amHz7UCkSQ)=FKU;h{mmB&m25!liW6?-j}k6n{(aa_a@i zi0!hoVV3&(n8WqUi~QM_PJA!Gy9?xV5(esTLDZoIRA!OQmc z&j2=f-%W{WHjNIww8~<~-YA_hL4B>7byY2fKGb^A+1WX4Dc3CwRrWB|T}4@$t<$u* zx!Fm~BYhCSA2mYghK!f?UsF65&Z+b6pC4uwy!2a8<|?jRHB34$OG{<&#_Irh zSy))2KyEF3o2Yi4d@n=q>e}1?`S^>}i?8kohoWYR1zLLqXoWi*L1o!MznfYeBC3*T z!Xkqh&c?XHXTPHQZ@>LP>b-m01y6vCv~DrDaE=5=J9mf{ znUGSX6)I*&oAg*wZvj50CH3MBPLsC}cArYl=hkx>77n`W@Kf$L%|d6pB0ENt=s87pVY;&W;SUhyu58uXuAyV?S$=W4F>gz>SER(?6SuG!xgh0_5~Zc3XQ`6avsn=!5xqOB zL&ba3b)4P{b$xWfc`Q00O7_W5Tyyo%3wpfa&*1 zkg=Q>%f0mG&dNrLiy4I6hw@DYJ(Qza%-AO$i?FTuP|H+LR3Z#lex zC4T4b-8w`ezmSkbOlIXdDIuY{gA>$TykN#cY)mrYUV<&)_y~8JKU$=H@#012Z)2qb z;^$ja!hU3A`wJelMwZ^^viLoOCBH zKk?pKYO9i7bILP*txWDlK2IQO5Dy9zs{8NNt*zQ1MH>)^SU5P6D=Os7TQVtx6t*aw zQup-qG@0ys?=7xUrtIjLvuE#Kc8^Zb7M_b1m&9!k_KH~L&UcSLV@HXu0mk7^FF_K4 z-NBtnIG9CT0{eHWrpV2u^hG`=9=I!gV7A*wk&}XTUOP~qAZ+#V7$G*8)~{DnQhKm* z)hfS-4?p>fY@0SSaDUN+Ei6#*<#%KNfd$m@Sl20&eNXT69ylO}pjHDKrv9nS{XMq^ z!IECZW~cH0m_Uf>;g+oOi3waOR+2lp^;3vOD3#;!mYZ&EPrUc%8s~0ZPB$nMG-(J2 zyG)RY^TtYcs-ff|`v?^+)4FxGPB-(9mcO-ler*L46O*!$QFPJboGE2n%C{yD)ME2C zp<~(c_oWw`(hhK$RE3BF!l_4;$WtghlV3s@KThB1L;Bz1=oz1?;gwV+Rb4GPqZ85F zEITTH@GP^$musL3QKM3mf~Pmzuiq(RTkvX58C8-qP>|iXuht9LV{-^tWQIUctC1`w zK3ee9Wh6I7wP&w1p1T2QeBi6g%Rko4(F63Ege|PIU%4;Lu{ej?&WQhZR;B8T)Ww&7 zoT#=rw=09^EWlR=GOZNC%`&Z68+BG!O7`0L{Tik%4;M65`o}GAG=ofuynFn4z4>j? zl+3NSKr2NSWRKE)@ltY5rTn6y=M=@^{`Dfkx_55hey^e^$wh%W5VMtE=X~U|=2eR4 zhFY@)8p}Ivq%@56lMw+OutrZ=$XxVK;LMg-6=G8~lr1aXbu~0jzqqLDfYZ5vhq@^J z0LQo~-n};o=@?iXvVUvhKz65l9l*Xo|L2Jquwj0jm?Sw=$vjryz(e$iyMosB?i{R7 z=>6L6&Uew)HnUuYftPh&B7i>em97S2iES`?Ve2PkPNKZfn5NodDveY?Dhb>6DS@z* z2YRYQ9l}K+A=!SUP4h&OKoh2AisTg9B-SZP?PT)i30jOTm1oLlCtIA8^YxgMq4s{z zFY(B{XW|1V3a-}kLKvyGc9|Wgm#d+wNw?T4`*Q>f_mhz9AN1$EZWZ3xIX`hDxyiD+ zETt_f`-NV7YF)f;{KPtR0=}sn%U%eIc=>C|(G#X%9+@QO zU7&Kvd?|Cv$~qgs@6FJk_%!-kleoT^{ZQc3S9e~f3pD2BGjj>^e9e7!a!6N3=Hdfc+tj2ir)@K1~#wMa3 ziv=UaOYT76$L$6i>bL2;MgZBK1~-a4Ok2+M*-^2vu{tj%m5~b{atfQbe6UXC)*X_Q zqoE$of;Mra;@(tquM?&tS1gLx+;br;lXd>!OGM$iPp@tXyG)yvQk3UYpZxUGPc>%+ z3cJ7Nb9dH{Y}vLg1{mqx_NUl9eVt2-+0lyO{FauMh>nA|_lI$(dV}uF8M@31M#*|S z6Co^jtx)>rVME!0Fgf+xZ&%*j zVVRY(xqIh&4O8D|&A0EL%4~DAvg>}k(QWL_HmEpnvKUv0y9flX*451jAJa{}aJ{Zq zI$9xA>%yPBLzzR8Z2N$|3pYJ_^eBFtp5qJp*I7?po|zqOA&dj7HS_J0KreApy!jkw z<3OhZjB`;Q2^!oqU}W2}r8}qWh})cPWte2EA$veayg0UZX;Ab2a93qeN(Ux$rs5Bx7n1E`AdwCtPBY;(gB=3&dJ3VX zDs5OkT&nQ7xD{x-kK^J)N!cA#2^)`o`>#+`5Z)`S*P0!#c0Da4qy2fd^Rpv(Z%$o_ zW~ocB|6Dmb$<5sQ3HipZ{9rb>u&^3(uvXe7Ply)!L(bb~ocWay!4M2>3WVfovk)LBK1{q~mOgq@5%ft2K=9?umjk`LymUs7 zl+RYZLFTVEYkgKp2qnU;fuwG5bas9O@wDWe5w?X>^weaXqIgGdHsXK}jrKWEy=o@T z2se5A_RQXQfBf-BAU04*h=fyZ$WiN;6`|sdI+*Ba$UxD6*$SP;M*#>_kbi`(e&Ei) z_OZcgw3(YU$?#Zm7c!~kEOjwowQ^+z)UvgO6~m({8Q`!WiFmU10yR<(_yDztODW3A z9s*Z+h~@=&<@EXX{t9pu_r^;zKSN?b7nb!4Gi;QgeNKNn!fv z_3PhkzJ9qV=Fc28e6P_*dpZq&Cv#erp>?yo(V7?q8%@;>Kp46ia+yMTJ)wnV21{Q%zjTO zIx^Sakm-8Bji&TBr9G{isvLQGU#e_M>h!K%yEbpz)|+WoOB0KG%Y zGOo^C@=Fl9q66E7jO4;h*6ld-I&B0+*6wPn`S}xIW-U2`pg09VsC|S&ax)8yp`QD! zVHcfl$QguL2`1EXoB6#-tdSK-m|Bj9dB@Gmxa6 z-NRa(I=WgU`pc+E<>-=5pk2n}JSmVGrC-491)+uuT9}^+#yoD`xzp5rmcEV5MI9Ke z&D*yJ8h`uNVIAV*##+^}0(H;3mpN9Se#cu;N!~RtejH+DsPTDgD0$Dv+~uGi?|tVA z5XE2aFxXIe*tDUdV74dN2xLG{Q<}jt(F)qB2kiZQ6l(DgFDK`GW&$BRyC;jQ*X5Kn zwCoe+w*rdlCdegEy#YM!4tG}Gad(wt|P zK=uYgX@b-EyZ!7_omQ{b?-LgnS7eu7nDiL~VQhAG^FFy>J-&_I5_Ox)6m=SpY%X$J zrltGqV(#OLiivi0Ux&19(~?Obg!u=#D@2r(bOBcE*Nx)!1|bgKhN5x%_U#_vO(Os` zqqZDtlma3OonzX84(!_|b*);>vv*LULEaf5UG#uh^&YNiUV7(;yicSw@G|*bM=aU` zN5%Sv*k`zle@eP0*i$XNbXZDCDtAMP?t-I}Q|`C%BFr!0kn6^%Bi*0~<@4=_brhb0 z<1_*am>v0HdZ^g|dP#`4hsnHpvQB@x8TN7|V5#m%Ahk%Yz5af2aWQHpi--UR#J)95 zOknjOPB2M!V87kNUp~A%H$5C|+JuxQ4>-VNmLSyWHWRv6DQC{wzUB+!axiWuesFYf z(A20ZBxv-w%()HM;KS$&bTF1)`dTW@v`(bAI{e5g#tjFocF?yOA${6pJ;hGpFV7Rt z+&aDA*ijToKJMJpn-4FG+4eoyU&Cr3Co6krSqHEc4BW@XMQ070@c0U@Ej&lI(s=4JVGE~q;D-=NB3&D`=M~SKITL2xzESJ$xTNNTC&_o;WLbTB zsWUbE^=lxOe*k^BoUb?>*e3Aa&LdlM4~pJ6chQ5IyqVC`-}N(u#88DUaI4;%Uk$fr zS7+co$w*GtoaB#bP1F>!`TTlKz(o3$_j>|5TSYxktyf}!*xb|RIBAON-wA{t+eOLZ zaBx(;J384T?2wYZeO}GF>2KrgU9D&ia@EmG!D2@m5gWQJ> zDWg!WL-hLqbvU4_2oo^}#_TWxCLo+Fa?$wsxH4)PxOb8eRrNsS)P*{91X;S+JOU&B zbX)7oHdeE3fC7%jLy+@)2i>S}2@r<@AVxgIic{n!a0h_e!g}Oeze}p^y~FDUl`+SG*FklAs=~F1x zxhD%)B4N#Gf#g7LS(>*4RbQ(`zC4g-Es=IFup^~|pm~Hn5l2i6_z54B0>9t5VFw~? z5YJo}B5K1ndfAR-uUtKI1Nw%*^o)!ts9Zd#VNrIIuXi=ahpb$=a{*87+uq1c9y&F*cC4;$*C;p1bNmW!9L>7mCN<-gu%o9ORX-}O% z8GuV0xcpo(|6CA^C>87i=*4pbd<6dG!JqR1wL458Wp#BbcmZ|L9#f8VIG#Lr?%V)^ zugT%gPZ0aag}(m&J&s{*K~Vj~!+bh1qk2wsmH<08Qu|d7)gBa;Ai4+zzBqen&+YG!_^@R1D>GaT-be>eg+osd$QM_vLsQpjpb$N>)zWDNoH?LoJ54?>l zfYX8SZ_}flDP&QhLO+M)*4^J92M2|)Wrq$?ZwR9T!0vZur}Oks9KAeX5DdEts`k3k zi;sU1G2nfBw-R3W7^5%NKYPEjb7@h@L$ zph}eIi83%e-t;k=%5aQuRK!SA2)dreFU82VV~1u*(nk4H zIry{ze!a42olXumR5CyvXHeU#V`+BN>0I!=WOq0xoS%Z2rvowR90*O=S?ZBQ@9hve z4Q9yVswji6bVNi1nZQhl*}(h0b}%d+<2sj|lDP;>Q;(?CuGI^~VM!Dp%q*dRzz37L zGZ|`1dnD=dtRY1|rKUzHTVXRpVq&_(p<=-Jbc>Zr{ot5l6N4EZuk#z^OR5Y$(n#Dk zq?!a#S$~pQQZ;c7rDWVP07uxH!iv9&S33vR*Y3;5_yp>pSlCc zvqta`+!WTMRu{rPk$3=D!$kqYIavRU7$c7vp1QIS4g!Gug%4*IiUyM%^SuDU2eRzH zUvL^1K&z%GV^wWIY|4}aH*Mp$Y^U^1q%O!Mh;!}Vf0l;*tm6+Ghc73H(q}bOJu_N= zc9q^Qhno)H z_Zau8R1H`==tfBv(&Ee2fy$PUp#arLfUt#`Sz=4Jr2vJ5K@jY2&BH%2GY150)4zcS;NmIHS6qAJY1R^< zDLApNC_tSG;imU)-+rgI^*McPe4KYWAb_(zS-17z6u$91ukX)(4-#y!fzQyORrF+f zEli+__0`HJbMpAm-poai;`a%uMyW+DG_g+F^=qn#MH@GTn9pRh%*l01nreA9bKoKq zpc^?u?wVM)Q~0Vc*rI5;peIDy9-D>EvP^^V@ts92pN&|n^+=vLeXq$T=t$S)8I*32 z1kL7M%*-LdO(D$o)`MEzUtLGqa=ojxU}PkO_?ItVB&M=V>P{1dHq2#2bgI8j1}|L& z;$TR&Uzj~ioxZAX5)|K}s z$i1GKH9z_AL6wAQz;*af3cE9J>?7Yl_AYgYhCt`>+g*M~j=g7x)00Fi+cX(VXJ!mP z_1Wj1FsfS=@$O!m-(leE=f{<7`J&1w;~gTgNO01J82M#`?=J>kfJuHd+7qntjprZU z0-djRKcG2_jUzdpk z%I-}Clu!fzHa<0V&`10DMfsBLQBhIEzy(LdIp8fwA4xSeHJidkmd(YnuZOZ%pCyhc zmaSXWOAh|7M&JQA_Zf)UVz?m&s0Od$Ch6RBt6u%?VKy`(HpUENMa?6tww_-8JEeU9 zuulO}Vj)xfppo<2_fH3gu1{g}z+SWE1q=Uo=d*8o$o2Z+c2ygWa&Oz_GQD4uu}0mCfOc=M#b4Vuly5e0RA@(syg2X61AuJ`Be?Q z-%UYt6O--iM6Wn|k#*N;z(7T0X+FBYn#A`@xsrdyuS)#TxclVIGgLQWJ9zGL%RY_& zHKy1nKd6f*Fg9Xw#)5j%%Cn6_?)Eo5kJ%=8523q{XwsT8gna!re%5ZZLl=gQ03bNl z!&e^f4|6;rC@82<8U$R$;W8vh^5E{>y9reR=i1@RZ+|{@>XazEF;o{s`A6UYo;-cJ z3{oT{bbw>8LR=$NAzrlZ;!Ey}e~r3_x-ucO{>I>g-6wrRLzA#xdgi_X7-?W!Kdxux zWM=Mt$cKf2a-;#NF3WA+p3o)(FzvXyxlIAF#9>{3H% z8DMox$lXx8#2J?Ro4{a7 zm>kHdxJC3O0Uz42NY<)6tGHCP#Vra(CpQaHy62xsR%3GpO){RY%v?q`W zA-WT|y<`ALI;#{6m?Zqn&x|$_NNn0fOs_Dg>4d|VNn#m**Hgl04a2?CZ<_)kt`PMw zFrKYzLxGbGsXS@12kaX3p1E@Tfk73a5brXrf49 zgPpK6*H}!XI^fcG)zyl%Z;Z{LLW>7nZ%~U-QvV*DlrG)b ztdqoXe)vabjzj{6*KXtYRsE8$i3**MGSHTrA%)j-La`V7B!ClRKL<^6Y~0rp3I!^bzBkGh-P-a%~NC~Zk$J2((4l-)+X%V`30<*7Wb8RSS> zBybmf$24^RE7%sEmbRkb)&%Ex0BJuaaSb0qzk6%c|)k= zM$#v=2g&KE5#vEiA1Wd38ZCt`*#yx#jOt;=C^S-h5-o{T7)T;fgt>;rRtr`tc$#DL zNAk`RDApxvBc1$Lh@I87PC9U1jRs1d`1rwv>S9ix!tn6~x&10cWHE5?9F8AXR65PBnEKU*P##$gd3ef@Xb=QeKKNQ(sH;%|NS+ISaO z7VoOH5S-asH!?8fNbK0LgUG?0aj;yY$mI9yA8F3m-^kKfw&i?z{*z>(H z4N�xl;E?OVp9&{aCK)M#6%_^BpJn@L$&sojiZ)`1g7HFZ*m_Ujx2$IT*<8c=iMR zRMiw`@0^PA^7=i-)zjb4!+ig4+D8N-!!}l5#_6fo2$0C4TvC$DKc#dVF<|wNKgtqm z!7nsa7s=KSP?Aqj@Ei>}|2(gTmXjhGl+wF@DP>4XgwKaL8_g2Msw>x&F&cQ04ij*5 zrGD@&CHR+Rt|2?<7}qwyhSbzwLnWM)!Nl~6T-?G<2x!lu$nSY77E2Hw{`bf-KoT#_ zU%q{X0YW5AQt$urX_kZ+&c*(MTJGX%sz-XWR_6=zC!BZGg%a1MthGYErKu_3qId_fg8kr0i3Y={Q7n|#}R&h z1)6Gxk-U?W?&5A+xUthp7Y@OB$NRdRDN}FhAr!tpMoVUARs{7qC81AUnF~GWt3Oz4T5d{ z-)_AkKl#77yX*f;Kf$suv+9xXS!fp;5Q0#{eMmPvixlfp_Vz6|RTsV@XY_}CfVpEe zBV#%I0C2UFWSDv3#qXQ0NRp{|z9S{w;+nV@o2WACJuHF%!ZDytB}pe&8SL^CNWifW z2GEC7tYn9_tysb_Wf2U=UmorxU>{Ka|r!r#M0 zLU7mu8ePD@)xtJ9iC!Duk2=e%Gm`0NEvUVG6i}MJP^!AVN*q3{8ty(9$Jvhpy#ZUB zNv#vkVlE1K6w-)fdOlprjdY+bJDBdY+mlH+wpeC&MG|x@+U0ovs<(CXY|mkD`T6@t z14e9%SCE!2KH;}^{rZEhCzpRbAd1vvHoXcIQZ3ZOxE*5%iY_5AaQkput_t`ckLz|F z)bfLbx9f15(ZyAKjr9!#lz@I>LL(VKT)WU_41n7g_KZ*zbRS}N?%X*zIa9lpgM(in zh3fWogN{G7ut7#f<}3|LpRRu#AXjqwYA0}7S%&3X4{E0Wh8q(GfSp|s?3!+O+STzhz9adeS3TS65C7ywzwotWqgyY-~((gsHeS{c)j2+gz_z3q=6m1Bn6TV~L8?!k3k&*E5_F9BuWf!l8_j`xIxbM4MHf3dH z+mSX=JW8S_>QZ_1M97{wW2IG6#NYI$v{bx7MjS>r@|?Pl_vVFkiVRoTiS~-Wm-!#8pd|GjvU(X_skdO-YbmQW}e5kj+7WfQ(qdK$^ra@qMig``-z_t0~ zsUCO@{&%~#!h3)#(cfU zh$SBZMIpXom8mAd#pRo2kDW3Ss`~6KDri4+o{(;28DVdR`tJZgiMF8x(vyTO$&*^u z)O4Qo_054moq}~|034VGVHb?6IS3a>3O=o8W~5jWtvamFKd;sV*s?W&;>&sufZI#}Qk`LFnuMQ5Tnde8p(r^c!MP|v&}d-X<@%VQukA^A+~@RI zXn=8y*?6RYfk}pM;Z!%5)P?fB`}UO>?wC(~>G~dB7ILVAd^lr5MVwq*5}?(5FFn$- zNQ}7lgtsvQG;af%^`3l=e;S3-mZF^OkdsFRA}JkBPOfO#HEln|AV#i!?uLkR@6SJ# zpwe(s&@oaii|r|wYkxAD82xvpUWiOmfc$7-bLk-F~=3!ksAZzQ(zVC9j{3gIez9TgsZ za^WnoNWv_c2~@1>HdQ}R6LI6j$&&;m6Js4M60@uX+tabQid|&%sz!9FOxapgA@%6S zB$a1Nj`dUQ_z$t?_LAmJ(%;zi=C$X0RCfzlILk#WA~pqKO$~N^G}%sO7ca<@hDk^^ zvR2=F!R_gDl1X&-QIpkoyk|039;2(G?S^ln;OQl9ixHlg>SX}Vz{~$Xjkx|xNcP{q zV4QdGM+WZjzc8J0IN>PkI?rl@z%MBH%7i@0ZNg@~f;$~KfKVOakqukC(M%Z7oEeNN z1uZ_I9|6^sUIN)2dq~m**|&G^+~M`%tHh-Htq=H#VGGu-wknpJa3- zfgGQ{0>aL)eH|){mqm=+;vkpkWnfPUrkLt`f8z!t3eF3it9%ch;JZeBXJlr$MtfbvnEE5uZMlto&4tnzs~8@0rjD*iyivpSymnmt zN@~}~6$~F0hFlm-@$ym=U$Px;vi`!rV8u8sd2I?OB%D`1w}Ex&s3QZz7RGsw=Z{cq z9O$L~^^wj7TxExX=e(@-zph%1t6Ioauh<#a+p|e9FfiS4W4_?_ohDm8RN;JyaHGys zV7oX}zHgEtfombP`4J-INai{QrE_T*AsmL!mD{YpJU@;xb2rT}G=RUnAo@ns@#Eta z3|6A#6ZCdmpaj@_zvd|U1Z?I7%~Rj6`M3d(HAo&S^Cg>vRSlkMKgOFyo=VT1k(mw3 zlIFGXItE1@`cfKXhJAy@zKLo7%a-BqiWn3N)4HDA~EgFHGa_pbiZW4;PgXzMSDsBDBha z{}kRAO~G_)0*eDJ5>zh{pokw3>UKBU)BnX2g&=B+vam5|*jS-@T1`wSq@uD;NyISunc-q?9TA`fu9W^d}0WNhaYcdld7@LOm z8UgAi#*VJD0p1}6{ad~^qB`ghqdRZjwCQt>TU$L94a+K4R;ipkuml3kDh$F@?uS3Zk_BaOO z-FBbe36vvTT&GY-*|a2M@s=57@bK~RH9iYe!8?psjob5XDsKqNJc`XTWgfbqxZ|M< zaOt*9XQZd2`WPD{+CAq&XI_X+M^{%Dd<`ml2B{ab`#&d_pK6b8{}_RPc~vD{uX zyT4X1;BLNruTFsO!CN7NRQK`-LQ0M2IbQi_*2$UXk#4YLd(LDeoZtN9#YXy_RF6E{ z&SlTpQun&~n7d8Vx~l&3G1xXvjyj7zarC($b%*lYO<;Xx0M*ZdR|Itx3t(^+-0Rhj zf?T}I!>A+p*hA=3Q=+55t^#c3}8rW3Gcy=%0>2w{t8YFeXC1Xo?`U zt2ix$lbn~)Xqr;!A5#JDHWB_PK3HdNG4JA6pC8t^2&AZ$=zOi#9w*k6Ce1*_ne8~I z^j@{ciSb%GAe+bEkgZWe3vNERz)J@2xCIuP9iIXlNFgSiQ2#U!j-5NT|L|Ylhx{_w zHIPvJZO7a_G}sRUcFWVKg|F)UKvD>s)c!`BFrKEmD=@z#Poy;dU*YapvC-UK4oWTj z+@Txv(T>a+hgcaIR%*y!Yr3|t9qw;KM1yD|O5tKAM(V4$RkvlJn+Uyf`lip4}Mt=qU|Em6?+Lf)BQA< z+~`}$&sfctxn1##B-##IV2G>&1v(a71CZVcXdrc9LD)y{@+Ni^x|qTg0-@~zjYc~R zQz*MA^*5LJ@C=cJn~$ZC|xU7{kSE52%UJsj$b~ymg~UD1J&*7 z*iIr&Gvt`De3m6VEyLqW|F>zgja4fnBcolDefc_?jp>ynurVF(^X9-tWuf)U@0yn< z`(^{W=lkK$R71xMY0*)jNg9%fat>b%YL*04cBF-bxOS>2 z%c+BRMfOc~mt*kX#FEk##|mAA$Ab0Hpyv`aAzwMRTKP#z0s74eYmbKUCs;w`SPzJB zwP^9Qg%vgq$qf?2ros)&;X8!aZzhuiq?W$>Y!oElgB0*2Xm=s*66kWa?RJg*p!B2O zZ&|*-sPdBtJ9My;`;$dN%tVNNFOaK$&qOC!whAjcmqU$ zB`PtzkzRsiDUl@(SxX=EjzTF|79|9hDeV88jO8$Ljp=z(1mUX*osI-6eChR1Frz$E z>arVdQ7hT*AhOH_Dy_ilE5e&{hEoZ?9QDXCqBd*JP`b3K{qB z+jkZEGiiw^KSX*vP`>#2`gT`QmS>jt7mOIoJ^^yvoqPA}H1~n!#*44Pmi_$CYp@p* z$k7}2&j}$ae5K|X+(*>j$mIpY@<99KAbN2>fH_-Pj=l#~BETk}Od7BhLjyxg+&2>8RzA7wd4hpQ~OG+zpg){B@YL25KtHp?Q{WakP zCKe*p-8nY3QUhEX%S}4|0M)9EE(&z;)U_h0la)+eDeE`}UA1jC8lP}4amwW{a@BECk#8a& zPX2S^2S!H)1X7k<qv~b1msXQ6}s-z`Xz^4WhG2gs0~GP0*^VMK$h125QLYX-w5fgRjxCM>%Y za!3|BmVYG{AT&LbtxsI;Fu3%QMvZ@Oh0eN`6n$yK%a^<7hCn+lpIk-2I+WWy>|cT{ zK`m7L^LMHsO_9?G>TIedHEMm9gOTd57KM_I5bK>kd2VadANB#%)WMKYLEMGQHvkxs*`y*s8o}iwsr!9s_n~ zPjg5h;*la|Zf-Dz*z*QTc3P@}bEA)rhQ}o0XgfgQ&sFcfe_=a<2K4&&F*(x{Cw?au z?ijC1BOk$=-=#Zl5)R-TmX(n^#-+Uf?Y`;{n`?v0GTL3D!<0gQ3HIg+PPbrUP?bed zSypP~b#Yi=??3g~Cxqw(z~`ILveG;fRngROl;{Gb_Ujn9HHTc9W*&dzwHUEBm60QJ z!jR*mZOQzjQtofx_%y+Tog4nrV%YAhyxC=sIwGIIpb^8ds5eE7v94ICOS9D;Tk{UE z3$CDdrd@uTc+D#AYA8+#jC&_5BBAmFgW;cU%u2JqXGULCarjg=AK(0!D#`Q;u?98* zn>i==l{V~>Ns2xvnFxWZL#td!q@4fzr4Xo#I)xh%1`{@Kta;zi&dhFuAPMf4R zEF1O8VNQb^!dBA=&I~wGKA^s9|3_qnlv^Zc? zUcz7pVR4za<(z|{X7%rFd~y)U*bkKux<~sldZ(M=N3Bl~Yas|(VjcfGzBuJ(q!U8D z%$J9Br=!A[Y3#8z*c=0gOqL9}ZSC~$M#j(=uVb>sme&N+8KlsW7$qC-x7geQx} zY2d}#?=3!3gJa0x@NM`zSMM)H{v@7P(vGrh#{np!ZP~FS4ggpaP8Qo4&hHsn^|tOv z!E}qM9aY#yK*r$^1=nc?;Kk+@gL))jnmy>{l`ZNv#=|lHyqlUg)6kO(Npvlhxsx6-^ zL5@Vo0%3vU=uF`P{O6uYf_6tT1vyNCw7CO^1exXto<_{lLK`GG+Trh?0dl60<0vpy zF;GoU5E&MA9v21O3?%PNH5uIJ`Q?}A?LNz>PW6}ta}tdl?ua84UiKe+zGlsu)7gZdm^TkOSY5K8UT!GM5;k6UTq*b!?1R#Z2PkL3Hm~&s_e#gdVh> z7{p_sO5`MT3T6~8Z1m<2ps5L7ehs2BlXk(zZZZZ2zhh}9OfOw}QnWCwE&u57yZ3Q4 z!PA6Kfn$)|o6w-dQ_fcS({i{}I{$_#3F}Jz^y!;kf`s!T_BNOzV^PD5TwNAK9&h4P zzao$0G2mszQ4{1yIcP|gy2HyE@)*aP9pt1fIFNw;iIWgCYZOXvZk*_(JG}fWs#w=a zM9@MRPqH^TKN6{{KBHPHfuIK*$yyIhoh>DI`Jo@j-eJ+=iUVjP7T+x}9K!Sz3#StC zlP79se27upg}PM{2+u}Fd-*GVa%m?MLcs*V1xb!T`EWL4%n4C{Tx~nfsPO8+Xi| zsj!A}a+YI9Kk+d!a4W^>njADikG3-ETb50mo?N@ekY(1aOpZK2z`LV6Lo9;JHg#K6 zKY0<~b-kCdKS#h+^pNnJIBx4P7R(@$N&_0KC&?ke0I&%4-wYc=0|R4$JHQL&roCC7 zgO}Vh5q2E~POQ@I1@J>;IZYT5J)U?kQFhg%-6ToBSPy5Al^Hg|6mCVlNH9v|?9~4L z3|`b-|CxWGDAC~+Agcy8IDu6$7+HSIi{)QF!Pgiug{GAjH1~0ue0+Z0>83+}?I|oR zBBGHqS|js7K}{hvClI$c{E7Yk)5~k_PBfTAyii#hwtX^Q`O$+oFRoR`a4qS?RsvhQ zPyP`n+zy|Gmy1y=&9ZL4BO~ZeK1jhiIkuY7@LcAWZ7sneA(QB?y@FT+qoBl!SJ zOXU?6w%^=ww1jfFJsP|?$+5|B)wiGQYieqO*mOY{Zu5Qr=bi&4yD%gtMad~&WS?$F zy=O;Js{dDHW$hb*y8k(@vcOa9KXR3#pkI$BCH#x6EJ@p&vwRrP0<(Zl&I5uARj@L- zD6rVc%E{SkX;jf4z`rnQcjmiqK_~eLaVJkt`*%Bwex04=A$Cq!(&JIQ#MqIBBOoB0 zadYSXeFv?5h@5F*;i5LDL83a90X1{OvO*(=AOG95zyZ^OZSoBUakRsMS~^1m=pSur z7z)V}B86U{M0l^x26QiwtWszHAKVK2;JQG6$-{1E-Dzw!h_0OUVt&*&YOqGrB0(5~ zwWjl6bv#7{K$$CsEujF%!jzYm#{AYm%m{CE1McR@=iBJoXR%R}%V*8&M$QKeQ!VPE z3i-D={M)zi&Hp5^P%w7=AKVKj%Lh1sIqS>xF8c+iPQy~?G_6<#^77Axx>JU^|FkWf zG^K3E0kLoZE*~8rO`sUJDPWC#$j>WzHwRm9Z7jb{x`l{pjo3&Z531j|_w*GUjzI`C ztSvuFH*!`rVG_VwynH{A+79}2>h0e9MBUcH;ZNEvmc>lkE!jB?Z|wfBcuB$=9qj=8 z5~$ax1=?F?;g>;T2~d9*}Ky!fOIevHlPz-zxK^48I7S=LxvMVz&n&$@ci|F zbULpTLXPgHw4iO)pb0wzp-V6GGy?eJw;+k@5jSx1Ga6DV*PMOq>o-_~jf+D^!ewv{ zsC~7tQo>=4u-!an z;ZrA8+W$yh-f49qIpz81vIReN3JHwFV~(FB2YKVbpfv1iFwjR~FRL9Z;wIxeHmjVh z@1)W(R<_eu!*4cAz|W~;#pCqP`MbEoFz-y(%bg<(N`l{goBzKj@qfp8@CR9pH|Y}; z)cr?!RWsxUvkHLJB2OhE*wJhKeL*NQNb$T9Sm4lsQxQDhXMJh)PH(qL!gS znHpqDDYPPyp%zK|^~65=ch1@S?6a@yjMugMYpqu6`~3{>_w#<9`?>G?ImB57F-|I8 ze&-2+)oL!-|9y~N4Ku2wqbs)jw+gR+-lOcam_q-i!*`w}O%af<`cEorExGtV@*HgU+%V#k8`o)P)>FcJb~oQc% zHsNpyRs!hmLT_B!WTG+^Po=w`@F80Ih2|NZ%2QOSuMjt9(T#|uOq10qprin1YnIeP zT&H`JPoh*+hS7>Jj8>Xk11Sut{y51jQ@U+r->s(#MVHg!81@xoIMCxah(1qJTn#5O zvL`)i!HlB^$?@1+!jMyjVPpHU{Zq_l$(E;(c8G7=nebdWfusw{waqI1A`edSxp(fJ zSh}~9lmcagktaEY@MT~OqfwP+szl&Z<#W;p`d{rLbr0&3@?~%OrKhgT6PSVF ziL(OWVS0H5y*@Qu>V#AFT~1>2^Iy?dgVR-u;!OS)qtGQ)r+c$WbvIH|ivbu=(aruD zxc(#BQ-;y;7K2i!&a@_#9(wh~Sj0&x^f6;>P$g5-+?kY&N>$~0R%WLCQ_C4A?FNEp zj#Z)^K~oSKE>1DxDrqLU679GB_vz;}#}fXg>Jb7|xFe0-!@PO3Ox_tj8 z&@zKWjgkK0tc&it8>LBo!IP{@{K@#s(EAh^>ZE68(ZOrKYv?u2=+FQ?|07ZL&HE;+ zXivjb)i4vip`D%G0euQ2X<-Y)wzrm6n$rw)^=R96iA|$jNJt3naqnBZb)Qb#v44MQ zY&1`K@Q7KY(p&YsZa)7nd0#-6p=EJP4jD-EA*$SA&K%qg4*f{(+Lw{t{pE2oww7D^ zucUMK!Ao4TMokM2rA>8=`$gIixMN2_?6@DD?h}7M_5dmGT%}RVUbTO)@X^C7W^-!h zrhF?oy!pJZd-AuTv0KPzqg-fXJ56~V1Gq(Bshd`fkX&zl+f+Si)We~uInu8cyL3NVMQ^#8mi9by``u?+UfRD_#@H;N27``!ADFH4vo^Aq ztc++ypIiQ1$%Bpg7rJQMWK?z@xQRe6un@ykY|HKz4*cGPnM(2&HHzC` zLF-PLoQfj^d7qZYWgy9Hh>^n5s6qDrr7FLs?EIiwq=2oqk(|q9MXZJs?kfWH6n$Vx z2&+UHMn~k)z0T(~M0!NL!x8WD`|Jyz<#VL7$FdW+pm9{~m2bGYbj&!V2ik#GA@dL1 zgLN-mYjS;yI(6!ZClj4xnYJNXz4&CAGePs~eS)uuyJf}%Y!Unm50S=8)@K=BB}TeW zC)CEcUhV&VhUo7H+6}wp4i!74sRxvTh*MX-+;n^>=Fs2adCe&M#PD+oMJ6Y?-1{ls zcB>1u{-39T{+q3Fm~8U%@2i9U|Nex8n-`@r7-sgb9~X-ow`kp9x}09x37FR&2V}Sr z9S(keqr*^+AC8C{u8pih+`RkA!%B*Y%LMiDYtk2?BQz6~|NlsOz5nJi*AEHATyQrs-_`%xpeI(7WwB@^PW{7)x^orkWnN60P?lszo>AE)WD z0M5=AaQ#o%6MpVT$mKW_&6OZjqCzzo(vtQ);auD73-}&3!674Z zD*veYI!5{n!d!voU^t|bmX&W!RV^itreY2bn0W?}!10(r0uq3u% zkjAZ${>AS5zN40F+&kJWFMFb8#_XSq@Q6qoXv8fEx;4SwJw^=3ToMYiYoC9wEBxt^ zBj6774s_GST4~C|ubqrMQp+zz8f~%uj*$N?ks(-=_ey|;f*46cG)j=#u8;1U$Y1>t zdO2KYJ`h3=?9*h7k=F*u0zwy4>LoiCPf$~lWkVh z28Qhr?Rz}Zy&|aR&;~h0Wf#r8SH$L3FAd5)pOTzxt^~(NUN?zzY|auYO^#b>YxJVFQR?MU0qz zX@C{R-Tn0NbrjgvnOA3vI!F0cl?ON|hB*mSJk-5voC_^%O*2BAZu*0b57S?UV}BQr zza8@KkDUF+6xWe1Eg zf&QqkIJZ*@fP?GrwQ?_-=Lc54H#3voRp{%_BnAJ# zj2j)$=>&461(=0?0ZH%l$9()>0G(_gltPIKM^SVlY#ykwLa}4*t+-byar;$Ku^>+A zN@o4vryFb(QN9Pf#~w7N`*!2QRDmy6{2G}@Tf+m#3)q?#?A6O+aRh4c3SO`^L{~{7 znn$s-h4sZERUPq{~5BL_AF z#DiIT@!2L-{>YdfCwdI^C(EVx8ujAa5Bgi7Hgd<>ht;Otf4K7feo#4)GY2pIMbX+1 zlq3TOi^Z_wd)ZR8iI(2j_}g2z2da}c^r>6WOKxmcKpOUnu_Au5CCeR6=5|Ph+8Ixcp8cWL4Y-$SZC-hyr zDGnYsv#{XhR)+}(4O3Zh5|ZK&Da2&yO3}}ZUHRFazQ?-ipJxvj$RN3Oh<_fn+NUP|HreN*9u9o=>3@NQhVMxJ+5(_smN;}6#`(T~ym}jrB zq(L8NJA8G2U`N1jA;WQl$HcWjgrsONVga;AtyK=i|ankREjh+K^TOT zBT6OAr6xpM>hW6qBYh?i#HkQI!aqb~H`*C{v#ojoKFZoC5+2P!2D3~CB}UT=fTtfg z??au|e%(hZT+d6Jw~7pj*7WW{fLFa$)xK^Z2k zJ`CiJ=6ER9BjP9+M0k;&8BU9_NOYlPOsCVeSqv*hZh+_B4vmyP(<;tBA$NQA^ztPQ zCnh^7ulKpC@Ef7EApb0t4Yu1CXRhdYhUyVO@DN~xg|dt$s`a9werw9d^;TR+MdB!A z4z@IeD5+hLLtW;kvw<&!M=NH_S&Jj7;yED%KoE^FowB7+Dzyd=-uJGF!tF%Assp{F z!4nYZXmdMi-Kpbu6+81@tx))%J=^)t*9XhLNDq^Uz*gwNu#jw# z6v}7>YF7E(#i%)UT#2}x@l~IId;`Ia* zyy$pGzol+J)f;x`dFjG?sDh|&IKsw+o3R0N*kC6)w9}~afIX=AgCP&SHYkV-tWG)z z<1=9Tx?q+%=6dP$k|@@W_FqfIpmoA$d!B4SeC7o)gQS=-aJ=IFpFxuqy3*w-aV&?? zCu>2R zM=*jBz{?aXD=Vj4Mv8-51MgqyiDVVCs1OaOy9bBU2@m|}>!Ur_(ed7{ezR4G2?a`C zu`Jcgz1i)-&X3QZeCk=${N&U#Bg9zq8l3R?l2i3_5m$787EOp03851+PeoVa33 zaa?vN4Ro3BDb!2x5_kW+e@FGWWk7`4T;}J`pGy#_XOHRP1H+-2StfRLM{u-BaiO|t zGs*0&cWEN_+~bvUj+CH>_(C)ij~{s&yq_SAPj4RonqNGpPGE}>y{$GOTfnj~%W)%R z(SFg(x1s%l0$Q*|u{d||gwZ17Qbk`WcTRU5Qt?Q$OlZ_%TD<_{5ux=IB@MSeP$DQ3 z%-75q$MjgYXCuT7TtF?TVUb+14IV_&Gc;eIh4yM!n+g+A_sdFVxh!OorJTLox~Q^*uE0V0Rr#Im|TfN5Y0Jf+WW$tl(;AF5gk#$^uRhI(X&c> zF!+K2*q|^WCFPKYme4rD53DffdAk|cd(K86wut$ptZXsQ-_oel%$e;LWgiC>5}GRX z3MWphfX)fb3g)0b$XjwDxCBf;2{J;*&Vv&w_1>{7K2;HjDzNzs60JHsj#=SwAB8S zXaKkLJ~^zxuKcRMG$bLSa~~+oLP{eS1Sfp^AL1G(|4M~WJ2aROW}y`C9oA(pQhrGx zNE$mW4XMeShpu4C34nQ-8%$rdSXHf4C;6~QE=E-Th`qW|16+pVE2&t~*7(c1RUz*1 zY*%`tMVM;g^|FPFX$7YAUrmk{k=Kh1Ht=DeC7t$LkLJ z3_Em~(=+V%4Cj2ytvc+p@%Q}CY6#k8a+S=tkX=r%f@#+GSPKWKYK@Zf#l-{95P&>L z3XrZQiMOe284@sYeq`lYV~Z@xR%q51%G#ryz$`|*z^X!`5wD#1M{S|+Nmzt4zuDh^ zM_^^npo)r@34K%*vyojY0tfEXOBh-Av{6xE^|57QZ3I_3n>!OHTUh|;+;fl$N<)Qp zxm)Ap%*%F!0UL^BQ7ML2)V-o35+? zf-OyeIIprg3dhwxEftXkfxlRd&Zsd`lRAWXdi~{m;TR3*|0muJ0aNNfXsL>q(g3D+ zzj=68>=Lt1)+AmYye&GysK(%%Nt))8wi>oz*eFaZbm(ZEe$DbJ)7sT1&g? z=xp|Ia^CrubAxd~oA0YT=-gshUzenQ zTGOOWLbt>_c3wMiXPlp(kov*6xU9OgPQAZDV5gp$(|~lfyQb!P@*UI?y=5RL8Ja1} zH`D-|c9l+@GJpdZKkh(c#=x-5%ihZ`Rr?09aTddGAO&nXj4leER`m=G4LJ^jh`ZZO zlkePV8JvD}VO>+xgupr5{jz&p1C-E%0c=xuJJ#^Qd%@sB zp#ELDj5nKjsJBNB_)FWi^YBbu*rg*l^BJA=t(^zo$2`FP!z!3p`UKlS6fG4laG(Yy ze#to%{pjbBrl!sfZ>Xu9aI%Tr{Iz~Wt96BibFx@<3~T>DzsE>ZpzdB{FK?s(Q=|I~%U&Wtg2);{Uf=JTdF`{>BBMpFik zm-?9rn{Fwcm*2(Ih6%>EXMKC$u1fS+{jl(-DoKM&{3l4bn^CCiq$5;;swOK2ut=Y? z`eJAOL6?C`JQ^Dt$nDVLb0NM%HB`g;4ywgE;<+9yL=FSvf}}k-x#9%>iopP z_GRx>S@tNU6jPDxRj*juee_nvZ1Ng~YgINTVy#qEhQm906Lu&jCi6-AAUo z7)Bz`WLeJZp)k-DI`(ltCplc6U~k`v`V4exBcW>MlKr;3bUU@!*orUgU|IWxd)0K_ z4|R{XpSpW#&aGS~UX65j*Qe3@j3&rjvX%^HU}2c!?Q9$LM|J7^PR5=)5HByGg==kR zXFhD0f&t8J6F=Wt6i1#umE+ttcAWjqcgeD(h4m^T9Bg^CW6jO4K6Bf#Yu7ch8VhGw zPC4M&m&D|XZW2i^M!EB$Dsw@17k08OW&yYLW0NU7#}!k4b>Y(MS!H%rw;rig;Xiia z(ZjaS8xklcthY;CW#aqywJx4m5|N&t8LV^lh3*y(kHe_au@T__Z>fM&dRnI`Ib@X) z183{ZM!v=c6c!h6%qr1f0U$9faF3tL?D5A25y~wT9d&|p`>d{dbVz%zO*eJxv;Xp? z`-l1-py*#q-hVIA*LPmVp5ZH27_rO>X|&HL7q0N82DDtc(pbScjXRk|g zVme|(T^x;;6>W~t&@eP??+1w~PP2#YrjeKsLjFG8agP1A1MXY=vwvryT2DW*piO)W z+X-7SrU}t^Y(hn4OufJiInXp({m8anjZT4nM}#} z^Gf~&q;aY|W`}9JF@5|sS#{Fn$$;Q5shI|WB=+Zt#6P>!OI`ixC|2w9s*-(^ch*et z*>;DNI~x0HP#cC~yr(izL7=lNDmFITRauTIN=Aq%$U)8L%u$CDdoFyD&!uyTyu5z> zc5uUW`yq!YxS1URUw0F{i7dLDHJ^@<4g_13s;GQkrEPa~GP^=;l5p@-ewkn_g{PUh zxww`WP@1_GD@yYdyQIhbl@;F7&9DHh@E$Za#ObLS8DSLc9&a86QGC~hfM&wf*v2hd z%p+A|_DZX`6>B$dR9oRs8y~S`Gke$L6MdTHKTVqPF}% zNMBTm$!}&Ne16Gj%ioZz=uAEB2XSw^$s1GC=M~|+PIsS=T`XL^r+6FL6JGoJezw5$ znkjz+IaILq12G%gipO&ufOX|R4aGeHS2Mh=#p+TX41NA~`TH2xIZ=b>< z?j2tq}4H((nZHE)*qz}$Ngwx z(BIJCsSFx$5{M^%{e}%!JP|szF^f~iwHkD4K{LOEl!NfX0#pErGDGFsqp}%kCA{Ac4c2y2Hk6tJ7axRb8vs5HA5W zpN|vqkxJv5|7Nm!gCx!+r$@0`tiGcO2~WB^F=(Z(xLWW!WbAwqD9X9*%=v!L*t8fd z?%s~;V9Ill^%wuyCf0o0l%Rh0E;aEaDbieqGAbiAO!`D19Ub75S>KO*=V5Nds!+my zt9I?iEMBbl_`;$_a99r@CRLKPfEZ7(v+EEaA0IXSLIBQKXsEB!@{ji@X+?q&#tB^U z2@4~9am#G?E%s+)ACi^udclg**`B){A%NUu4L5Gulozj-6f|%!NOS$Jnwkybl$|IH zw!>|Gha2AMg^vdjTZ)&B_oI@`B4I6f-{BA!Krtt$KiCA$cpE_%Ahwq-VN4*`(hUBI zN9lv|2#3k_nsNS(mkoY@;yvwwYxEihr#qffiTGx7F)VByWER*`zwlXUsj0K^){t*T zXLJ@!k|KK~`DdNV@Io_!OXC|Zqc3k5yDII8$uNKLh?<8;j>F`mJp64PC&hYv0>E=!IOHS+cHh{zqTJ$u7 z&6mc?oKGh#m6O+kP_7%iWzCwpGOA3vON*=Fz%y=m45;!gRQqpk4D z&4$mNtI^Wr6M`zZR{e&J;*Oz)j(qo<|H$=RilDP4oSN+o)`nk^DJqsIH9xzVBHT^n~u~*-| zJ-c@GBg5LN7rXHBZjRZzWbsKp+BTy{H{8;$?@LlC`3o5uSN33Pwb`H~uqI2Y=+)`B zv{zL<&aW%JLh#PkI8KwfQw#HU`ujh*LKZ(8CVu;Liu0w2sW#w`E^3SflwtJSt2jQhm?FlZ zVm5Bv02Foo=#BP6Z7`5}tRFR)zGe7_$v(dmaDL*T_V?`4=49bHu5J$Jm#FoLZWRVY zz(M$k_P@IB*GrDmY|cT~OwITKNd)_95%t$r$7#nXo0(1Y*v7^tF0?-^JNX|#`t_3c z(1gf-UR<32e1aTbqgmb8tBzO3KWW8x@9lWncY010B9R73(1jW|Y2xlZ!PQllovMhM zc^L(Q`-QRqkU2`GweY6N9Y&EQZ#St>1%YQ0z$OTG`*}opULVic$Sjm%|lfp2OfW3 z?=A-qR&HaxybP&_6!K)22DD~_*Ph@N2S<-{w*XGqLPvNnl}>*|-S)b{DLsmh+4XPg z(2UhC#Q|{j&d^Ylci!*H%MbCC5%UVI4fx2ECRs`~e#he{Pi_mlvIV%EYbBY^9RsiT z8u+qM#y+U@AwyNwpWB$SY{L1-+a_E! zC-X7dIHsdVXCipQ;Q(gwcu$&yj7mzgJ(bA(H7TmOHtr{C>X|jvP&HhN%5)7Yc!0QN%=Ew?|@B66ZGkmZD`T? zCd9^vmzMWMyHP;fKw>1T<{suyii^wMyc$%|<89_dO?Y9hWK{=V?Jc^4 z^Qmx&74B6|h|)tCK|(hEPIW@gXLWV!4-!$bU7fFMxQ;s)e$959x~_fPFjknvNl+`< zC-kZH78jaURcmT$5)OmWyh)x2Tf+4B-*pr~5-5_^0zI8&qJj6qiDqV-Qq(+yf`V}E z^o*Kh#2X|XmBCjOEGk{P$oR}?BUhgMGbdzda0duFNa*o?*6DFIg?|+OzM) zTrVDO>NczH$Lu1Un*{1@07cS6*fP`SldYgnBs1?SE9WzTPG|c0tu%>4P$xbp7FT$B zNbe|fVwewu6PfFBr{C`$tb<`#Cj&P>D!-kiM#%6jfrYSeVopq{uNsw@d0X~USI`HQ zc$F{5&h6Ij)oV6>n22!RQ@-_w&Ivj^!rUCF@~4%gqsEMmH*`h&AzxwS?AlA}VSw1Ztk2%x7d*HjFz}#h3*Xe4T!Onh?TA>oLxN{aby2 z_Jh#K$xG(Hoz|{H2lI&&Rd}x#zSY~}pE^NcfFEB*Ol_XToK6hF)j+-l)=zp-{~2wi z?t=#H-n_X9@NW})uSd;;E3bDnaOG*skMYX z_wF{a_pQMoKw36rW?G9W2Gw<`!)`}RS(if^ZkB}KkVE3k8`S-K_Kcl6RaKB663zy0 z7EK6RRH}3Fqfl^|iVAGtaAK(-g8ok$w>f$40S7}J1K)4P#J0uWdz<%T1e$`JXe0;( zgZNxG4QKn7U@i}(P8Oq|_x#j|{K$*Lnd&eul|pKJRXe-O7fT9fIk zF%@D6-}w0Li36%IfpQP5ItTK&-Nj~M_7!@M3UTwGmu#$TD)Fv%*R43Q;N%S;Y9wN+ zsGZMM$9{P{AtCaZ6HJh1ZWEg6q#uUa)C|{C=;te<+8F^Tk%nvZ@4t(60`XR1|M2}V z)?1g_^)_?&#QAIN7Q&M9D9Kz8wVbiUo3lzhELxD7nGq)_@JsV91=AO_*UhbLSVMjW zm6ar!-i21`fEs1Sq2g|??A zO`Epq+G4Lpnhwo&JMIh#NrRt)6xqRl>(&IXE+e?K*0rCdrfs(#VKdqM)R?(HQ5-C@ zsH=eAcAK=7^e1(KfjQq8TDw_W0==ZBr*}6p+N(R`;*Nd$s@5Kxy~*GIYv0<3w$b{Y zwt~^5Keuju9n5Pu9gURo2&XAvK||ZhmwoPL6Wc_Aj5zz$teSAO4FuQP^Yr~lg50n9 zWd)>H?t-`Yr%LwB|Hscvi_R?p%lNtO0xUvHpPy4)$@gC-?A|>2`|rx_wC$Mi@_+je atGl>0RZGqr9Fr^@9kWrErqLs2ulXnYhy_#t literal 35057 zcmeFad05VS+b$lKWy#R8NQ$yhG$~4@s4NM0Xr4z!^E_*2p%x+C8cGpB`~B@d-gp20*xRw51Nq+fclvy;>%7kMysqb(qWp<>MQo1+D)%LcrF+R4_T^YB|1TwcSZAM+Sy1VdQ_yD9)FI z7#RN2qa6D~^+Nb?m+OU1O&^xNyp8*<{^T#5fwi(qe_W4ZjnV#ZxZk}B+~q#gAKLW9%VXtFKfUVe(sLeataEPBi`Or4AIxddZzy!81>L!$ z(sm{_WV3)Sw|~V>{3j?V_~4LhT4vZ4xonpa?T#;Y!>t)dtG9|+aQas~66^8Ide&ib z+<)`ldavoO_@CFYB^flmaN9fGLmzUew4w&5@YDrIL6S4Sooi9zX=ZTm9 zVfn9l(K>lHH6D&fn2(sPJS%Mf;h3JsNY3oL%S_qMG<7a6E}lb&bXKok-CpFHuAZV9 z<=W$=(N*Z&aEtvAW2!;mo`xa!jKziNo{mMY_s>=?%zu3w+{zYd@#@v9TFL3o=%TSF zI$uA&QBhS55)u+}!XfO3Tux`1yircqFF9aV`@6BRv6_pcJ!}5rER9`YtF(9V&Ye3a z8Wf~?BqX$1Sy``NyH@w?>YAox4Z)jiB1y}27**F@QD>mWunL=)H6|!ky?(8_Fgq#Y zJ%3?gW+3AF&6{ecPu~!=ZjbrW`!d;apeDGh(SmRPex*|({5+qv7k^z-_Jqa2Q)$g+ z{@a{gP7P4Gg8T!C;JViqBk>X5 z$|fdpoP(o9W6ikp9Q)qm>{6bZ8v1U=TzTvF#WyypdM!-0P-AeDW=%K z5Qw!FM@ui&-=pxbQNzG1Hm}E9M?hfSCg8)@iHV^2c+)`RlwIeRuULTv>dLJB@ZP;d z7X1rDjaQe`D$QjoB&q-OBn-=_4GZk0RROD()aoffAp|_VW`)0zT%V=j}>Ei4u>H#e8qL=RlOXnZ; zz6>};?e?Ct);XKwU~gZSZ{N#FsjRG=eebuus-Wpe(;sFl72hqdX=%xOPE&Xwp|;t- zVsl}0s*Z-4@^$|)M@i?QAoA|W5?r@cR29odHOu6Thv88`!VhUL9ywx}A)vJ>Wb3IGVH^sUW+H)+c5sQ{B zTZY&6p!-7aIR;U4D&q7>QN*(Ow3L+8_~c}hNQ-XtDvk?B5Mwh|MxJ|l ztFod(cC){%f!Aac7tOjk&aSKCgcHW zYTo7W{$|K7;gnG3zJ^7hF*UC{>D{}tZfMB584|A-pOMZcQ_W|IeyKiXhakn zT7iRuWBkh(CBz32tJZxZ9eHox7m8RkK3?c(SNG7>)8nGp*w`?xUF)PNPpxlgnC$i# z8=R%3ho+~eTQnub3g{J@%}xx;rVYG$dU=)goWpourEG}E+VUx{iLw<;o3y78MiO1e zx*6B43-P-G7epv|MOlVAQpZdw35Y z*3GeKqT=!jT}F~{-{PIOgiB?HD}DI2zQ(3HN96i|1-6t zE8jjEyIo4pzJfVPDcTnyI%dPJNfi1;mC+PG>{ z$Ly^|8s79JXFLnF^KSkXMmN|G<<{o@%*5eYX4j_`>yl1(r8K38%eX>vT{ijt0nmT(2(}ouo;^E zdUQ^n{43+!%k_TN6a@t_EROf5n(Qsgos^ylTvBh4GOY%v?|F6nsn=8eel8(ZGn z>B#u4&nbEk6BBbPx1p=6>#nHv{jNzt-TbrPW@p)`H`cRDY-ww2yIK`~$h9Un-FrF(~k&~9i@nY`MKRorsYFzrV%CQ!z5gW&B7%;nn!cL`Y_FZE@$#BDk|!du*rY_ z^Dc4mwv(q%A3#nU4$08ue=*(TW9xdT=G%IEPCmZs1f9IJ9hLbX?^|vQv8;P^6@SKS z^G~sTS9Zylv;6Jb^xDk%Ia-+Q6P}3@^M$wg)tYaA?pn{9v+5Js%N4P6)YA1ODQ=}d zEzt~?>MyNWb-iR!hDj0$Ym?-+cbBO#H4hH4+>iY5!JsU?WJikB~G zpX#E?U&-QQ^^Mwp?uFURc>l`gEP3BL1R_d3UwW~!na;TvH+ZT?x!=8edYMhqJ)4sU z3my5hwl4aF^VF~w$>^s-1)36+q7mU#hphYh`aZqf?i0j0IozhAp=*CNra08TGei~EjH{hok1Qxx#H)wiWU}0 zM`dJ0fJ;VtN)1eF!s~EW5!>#=l6cZ9-Lt~2mlhY?X9wd62-~ZZdpaW{L&E;U6{&>@ z6&+SFF|oi4KV-ZB^L#lsp+Al$F?V{EXT9}Kb!#; zLd<;*`U_9pYXa`sWov8u=I(wOl8Of#;ziuX&(D1O_UZLKVfUfL_yf2buP+rWZ2OI_ zxD2;O37giK4b(iytu)@P@GwO!S-mZ_6en|ETA0pppR_=zPyhmxaQbw6Grwje&dbZk zcPc=7G2dmZThB~SK|!Hpyo!%H?oOMV7>s@sCdMZ&u0DTG$ zmU97;arcBxEojZD*RNk!wz5iDod2p&0-y%~RJ+}0KI(nBAB)yoz=?@(y#WNQX_vU? z024UlMRoK{;Yu6*wtFYMew^!od40^QSvD~nejeMu?0^lsg#4JcH>)T}#**!yyWq1p za{}o$G%qiYdgl(oMs4jYRo#Lg++TLL{06Hq zJb7|lTN^P7mtmvh)10EUKJQL&u-dz&_}Ez1KmPc`mopmB=|!SS{PmJFVGnICR#Xe3 zKv36HlDT3{CO5WTvgFZ|Xnk=~40XRl->c=-qb1Y&2(*As zzgg!Utg3Ht=kZyX3`PF1(B$9Ti=ygk$iNauJ}~LXl#w*TSnj#kZ{ECV1q6!{0)7?- zB-v4U_kg+bwGOw$w=zlqQ>G_Zo~XjL{Pia=ZPcsT-KX!K<(q3U@L|>r@$tzqt9y2m zLgC3?tP19{o-=y>*9I(rWpghtrLL~-zO>Ji#f34SWXqPc z!VlN^CfY62dG_v2?d&+WkmHVLFX6N3Sz!A%aCq2?LdB{f`B$Lh@zf)C4-XnJ%HGGr zfhRZuIb&mD%({!+t~Q^Mm(QIXU&nSJ;nf^ZS0|(VmZjbRY2F<>{-93uhZ^u6IFQ{+ zljy)M9g){n)&(-l^~!VTuR2;SJ=3?5;x;YdgQz3hJ$HAh#QnmB*2SNt7r#Y!nR;SB zy_Gw4>Q04GU37qyg+~cqp3qtuUqiE@#>7}Ro&F)^IsQ^Sgs#y(ORT=Uk!rrQ{G&_L zK1&)2HvoyO58WD7E8RrviA->Lf*3N3FH}RCK3ZL0U+*;DNAW9BpFwd`V`FkK?^OZ!NxjsG!E^*c4QV zDt|PZc^ekWRzUUnGY%z>Uw^mHk{5#LCijJpVYbaA$nx0A|3u4Jp=ykRP!uUTVsU0* zJ1OD6KPj;85JfF;8jqv|aIhasE-a<5kd}5Q$Tbj) zGf>DjIa@o=W-pK6HYwCMsO^GOXbO0Lk_O{=wZVCLu9y1rx?I8lM!4Fx?<_Vf|GDG& zAGdZd**$k(riEnOP#T2xl2*3TuI#devpWcjB0&&1gmK$?Du_T^Pf>c9j2G@51DPt`^lt-f1cK7Y>5#T8pyxHSJY zpgr3>8cA6fh%0NIprqGq_GUr-l&v=Te~?suMy%_6*xHwY{yqn*(t%tp0Q_T$SKG-| zNgVq|@+EM@n!R~Hzqd>)I4;&{Cv(ZTdF&AmG)K5&9`7}@Cs_aoJ|R^oLH-vmY&uwz zvayNTIh>SS8hXybUB+j^L~|AnE!`gU*Bd(M*o?Q11kwKFGW)u9Pc)UJ?=zbg1WoTU zRhs+oWN!3M(DZ`ogQhJBRp50PA5UFB{qV5Xs`c9xm6fR>f(B_w3$L3u=9<3ckeune zW87YY5<3Z*&9Xj*(q7_0qIqywm}Q>TqSh`xZJ-Y8w;f@J-7=p&d*)QWPK>Yz_r+`< z;;?GfCFGivi3umlRS)%;;_e5?V{%nR3DQX1m-=T`luSO#%}=4_VyhpciR;+^sgE3k zgdDj;#@7ss7Wpl6B+p;Zvv}Jfm)D|>)|BbMAeP^4ioYDE@E-U|BNg*|<4?6wTxmsz zgN-BfUMr0SPL%3LeF)0PTL=%|wvJWk7+8SDZ1V=EcV)|B&x`?~@J zE2r*0rsmVqimY~t3XEDyI)1z{((PlJ|7NUU;GqhD3wc0A0Z&c@aJU26XPdmaFGo8w zhv-B|ftwtX+DP6cpLT6ntz7p~%l=lxhmb7SaPhBSZ%0f9o?l#C%??!1?)krw;49!4BKfCM*)0u)cFo|6F;MO>3K{ID)`f z0Yr6yjbdECzWK3Xxg2{+YDv7exA*whuMJ2oj0-K9#-}K58{KpdbME&s6Bz8hor%l^OrBmAztT=N38XpcoxV< zNLhPyu^BBaELcl!HBk!ja>;lkaO;%vi-%yDI;2*)b~%TBta)G=#!%T_gO}7=WzgAd zTe{FZb*=_CH)2zcdVH+->#dmLhw-SH#jOIOZ`*xw@Q{2G`eAS|ac->CfJ#^|=5J9p zKm=7)Ra69ZAbC}@F6|r(bG!zE995Im_v+=_{bAXhQio@m$|32V0IG5JQ*^+##cvuM zcxFv`wg!Oq$>5V_-})>PI7kPwa&nmq3n}Gp{t`9UJ?1yXhnH+W96sarSmMIx$i;8H zOq7*-YO-Fow6si>kHzjj0U|AD0idhhrFzr!!C*Ppl;tKWPsDhZC8)RO*`||765+F0 z5K}xb*pzNi`g~Her4+z12YhBh{-(u+*L8xETx#Kl>dJL2JSgfhEi=lMm7_vYfloFz859EDPc{Y1{nvuDrC z=4v6kcV){<{s>M@nM>KJRm#M1;oY(~;o=Q8thCRkJW?W}5bhHU%YD~Zlax0#_Q>Q< zfgz4(^VO&3R=h&IPO*J5a~$xu9&grU>wVjWKax-gtS{BLeEISuYC-d7S5~)Yobw}( zGZi7ZI#f^tduMTMX;I;2!1k8s*EXJN%B9{HHoXP@r2oT*N2JuXq2+5D`^AmV9_}kO z?p2n#tf#Bnn4%>-(v~$q%_~M#tg77Y8XEnu{{u(7a{to4)dyEdrZUKJF3D2N1uvvd zJ4}xC^w^vYK0jTzIo5EUz3jS|SUWLg&TAbIeJ&2zBDeb< zb#xaWiW`8LlhgyCAt?3frHd9GJ^_+bnu$Yu1yQau_XCdSZrB=Y9Y48+ztZbZ+2AfL z%D~Ctt|ATPZhJL0;Mv?%OZJ_f!1>nQ#ah&bsHZHB&dzEVF62M-o;AmsRlwx}#N3G~ z-oEVmoR_+KbMG*?AJ;9K!otGN;362!19tU)aoY#FXByk#URPz5z-A5(0iH3>z@4;1$!Gz$v{_px*UCK1xVg>;OE>pj@KVbH zsj+EVX;YdW7vvuvcB|Bq6g25d|MKOFgvV4$mk?6cSvWulRqf4hj6lHraxEED> zdU-WvoI62rlMy=A;cv^nv)E_J8&9K6^uRaVX`bGV`?6ZoTCzOyha4Ro4uDGZbJi^N z`|#nz^u|GWhbk) z=AK`-TpgthRiLZL1tlJ~K{xMWoA^NiUY!j>|Anu(2#MOe^x$oEz<)Zjh1 zfg0@*FsxPTU1II9){;qIX{b3h8Rf2ARak2MEMEzTwbW4C7dKdQ*D!ksa3mc98Q1YT zeb@0D+;-cWVw-bDTjt--eHec$H8;%Ft@foaBEkIqv#VD_OpT2PA%O^vKWl3b0fXD_ z#ed5CmhS-|;Evs?v!HCOvuZ#3yuPV*@LYu26+oEw(UPndd5ZhM1J~=}Z$y?}x1>Lr zP3jWT($aDt$!W1#D%D8WUr+PDal`4t=>RqnIpuC9^@_m_&d&>NKv{fz8{}gq0 zYJ_ITJrL`?iEqSJNlP;z3x$2#$Qenu@fdyYnSMs|Q%Y66U(YTC)*595ac67$>~jOT zzvk*pXQyt*A{g3|zWt)$!ius^rfcU^P4!kI5b1tkZCIv~^lsqU@_kWVMXp<^Ye51; z`z$TKXiQYOS-P~?@mD~A*l!y*-UKV7cr=Q?t&j~_kyXNF~qKrhyfFv!@OmgyzAuU557Lmctt+_i6C zt!9?V&A|KjEjrZYcMi4MaR7-8EG(v0*jE?u zGH*ahV@`6!dd&5!ZAsR6`wiLSCa^P`>Dz}Qx3){$@ayK+ZV@{F*Ozpkh2+{ovF;1e zBl$DrC{P`{q@=pP;Z|;fd*1T!;ltbo;R~PN9i=gG9B#dR-f5&g=hgIs2M?@4Iy}F9 zo)C~5cc0$3e!G;_^NI>qltseW0_RU=vI`@{W$P%EdM{YbjWz4oiiGfM8HGzYR~-aI z%kY@C>8%zlJwOhgzK}JGvs*i#6gqtP@QrWZE)*FDa7bG4nT{>6yhJf!<%|@OT{{kf zt6kFOLB`v-)Ax)zT$HxuMCChc1+1XFKt+9%tP<2h$IqXSvDR7~yX-$Lqpq%gsh=R1 z4DhnMcI+mS0-T)#gRopthlnUm=D!-^`E9k@f_JF43K+Qax2Y6>pI`^P6 z<-GDz-UcWIY6BWaW>b;Vpt4^-#LV+fpK70O!F$RuZ`i$S_wE-bPoCTd%F4PJmAx$Z@VWg^BhM~g-m1Go#@9E1h(T^_ zaJdmQh&Bd>M5JJpS3`)DnM##14vgmXhcp&@m z+}F~VpnzF{NRH#&H&MZ`B2d2oReb|hcqP%h!y}vozjX{5n1^gaBbMyzn3-#N<0mO;U(QNnhXj^6Kj9+vmYb z#+PM+mD-B(@5yH%R4J@UX=H4cXwotX-R}(+xJ}E-0Wn02U!X_40i;ARe`K z7Xc~*bfmvpV~Zc5tT|U3`A2By30b{O5t1IcM-G878F*}E?5keRJtvi%Q0tCu8g1YE zVtq$j>?xLWgY~hpqka&BSV6<)_w`KWcGw6jtEj9=jaaf30sRq;=eC}9;oYTGjPt)} zXtOLafJIb3U(&pw+vtN;AhdqlLB$WbL894gzJ{KkpRQWpp%Q(3-MN}@apOe}Zc8(f zwF91hn*B@5zF?sVW~NAP_xyb6J#yRQy+|qi%^veef$Yt_0|Ta3fc(#`@iyK+WgxhS z@YbM@=?osoVkO*hFiK}^4%IvxJ?GI*E%4QK-6bBA;0fAOUp6$T06jMdzdam{00F?$Mr6dv7h7!;kSxccj)Ko31u~dm1&Fu=)&*KW0WVL;X}~K3LqkK^^*%%0B}D|> zP10Hv$Z4^LNkzXW!c+N>>!>Ou4feD-6l63om9-Gy8w;H+SFTz`iVU!R77cMX^*z7H z5+WN}U~gv041msC(J&Kx-=B46W@p<+uW9F6-2(w?T}{5%x+B*d7^o22g?ZaHepgr5 zrbLw!SP{**nR8*rlSW|3YeAw;R`I1Lptc^Lo(_$Vk7pCLN`QPd2u`TpW2{8OtOOOH zJeGR{ev*XcUx{jj@baXjgm8!(ENvYdQ7&{Ee1yNQ+p@3B#me`}l}|N?#u|wf%OpMj znd%HkEz=OM&@$ANJPCh55|A=ct4M8!iqov@;xbZBkK9X{Y6S6I7H$1nbeW2GO{*#M zj)Mm^31tonH69Wq5hpKQxRpHatX8Gzbd37TQQ6xx4P}Zd86MpwTChAk99$z8C9Vtl*6Y#QnKM z1i*hMO%F=PZu#(pCth z9kI4=U|1%hOYCa*@1N>&|>7qF6u+#ai6-+i#Dg zyk?1>a(|!U(TpLt57+h(x5wm2hsKT_J8)}?P|@hZXIobMt1+R-q>L@Tm*ZVmFkC$9 z%d|ZWfI|+;PQrUWznt~)GKPzec5y2iBJT=7+VTkbgyr>13Il^GLrpFN!={EH9u?F_ zoDlSgE+5>rje$X6jZJVD>@TkcgoRaTE$J<#OAAolr+0~()kacvRx*_JU-Hmi#=!k& zg%Qu^r4=hyAc_b;-BeTdXJE)LRvmngjk~QDj)Wxe90}O-(6kBJNF?<*ca2o-SZp&s zZtj!%`uY~##Rd9alWK&zNbG00n6w)|M}3|rYSnt=EG~Sx8om!DpK(U~O2IU|J58cQ zfXXFxKXwTX+E@#UWf5d^>(<9WkAKK_?Pe$|-g5gVh9mt+^^&QGMzu=|Jxk`0eNlx* z5Lr>EDGxGZ$>(Ql9Gsn-s9dLrm_d9f5)OTT5hZ8Es`X}|j^!;Ri5KXda*~q*4Fre& zyQR>{x)n<^Z+*^h@#XNEQ0nrY8_8_}8cqcCkUw+(KwW2nV-7$$*QfVtWger#!4e7M4))t$fBizK2kPd_wkD^sE7+O6=mW1v zJmZl|Q@(}=J^PL>R>luE^$r*Ex_k7ig+>d_(1a{IlFPkcRaPpYRxP*I2)cdy*Mh>r zaj}$`+^p^2^i8BUrO_OuLfV+rN=N3`t!Z%fd9 z3eDtbmk!EpHN3}3yLayy6y>!*O^;v|su=z04EgxQal`%FBt2L4-#XFF zz{SGOo{B|wsw%85tnD&x{&i|fLA=~F-7aS*MPD+zX7kP(XK3~f4e|O{D8KL6P5;N+ zoY>dk86D-$oKYk1LTuIU-VYykZr{FrW^S(f<;(bIxu?ul;*61~F1P`pGQ*Z_MB-C( zaL5EJh5VILR#wK?CJ@DpgYsYF>)MwmhlYkOU%tGrzh8|<<_|-8Uqfz5Nj)HOHFx(y zLXo_Gue3ul4fH-XE>M1_IK}N!&L5Y5h94-gZ}Za|2(*1PS~8p*VCoD zx-Zt26Y-dn=l9<)mR;Xjke|=}`|n4T^_je6Pd8U=SWQ0$i`eRp09-0+4wj0{ngMrE zb@li41zKUVRhjQUWv*e<#3S0o!gAu@KjZZ|o0G)*GzVl#rY1lB!*W**G)8`v(a{nUdnl@{ z{r9I|=+lpEvOuhZ$;~O>6ZRJ7*84->O^uO9H!*9bCdt{YV`9=UGI|7ep_%Ipd`sB~ zScv0kf9j|&{EFz#zz|gQ;V1Rvq+fNxYw%Xu+YS*AU*tBq0J=LeQa70MvGzU5d}7=} zoSH>>HPWRbF)YGw#D37OY*a$VcWoe&2VxhYT8VzoxO#OUzU98CbrMmdt6sh0 zUZ~+V{r&{{;|^SEsv3Uk4BEWv&6_5CZcXtbxgZe{HB@XN{{9QEzz{!zUL>n<_{$Zd zF6%)_Bti?UR6ywli2y}liyZ=|ch zF&^SiDqw&f*_NRKdYd+Gq=Q0&BO0D?EBt+FkN9zEub(;tR>S_pyGa{DKR>ohU+p`f zB6HM%gY{biO7hHubJ zea)r|rrLR|WP0fH`*PgT1J1#+ODo!8Ga|+$nBrRXY~lZeYa+s_f?2~1lnQyNm-@k+ zq_|IyrUK@ew`Cfy43Yd?wqks0O5pQ-yt%(N?7~af17h+7RM@PdE}|Y0RvYwybviWG zqX3GJInAMma&;_YzT&r%EWaZ>>)0h!32=mB-IQag(eeDW>|`b|m#VMfC!!*P*Xa^M z3~Z>6J(Ytu(S=fNWe71nfr%f$QSIkBRG|oLCGD z5-We~9AbEa!0)A+%E~&>@u?t5#J4TyC%!`$1KZljOI9T*kpk2kTol-kdFyU>BATD1qM*<;YXcLP4UfhLu|mcj z^jOI4%C$O8jpwm8;cG!IRJdE6hNA1#<2B90%PWUsDjhaq^@P({&2b75HetGy*`6yD zq8nKMmBCr@{P`ycCyND!hG>*`2`HU_mdN{WjG=5K9!$vk-dv@^1JUs$ z0Uf!@lk{@ZVRZ_XSO!wOiq}*U)H@V-=GBzI`rM z7zKuHRa*A3Z1^cI^}n=JpgJ{kEBVTxyuT4|Bkje@m#3(xa(9X4)f*i3yK*JS-0Pjg z^F3q}HgzL9sDb~hvhjWN=+UOlo7LOiFeGjAT}8kxS{EJ?(4*>s&;W6nx2bUd{{2|( zC7e7$LdqzOjV)GPq|&!xWV<7NRXy$mH1?fd&U^zr3HkMZs0)Ie>c6?s;{T>oB)4B_ zZp$)FCK&6NUv9%*gz_*JZto+I9=Ou@0P{)7x(Q0xccU^N`UqUR-Qt0|RM)4jBG0O| zSG{eWov&$xmpZV+Z5VglCqC@}R41Y&_?%4Q{2(z8V5lOAKoY$|C&HNZV;hnj1Q&BX z(r(WA%HK$tO9XOw2difENIqvs@>HPxxpK_`!)HIik7EW_n|Meee=3(?sB?p|#3PgpcR+>BIdD?t#`s3$N_Aab-6&(cd1y<)LDN!TkVl%8 zz?@Rt3_h?-ucZ@_CJOSXz)eg{qJ&WF<$7EY782Uly+RlE0~mWwQo;5Hg@x%P+yHvt z+ZZA3qo0_V=t4_^eH>U2UJih$k&r|<9WIr=zq*EnF&O6OB$fDEoIJ3uqQJ}1(O6ut zLoO)_kf|!&4o>Uc=ls=J@L+N|U(Q95hNrJz;Q-nao1J~2$Qf-Zq%i>*NABdw$2u?Y z2wcLUM#`f_=xP<9_i$HWt73aXU~ai*RFxD30RqY<;qVTU(DbWO9DVT z#XGAx^_^mkDb^~7D$5xB1w3A=b`Gc7e1>l4TPLl6eyVre12_Id?MyQP{g0#?Y7U2d z^@-yf(oECJii^qgiTk>}`qq@p;<5t-PgX}VIGJ-%&X?Yr?{c zhG7nzz>x1&n_3;)l2~k-Lx&$@fxYttO%%+b$=8AH&Au4qNCJ`IlW#_sj1i9Ll@qioV(BIAQdN!KmV#DJ zPEPlRmI>^El`_|G2{OLyEfZw2?K7oIyWsL*W79V5sHv&Rom6dx#r2urClEue?ia+T zqj-5OM-y>Nq!Es1tep0j^SvA^B}nRt!F(EW)SuUH(=Ny{I1 zL2YdW-ZPV7n8u{H>jHAN&`&J~}V28oZXU>8{iC z#wQsrepMZ;W!t z#|7}{B*Iq ztkw!`z{np?6_Q|Zp0{S9<0*!ak;l<~(3|uZ?a^#p&f=9@4^$nm8?Fyp&8IIgL2Ko*j z;Agp^_#z0r6lF99@)pMn@7^7!>~R4}S3nFv+4wg$wxq|8A2W8DKw#u`^1uZo9A3*r zfAW9B{Qu87;Qt-)y&n4h9Mo!ndUDVrR%P=u7{+G>s}c_5Nvl=i#{^M0k99xrUxjM% zUFQl_funpEdCT4Feq~{a&(A+7z=8LF%aKWO23%p%m3fwR>bi#+7^*f7aFlJsk50es zmc7i(!UPVstYFnXCM_FoX`*40Q}bVG@ES58ZSEhYZ~3XY#}!cKC+ za@N1E;reAqrnBDPxhi@=c=rXGk&{Vb3|-JTjWoAu!@ZG)mY_Y^YZ(;v$mM_!EK?1) z|57Hqf+3QVTn!vPXC;gaF1&k$i($jXd?gnXTd#ZAt`66-2G=4^u0@?9z>#mpz;JK{ zF2$2vO6oxtCfx%J4B^*sB?7pTIE}`N6)fxi{V*=_KJEVdC(rSCR*{FhYw7z>uH%6v zc+%A??$xx4%ac90O|cruBmFpk0*r0QCkM!14KM7&Cip+z`AyFGUm0lr6%CWlm_2|# zPb}1kuMmC+=|Ot25t!A6BtSYPKvU!f;)kpO#x9K*j7vb*$XkiipF^O<25q2ZEh1TO z6|+$VO-DMX9~`tlN^n2QfT)RyiQk$+(fbQVG8rr&%8Ovq!b@^JiS>cIP9XI^cW~cB zzVYIGCvVX*%iW&p<>NH4vK6S6s zg{d%PqBs&(!OjW%s>wz0AKXk%$~5}d$Rhhrf>CTQbWY|p2N8QU1OkB;ZbRevao-RD zJ?#_;c$HM(ue*h|zy~dwkHvH*;_p=c1iDeL8z# z4vJF)>E|V#?C>cNK@(^r8nw$1s)lAHnx+)3tW(Xbn>T;7(*g`cQvx#!i-y@f=qj)m za`l%eoDlxK1ufCkAfy(QF0FfeX*50zAfPD2MK2^Qye8NlLL>P98K4y_YN_9jx#* z#1vPi;k~U1{_@=&!AUHISN|+kLtz0p%v-*1)AsGQd8fgyu9;tlc1y*C8<;>%&p>py{+1uTpa?q%)ItgsL|PDRFnMWM1w^C~6Vm{d z=u&;b!=vNYSMZIvu4qM8@KGI-t@rNT10|TL6B-c_VbPedgCYUD6l4k91+-6mHz$vz zWY?5dR%S~L5-F>%(x=J!xg8J;V3f@3_&36jFEiaQjhBcAs6y>M*Vfde2GJx$bs}SU zY)lZykKz^qUw-&eW8y18*+W@66!Nun8ub7QprGpi71%!*k~$8wKdUJ~GnH7%7&epc zlMeB3XM)|q!LhpNz++b%>OpBOd+#chz=3%G4#m!M1rp&A0#ei8(Apv)-Bv!s!-R4m z80Em5H^Kg=2FFgP)=y_k=qz52L@h^(yJzL)q)=B+)h3<#7U&y=QX^v1nJ;I>Lsq0N z2fuGgzP_v{_!Q+0BY=i_f+y1rq~AQ0EP6Fdx?o^5O(Ebs;562E=Y4THJh~lvKoR=i<%ujm>ugMT}Ja~O7HWOI)?w%h(`g3r2 zL<8lVn;!1>dBA#wY?H`&wf`KX7il`9J!pX%Lz*Y`o;dW&QIrw;PBok-Cp)ql7@wB{ zTTme9&8=ec9eK8SB@gf3O+b_NNh&cMcf2;D9{2&OtMMk`XO=vAPhK8iCEzQ1B1Ji_-{0E;HD8!poFNAfh#(p9I~ffk!9F zd1b90( zX?;vePOFEyQ{7F^s7DgjzB7;vM<$Q`2O{}D0gxdoGL{&KA)HKqM!Fxv*|do@f|#mM zcGje9CO2hw&%uID%;@_pIHR5Z2+@p4kTg|iBLANM`k+X*4ViEtIdL1#1qcUesCTeC zmb<-!4ptNobnuDbV#kbtAF+tIApqyr{X#@X_h6T z90X!BG9m+`F`T=cTL$5VIoE7R&T>(Yc|5!swmlkYbb(sVMS-LZ^Anaz14O4pBAB5r zX0F>r4nM-IoE8PS)kh67jT0G70PB(}#1e83Age$fs}MPsRDk65zLlKTbrYrUAH|gg z;ziHk7d&%MNEW7=SV$9`<@;38fVOVClrHJ3RxNU|)@dX6hBMl-kF`TA0Y~&!tb+}-8AQ0638pw3zLSVCqDt}^W*-)VswHo!52$1fc;P$jDWM$e&Y9lV^qSift=!^#jC_9 z5p+&NR0!oC7#MhoE(`MWQ&_O6;I5{|dVDG!;D{n}#E;(y*J6UKBvg$S&0#DS0h94Th1-VA3gp$}$IqUKHjT9M{(Y7D|XH@f=q=KC`!Qm-!~lM))#o9%JM7u=Zh9)gHe;j zq(_!M7?6Yf=eGFrZ~m@2F(`84sUTvJc^pbugJ9ocT3a>Q0i}qo5f*T$jq=!RpRg|j zk?{Zh$lVoaWY2;bB)F02+QdUfsfTSAKr<21I05!d(lC!s0O22(2S~W%@C0C*OZl)s zDO|e~AZI_3IZk-c;RWX8x{rwTHP*60_y_oGT`Zlvmjc+4+k!C!M7<`K&3k_b2pRcx zsbEqa9OrthGvV;0;s#om_rnnh{;2<-cZ&Oj>2i?JPfI@7=5kab7ka$ex7NDn2^rJ! zUf5Fv1|pCR{P^`KS813PL`+AV6moy$iQm1u zg$zF+eo@4jy-Z9@C2&#x`&>hg+PGT(N?H|P$dd*W1u<=Ht7Lt`#Mt5r?PZ{!);u{v6qk+D<6>S3x*kHPsju!mtQ z1;=_YOO6~>X8bxaYLd)jfh%;)nHN8iA09DB0(gQ!6|jTUZd5oNg;^G+>h%4azgFCN z>K%_t9Qp$E;TpM59t(+ZupCoELp7o@F}a2q)rH0Axg@~^`yFM-nGVTRP`dC>t49gp z5MRUZ&KQPJylE1vguk!oGV}262F7>eN^e_swq!Td7gP_tfX-84Sih4DPzY$edI>ZSBTa@Z%v_-=(~Fr{zb?z2-=JXvnp@7n1~s zOL^K0MUZ8}IF8_+K5T-=;qS;~H^UR!c6nX&=L>W58xQRF*rzBNetLgU zFZ<4!mgM0$)w|)FE_+;Oi2MyDyMQznpl$IN->w!&fX+ldi`$DNo^jVc(i?z;YYt;`Y|?x0|JvN&sLeE$3=B%JhDe+RBe#t6xLP?U^JOf4uN zmFxaq`1^y$Uu^{>hkHfsMI1dYsGYp(*9?ea4p%F}e%hzMzhX?_Q7YJH#aV;1ZCE zN2}4Ll}a4asJsZ5{Lk%lhHJH9YKsUO^@u&0LVP6MLapwbP58A!_;p2GHFsRaCQasn=FKAP@U{f-WEY}8~4-y8JlHYD(Raf%mVO= zpE1Cz7@19ii62fMzc00>+bS|N7@pBLp9tDsoWB7n${LUn%g_18W6%doEcme2OYo!a zO@=DKEhJR`&&tQSh^$^4`IM`g;79!B-{H~1Y!cdzoLm)GH46EvimT$0QWHxprX$1YHRbS{q&K@b|0|z`{bjtA8)2kFTqa z-`bXlK1ywZaPeFOiDL?hCX|!(3f$SF&Oq? z#6gNPcKyW;9zzl#&Y%as$aPF0;LUevbIoCODvR)18DhS{!iXn5y;z0J8Cv979bwi0 z!iAE1AAR?6@(&;Z%D0(pvO!}eMv4TE{|mv`xtqN~v=0||oPdO74j@GzV}VTZ3H^cf z2o;aee%9O;Mn|uzGV*Iu3D(DufJ3A(Lxpw6a0_unLkT8;dJuO;G?^?>x*(pvHi#!1 z_9bnn* z^xgn0*86CQg7g;zHjj7)5UVU&A4dVFj@`(mzeNFI2xRJjC}f5oLDZnJ$P*>g12FW^ zy8P-cD2F&F4Xd)Ios02p{5 zJGCeoyUSaeJCC#nn-+e*hs0;@-z`ntJpT&ipqR<8e=IKlscXVG%>#k%yWN&#jt8~c zvL1uemeh((uWgm0i!^uf{ z9tr{eCt<-j6k9?*d68 z5SBuv9fiZi`bsFa3x_}iJ8Gi3& z(Afscu@SrafJGxWbjF~>o9Jj+9}4FmsftlzUJZfy8r=^dD5=yBc0!5ZzU8Xsge1Z1 zbqN7d(}z-$F{Hx6S&C$y7ie2KDp-JR>*LeVOA309vw!mVz{ii9d3y&3d3=0)DD}u# z!Dvj)O9M`>#~C1=`!P9&jl?W&10i7{7H+P`{8&LiI{SkVCZ{I$fcLRQ% z;*t_B3L=uq@#F9p2nYy}V>n?}{I@w_yAkrK%DbIqRp^8Q{-?$O?ITxg(zHo`fObTZ zE@(eE6UbOuSf`cHet@3dkq{iBM7kctr9$-d^wd!3Q)IVSqi!XwY@8Tk!MkTqydokmZnhUkSLf}30EMWpcO=6j#_hmt z-W!OAkG>#3$osZ!Fb9X#La#Q{K96yqU;KMfp^+g3m~*1iv5<*1dBnH~OpUelFUvwV z`Wp(SX>5QCwF&ik5=bAk0X?-LJ*H~phh(NCvX*XU4MdPnp$6V{=tM){7VDrPtuzs+ zf*4PxG0B^Qwak6wm(O6#%+7ATwc=N1X0^V*{v7(~;628mcn{0RD$n=ol`E>Be(ge; z)DaVou0M=OOMPrTpiwur)h;CS<3nDrMjgLB;%tn+SBmOBYd6*p!Y~SbSZjP`z$HM{ba5qicxF0 zf}j*AY1m*!$@Nh=h?msy-Adx%r@C0L8 z^=W~0Hq@FCjW#PD426IYWlK7DU~nS#?Bz6!T$Bf2a`u8Cx#GLJ!DHsZF{?N3CN_4` z{Rcc21BR6s9R@I4HQ;qxGl_ra-u9}^il?fvzcrUta~79Qo**yIuHZw_cJ45|8Vd~W zB43&C@PEz(|Gguh$vOW!gunkw?t*BE3`q}RknJc1ZOiA`fk;M;LH+USO{!7p3nL?= z@3w}}Ly&`iI|S`*qz9E!((@2J_{hR@kb8rkp|2wqtyDy^XE>5`LIFowqO}&B3O@xM zC5g2={%5l+bMDAI@2nS4n{HbEhvkY!Xx+!UJ!dQ?11l}2^CqME#tF84_n`kL6!(wj z0es?r<+lER-|+v(gHj8th3;qt>hBDaD%w`mlhsovp03)?jj46VCK0A(ho;yR5*F^(80mL| z^$L*mUzjp?k}9W+iPBHLeON=OhY>gqLaJV-0Pqt!BhH}HwiF#wYxpfm1L3!KnDmON z8?3wO9gWP9a(>OIImz5kY7Cm~n6PTC?(Asj*!{5-F&VRtOH4svdUp3;! zCNvm=n><=wTwIKn4%ibY6eV0>mjs5$@bL)#1Qhzhh84_9$1zom3PDR9zv>H2mE8QC zcw7GsCMi&#A-(QZeStIzNJWi7k&l+MCL2AX5M#CoDmqLR=@NpSCMX~vfRhKB0qF-) zePNVA#<){sQd0Qg<7l?go0%?0VK%bw|q%K=p-$Wp0pI=I>lw6*i>DKDU;It{E@}SC(#vOzaG<0s*y3)FFx=^^QT@ zBpPf<_4@qzF;MNBv`62+Hw<%1eH76D!8@+noj-I!c;xlDwmR=uKKv|`_gp?^gf9u} zQO~tKBvJbZw=DGQ1Tclvs>OTo$mTt`vu(t0{-uMxbi-85ncoKX@YN`|C3VbHa?pBe zG5YeUONv(i(&4xx#gVJiDUsa$Z>gvi#zsi$iV#-~#PhE9aaz^@!@fXZW zF-OZA**p36y;Z6#WbH>@_^xt0|gFNh4@bI`elU zBA%Hu=EsD_wuv_Dx1wn|zHyZ*voR}2I3~Ev<>cd48nbm*e?E&+@z{wI9mmJCP7E?| zKcPdz92543eP-MjIXmoDTZXz(ps15c)WMuya9zuKcCP0{d&Ka z=kl?7)_Cu}@`15`0F{r7NB!A8j6VDUYR%G_6A(UTLS&>w)i=%eYagY^qRPOG6#Mh! zg%s2>y%I(!$FcTQsZ3+rJ3Za7()mHnptAgp@nTp`;d4bZY0I+VLNCs3%s<+60(bW(3gYE-!o`zuWld2?safcBBzv2l0Yx3G(Jc~*wF0wGLxS`LEKI4pJ|9NxgK5IB-rDjaz z8Cy?mzj7`wm{^1SfGcxX%kPG;vMq1DV?IL4*YmqU!B<=5{@GJW966yUuT9QI`+y26 zazPn6Dnc`A`qq@h``MgJ2{SvKoUZG!?r5vh{^vFNTu8c7r-IZV&)h9V9ekKti;B*9N<3|~loA&hCdYChzd- zTGwJ>ebUOY52LH>^6Px{JslH!W&QunB3noFqOq`15G%8@j}@w$`lhGMI=eC^GE*|Z zS(DpGw)n&hQ?W(*SUPE+agzG_CI$c_4!-$UN-C>H7EKF#my*)gz2rNjt^ps4F_LMT zP+i62L@U?N`fYgV)yANuJlJ%%yj}iIb+4}+v>5GPd)vJjaTM#Qx1B#n9HqRi)6X@} zemv(tod)I{~%!q+|6}silbV8n$G4rFO1tUk7%J&Q`Hwl3Tot6+RI@AnQ1ab?*K1!@@2_#nvFMwTee!X1 z)oJ~WZZ1yW4tjs~t-o-0%gR26Rdz`_7FVS1q(((OI^rPO7ngC6xKlr^y6(*z%PFgz z2d~xjvx#1$rBHe94Gg|A#r=hsFTO!WkhQPO>4XqW$$l%JJNd6)03M6@`Ok8w#Hl3z zS9P|pPDi%?JsqhEp%2P(y`otinebmP{YOk10qH@+{Jw(7bQ@OW+nJ7dQ!XzyNxGiJ6C3Gz{{mBgUM3lVS z%9q37og<~=9%kg?;!<<>=P5kXDd-;*QXoknMcXYcb@Z}k*Ve+2=poYMKXzMTkGOk2 z?g+6c2H6S5%%~*j@@We9B9d_a&0ZAYc$|Xrv)f~$Fc6R|3zf(M@{ zj=T`;4q%yjEES%j(sh`*uKS(=XIDIbNuA?+qRd1oKu#ksAUQNicL*RI2Fgdue|Z>% zOsYP<$!Ux<3bJnIaDMK~zU_l2_tqN019Uy<7gxUO%?!CJnWog&JI$Y;NY(xsTp}U! z@n5WOwuF08;}UBI*;6vM0oHg>guVcpiu!v!eH4*Z%VgLrTSdC(zYS`p*m<2@*$*Ds zX8oZvh1q$1`xi=vmp8$6#TS_}|3X{$=^5O6J6@L^d1yjc`RPKaW7L;bP zdv0M4lEASi`fDq|h}FxSkH?^v;81@CDKPA zt?$J<$#W?Jf6;5!>FFZs0dLy9=W0|_p2&Jb=OAyZz9mo?5n>mmDFKVaxtJiGrn*ap zTTNjE@;m8UI9@lrXlE<59OL<2`$(f5T^w6t!z}~^_X8OwlDW#A0ghILYQ;Wg=4aX+ zWESEIKYJ3lk64nR(WJTkyNXqZIo+kK!pFo|DZxc@;Il81ghjj=Awo>(zVuQGW)2mjL!yK9=+TwlT5gs#^YC#I zi9Lx`oYcP6O;pplh=j#2?+s&UC0aiM7elcm6heV<<;Ubz&+dH1uO|cP$*2bnSBTmP zBV%#=0frUU(BX&FH#y~tE_M~Mc51{WtgDSHy%oV@#37I=$T*nMvQ>Q^E4-IY2jM_u zC}NeP1vE^+c%u(cM|uL~4U3ntJAj0YeY4O`OkzPPC$svHuS(_`MDt>vpD^eFg96bU zFR$#?V&cyP&@ty%6l`O1PSr`+?;&m+j*t?7_X`Js?H`6HB%TjLN)3He!sp(_5DQC* zgYp`AZZ{jF_jqJmpX9GNRSX3*$LjBO#6}VOZW_-6T#mQ^iLn2k0SlcO2_bW3{B-qW z{pw3ys<+0|19npal#Q(ukp@eU zXV1J*05h;6ERSra){*g~WSu^_Yg4P6D3X7qRPttmEX$E{U38R^=pG1+K%Fjp?wa~I zn7$+-M$|vf9M4W@+;?2i(aM_~?Zo(AN`{dra+WsU>j&|}eO)C4B@LlQulqsr!!yNE;GBoK;n|52#4l}!%H zBQiP~$#0k6dAlBrl*Jc_DV>FqRC8Ozo2yIFv=iI|%_qByA6pDXp{Uw2alM(cEc}4h049VY?xPIjmRk0(+*{1cgyUfJ6y2|4GJD_0 zS%N`8GlFK7Y41fk!B-v!JT)MhC~o~dQL!3`L%w(|(4~=cMovToC*sH|bpI$SVX4cJ z4BmGj%*z>Kg2k-wh^dm92F`E(ZKoHcqo%C zy=SM}1dIw3FoP1sGIRvQ$ENk**Lh#tExWYs%dmLVI|Tm{P~8^A1TuWBT+^3XFX>^* z^jH8^SVAm8ELe_B(W(`y;>gg?y}%{k2s)*>68yaNy*_igS~wME4;217La?;$hYFsJ zI3*F)Nca-B1lLQBZO@>@g=|Z_5ZG(jA?61x+P=D_3%iXg@!6`?vUTfQc*Hp71g%mo z->R>gm2+AWFflHr-RiV*<>1^k8nJp`Y5jERV&21{meYRulAtEP&$%PoPHe=|-ZPbY z_?69E@zhwNUQg!~dCCt_7gTMbB`@NmRDMUYpYwWw5YSs3GhxDQX<3S-39)6}iXe@h z*e=-i^Y~c*E87=bN5Fv>CmI%a1&$?AgT(GQC~so`D1x40o-N^lFCh}bzyk|1z6A!! z8IZtqfI9nnz??A5SE_x%+Z>@YuS?a<`CkG`?8G4vhZHEUx}&VwQ{mWT$L{=Jw7sWQ zYn?9cHrab>`c!j<6e3|8z3jJE%H_YzE5lSHKFcG^b;ObSMPAH@ngyltD)o4a)WjP} zzp!`;5_+{dXke)xrF2k8XX12YL!hLqoVZn`C&anZHTD@f$AVM6@nBT(l@d^D@g&>= zhKXWHpz9RHK>4x{d1=Xr$Zc|N7e!1IP1|{T37y zFW5XHTyNakrUjn6@7z%t2V@-ecW{e0_|!A>lUDohHP!T29alBg9%&x&neGp-R(|1O z`0M?e_k-_gAFdd%^hSxtLYtGf(_d#l9~ltPpK~ZLH@At0$K#xjrz=Zl_jp@hSND+A zYjbgmt3!;qys- z9J9vMq@+2(iBnwDH*#RzyLayl@?>{1d^@wsWTU;DP zmcqzNb@1RkW|H5yyD$3vw~>ZvH+J_6rKJ)^*wg9M=`!0e>CmA>l`4cKOo+4f@iAvr zHEAeIITaEzcwocfHXj={8rRvJ9KDQGVN>nB7OQ`nl@ny{maz$1&v(d(FoWT=FQ0hdWz7vc=6&ZnmfG-T(B~&mp<>p z!tM|F%6vv<;hpJaeQZyzeA|bGVq|30u4~up#9Tb?9^!)xYp;cB;hl&bJ3Ky&q64F} zx1UW`WIb@;_Pu+rK~(KwVKE;-S(kx}e_;4HE3j-sNSa?`MqEX%dHC)qWdYwIHg51A z!vItN@%}v~2>~FChc90~kSjB=vFS#FU3_T9SS)dM?k0p-4aoM=U9LIgZs9qf{^5KA zV$El%3Z4`u=`h+C;v+}fdZ;HS3)T(-q@A2>M&n}C zV8Rzya|an3uEl+fjSt3q|B)lxh-hp0j$ylYUB*@Rq~Co_%j~-L6K9!e<-0enC@KDSV1z38RjS^)lZ5JcF1q%pn(PJnjJf&RUq!I*3T;|o1Gzo z$~x)azd!3CJR;&eEyS;OIcRn49UKtqmM z$BYV8Y4Y&fBhl*U_!+rl(|c&kU~abvr8z=3)%ShkF}@6+FgYhTGPSVC^XU(iy!8C) zHrm==7LAE<71XvMoPR9He=&!ihPYjKso=l1#HNjh)eja>7D2~pQ?F_L{T%RNSm@zkP0)kQcSuE!u@u8k~6}bZ`wJZTB!=-(Fp~NQ<&V1fLUmu@M zYs5TLK@HJF5fHFB@GxWV-ZY+h{`O1Xz_)ijmkdD<%F)H8FWm|P+#h(9?5Un$VBqW!=M>-S z1L(DssYkyj{5GjSew+MW!dH0s2&M9}Va33p57tu>9lkU-7e573w_Kh%cf@eOfE|Q$ z+GD}%``KU+Wiqj``J1!TJjZ(d4q610MUcJCaD~%~Ezol$>@XH>d4?uV=T)~03ber^ zIGEfroa16=7wuDAod4pi$Ii8Dn}~x!L$bl#iJrP%n&3Wk*oWVskec^Z_+>jV?gL3J ze&3d%N=Y&5-#-ejAN@;=EW9^mMpOf@+M{7e8Z#1fQ)NDbT={4+#1#L{kwOIKfd!9YV&i%ta{n{`lf0(ZS*m? z*(YfrtW*U#-xgLIZ$=nVD2T~WzrO$JCqt6G$DA*=56WATRJjAIvCZw<{ENYZyHI>n zoLvVcfKzCD*cE66T#KU?^7%Dw11Zf*%Bos)6dK$g4Aa(by?Nru-NB)ut;w9GXD-=5 zSKz_P%io;4aA6(Ko5F6>14OH?#SD!>$?e+wzC6^GaJnRCptT?(++DhKQOLQU-#f5d zr%tVGIc%`){BhBEE}dUsK;4B797-NX!Qzxsg~CH zii(&IA66oJ<<;fH(W7UJigX7J8e}mu>k?hUT|)x<)59>aw%$qAPD3F0UB0W6lY7=z zw^qxt){T54A^}Poz^giLib%O zc+Z~B)0X|U!P?roHzrnQW@h%|#`W8>$H7G8 zvN^4GSkB^oR?&txH8=kTT&VffBf$jeDO0AbE%A9oJ7Gz6aU0KNf!>{F6H=N7b1>}^ zrcBB6Np^O-*XzC85Mm`eWjH|n=2aEJuEiQ{x>#?6mAd%RwN4<&d+)iP*Q5c|F6Skf zq(uP%j{rq*($AYWZv;62sDrIlb`ljZtXe3EJ~?pj${S6hMyy!zHDR{5%~0Q8 ztvD?2oLH!jA{hdH5fs?@Cl*#*^lxooP}@-z7ZYMXgjGLZmR9VX?*deo7lD)LgD?9~ zd(|>;(bx^Q>CtooMRK&clF(=)%Vb%n=ut(=ZJ>cL!22l-kc3a!=<3D92 zk(imenQq>6Sy~ zS$XRT3ZC=!1CeD9PAf>wwcmOGk9xrJRL7RCB+Bx)+S&&b`l-K)fx+DLKi`M Date: Fri, 10 Oct 2025 05:42:54 -0700 Subject: [PATCH 038/104] add conflict_policy param to accuracy evaluator to allow for lenient scoring --- README.md | 6 ++ src/vuln_analysis/eval/evaluators/accuracy.py | 84 ++++++++++++++++--- src/vuln_analysis/eval/parse_eval_input.py | 26 ++++-- 3 files changed, 95 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 0e7b28b38..4ddc06d72 100644 --- a/README.md +++ b/README.md @@ -775,6 +775,12 @@ This will write the files `status_accuracy_output.json` and `label_accuracy_outp Under the hood, the source code of the evaluator being run can be found at `eval/evaluators/accuracy.py`; this custom evaluator is registered in the NeMo Agent Toolkit as an evaluator of type `_accuracy`. In `config-eval.yml`, under the `evaluators` section, we have defined two evaluations to be run: `status_accuracy` and `label_accuracy`. However, the underlying code for these evaluations is the same. In other words, they use the same evaluator, which accepts either `status` or `label` as a configurable field input. +In certain instances, a provided ground-truth dataset may be self-conflicting if it is made of a compilation of answers from different sources. For example, a CVE may be listed twice as both "vulnerable" and "not vulnerable" with different justifications. To handle such cases, we provide three levels of strictness for judging accuracy: +- `strict`: Any CVE with self-conflicting ground truth will be omitted and not contribute towards evaluation results. +- `keep_first`: Use the first encountered instance of a CVE as the truth, disregard all other instances. +- `lenient`: Any answer listed in the ground truth for a CVE is considered valid (eg. both "vulnerable" and "not vulnerable" would be scored as correct if the ground truth listed both). +You can configure the policy for handling any conflicting ground truth inputs in the `conflict_policy` field. + You can write similar evaluators to calculate any desired custom metrics; read more at the NeMo Agent Toolkit documentation for [custom evaluators](https://docs.nvidia.com/nemo/agent-toolkit/1.2/reference/evaluate.html#adding-custom-evaluators). ### Inputting Evaluation Test Sets diff --git a/src/vuln_analysis/eval/evaluators/accuracy.py b/src/vuln_analysis/eval/evaluators/accuracy.py index 3f1d7693f..27480b38d 100644 --- a/src/vuln_analysis/eval/evaluators/accuracy.py +++ b/src/vuln_analysis/eval/evaluators/accuracy.py @@ -34,17 +34,73 @@ class AccuracyEvaluator: '''Configurable accuracy evaluator for either the status or label field.''' - def __init__(self, max_concurrency: int, field: str = "status", multiple_reps: bool = False): + def __init__(self, + max_concurrency: int, + field: str = "status", + multiple_reps: bool = False, + conflict_policy: str = "keep_first"): self.max_concurrency = max_concurrency self.semaphore = asyncio.Semaphore(self.max_concurrency) # field must be either "status" or "label" self.field = field self.multiple_reps = multiple_reps + # conflict_policy: "strict" | "keep_first" | "lenient" + self.conflict_policy = conflict_policy async def evaluate(self, eval_input: EvalInput) -> EvalOutput: + def _preprocess_ground_truth(ground_truth_list, field: str, policy: str): + """Build a mapping of vuln_id -> acceptable answer set based on policy. + + - strict: only vuln_ids with a single unique answer are kept + - keep_first: take the first encountered answer + - lenient: any answer in ground truth is considered acceptable + """ + # Collect ground truthanswers per vuln_id in encounter order + encountered_order = {} + answers_per_id = {} + for idx, gt in enumerate(ground_truth_list): + vuln_id = gt.get("vuln_id") + if not vuln_id: + continue + if vuln_id not in encountered_order: + encountered_order[vuln_id] = idx + if field == "status": + ans = gt.get("status") + else: + ans = gt.get("label") + if ans is None: + continue + answers_per_id.setdefault(vuln_id, []) + answers_per_id[vuln_id].append(ans) + + acceptable_map = {} + + if policy == "strict": + for vuln_id, answers in answers_per_id.items(): + distinct = list(dict.fromkeys(answers)) # preserve order, unique + if len(distinct) == 1: + acceptable_map[vuln_id] = {distinct[0]} + return acceptable_map + + if policy == "keep_first": + # Keep first encountered answer per vuln_id + # Use encountered order of ground truth items + for vuln_id, answers in answers_per_id.items(): + first_answer = answers[0] + acceptable_map[vuln_id] = {first_answer} + return acceptable_map + + # lenient: any of the distinct answers are acceptable + for vuln_id, answers in answers_per_id.items(): + distinct = list(dict.fromkeys(answers)) + acceptable_map[vuln_id] = set(distinct) + return acceptable_map + async def process_item(item): - ground_truth_map = eval(item.expected_output_obj) + # Build policy-aware acceptable answers map from the original ground truth list + original_ground_truth = item.full_dataset_entry.get("ground_truth", []) + acceptable_answers_map = _preprocess_ground_truth(original_ground_truth, self.field, self.conflict_policy) pipeline_result = json.loads(item.output_obj)["output"] per_item_accuracies = [] @@ -53,21 +109,23 @@ async def process_item(item): for result in pipeline_result: vuln_id = result["vuln_id"] - question.append(vuln_id) if self.field == "status": justification_status = result.get("justification", {}).get("status", "MISSING") pred = _STATUS_MAP.get(justification_status, "MISSING") - gt = ground_truth_map.get(vuln_id) + acceptable_set = acceptable_answers_map.get(vuln_id) else: # label justification_label = result.get("justification", {}).get("label", "MISSING") pred = justification_label - gt = ground_truth_map.get(f"{vuln_id}_label") + acceptable_set = acceptable_answers_map.get(vuln_id) - generated_answers.append(pred) + # If this vuln_id was filtered out by the conflict policy (e.g., strict), skip entirely + if acceptable_set is None: + continue - if gt is not None: - per_item_accuracies.append(float(pred == gt)) + question.append(vuln_id) + generated_answers.append(pred) + per_item_accuracies.append(float(pred in acceptable_set)) total_correct = sum(per_item_accuracies) total_items = len(per_item_accuracies) @@ -83,9 +141,7 @@ async def process_item(item): reasoning = { "question": question, - answer_key: [ - ground_truth_map.get(q if self.field == "status" else f"{q}_label", "MISSING") for q in question - ], + answer_key: [sorted(list(acceptable_answers_map.get(q, set()))) for q in question], gen_key: generated_answers, "test_set_file": item.full_dataset_entry.get("test_set_file", "unknown"), } @@ -150,12 +206,16 @@ class AccuracyEvaluatorConfig(EvaluatorBaseConfig, name="accuracy"): """Configuration for unified accuracy evaluator""" field: str = "status" # "status" or "label" multiple_reps: bool = False + conflict_policy: str = "keep_first" # "strict", "keep_first", or "lenient" @register_evaluator(config_type=AccuracyEvaluatorConfig) async def accuracy_evaluator(config: AccuracyEvaluatorConfig, builder: EvalBuilder): """Register unified accuracy evaluator""" - evaluator = AccuracyEvaluator(builder.get_max_concurrency(), field=config.field, multiple_reps=config.multiple_reps) + evaluator = AccuracyEvaluator(builder.get_max_concurrency(), + field=config.field, + multiple_reps=config.multiple_reps, + conflict_policy=config.conflict_policy) yield EvaluatorInfo(config=config, evaluate_fn=evaluator.evaluate, description=f"{config.field.title()} Accuracy Evaluator") diff --git a/src/vuln_analysis/eval/parse_eval_input.py b/src/vuln_analysis/eval/parse_eval_input.py index 93dc0b862..336a220dd 100644 --- a/src/vuln_analysis/eval/parse_eval_input.py +++ b/src/vuln_analysis/eval/parse_eval_input.py @@ -32,25 +32,33 @@ def parse_input(file_path: Path) -> EvalInput: container_image = container_data.get('container_image', {}) ground_truth = container_data.get('ground_truth', []) - # Create scan structure with vulnerability IDs from ground truth + # Create scan structure from ground truth scan_vulns = [] ground_truth_mapping = {} + seen_vuln_ids = set() # Ensures vuln_id uniqueness for gt_item in ground_truth: vuln_id = gt_item.get('vuln_id') status = gt_item.get('status') label = gt_item.get('label') - if vuln_id and status: - # Add to scan structure - scan_vulns.append({"vuln_id": vuln_id}) + if not (vuln_id and status): + continue - # Add to ground truth mapping - ground_truth_mapping[vuln_id] = status + if vuln_id in seen_vuln_ids: # skip duplicate vuln_ids + continue - # Add label to ground truth mapping if available - if label: - ground_truth_mapping[f"{vuln_id}_label"] = label + seen_vuln_ids.add(vuln_id) + + # Add to scan structure + scan_vulns.append({"vuln_id": vuln_id}) + + # Add to ground truth mapping + ground_truth_mapping[vuln_id] = status + + # Add label to ground truth mapping if available + if label: + ground_truth_mapping[f"{vuln_id}_label"] = label # Create the question structure (container_image + scan) question_structure = {"image": container_image, "scan": {"vulns": scan_vulns}} From ccd6bde79ec92d36dcee68c13c8465b45004ff5c Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Fri, 10 Oct 2025 06:29:49 -0700 Subject: [PATCH 039/104] data model for eval input --- src/vuln_analysis/data_models/eval_input.py | 76 +++++++++++++++++++++ src/vuln_analysis/eval/parse_eval_input.py | 13 ++-- 2 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 src/vuln_analysis/data_models/eval_input.py diff --git a/src/vuln_analysis/data_models/eval_input.py b/src/vuln_analysis/data_models/eval_input.py new file mode 100644 index 000000000..3433f7c8f --- /dev/null +++ b/src/vuln_analysis/data_models/eval_input.py @@ -0,0 +1,76 @@ +from typing import Literal + +from pydantic import BaseModel +from pydantic import Field +from pydantic import field_validator +from pydantic import model_validator + +from vuln_analysis.data_models.input import ImageInfoInput +from vuln_analysis.utils.string_utils import is_valid_cve_id +from vuln_analysis.utils.string_utils import is_valid_ghsa_id + +EvalLabel = Literal[ + "false_positive", + "code_not_present", + "code_not_reachable", + "requires_configuration", + "requires_dependency", + "requires_environment", + "protected_by_compiler", + "protected_at_runtime", + "protected_at_perimeter", + "protected_by_mitigating_control", + "uncertain", + "vulnerable", +] + +EvalStatus = Literal["AFFECTED", "NOT AFFECTED", "UNKNOWN"] + + +class GroundTruthItem(BaseModel): + vuln_id: str + status: EvalStatus | None = None + label: EvalLabel | None = None + + @field_validator("vuln_id") + @classmethod + def validate_vuln_id(cls, v: str) -> str: + if not (is_valid_cve_id(v) or is_valid_ghsa_id(v)): + raise ValueError(f"{v} is not a valid CVE ID or GHSA ID.") + return v + + +class ContainerEval(BaseModel): + container_image: ImageInfoInput + ground_truth: list[GroundTruthItem] + + +class EvalDataset(BaseModel): + test_set_description: str = Field(default="Unknown test set") + containers: dict[str, ContainerEval] + + @model_validator(mode="after") + def validate_consistent_vuln_sets(self): + # Ensure all containers provide the same set of vuln_ids + container_to_set: dict[str, set[str]] = {} + for container_id, container in self.containers.items(): + vuln_ids = {gt.vuln_id for gt in container.ground_truth} + container_to_set[container_id] = vuln_ids + + if not container_to_set: + return self + + sets = list(container_to_set.values()) + base = sets[0] + mismatches = [] + for cid, s in container_to_set.items(): + if s != base: + missing = sorted(list(base - s)) + extra = sorted(list(s - base)) + mismatches.append((cid, missing, extra)) + + if mismatches: + details = "; ".join(f"{cid}: missing={missing} extra={extra}" for cid, missing, extra in mismatches) + raise ValueError(f"All containers must contain the same set of vuln_id entries. Mismatches: {details}") + + return self diff --git a/src/vuln_analysis/eval/parse_eval_input.py b/src/vuln_analysis/eval/parse_eval_input.py index 336a220dd..cdc37abe4 100644 --- a/src/vuln_analysis/eval/parse_eval_input.py +++ b/src/vuln_analysis/eval/parse_eval_input.py @@ -4,6 +4,8 @@ from nat.eval.evaluator.evaluator_model import EvalInput from nat.eval.evaluator.evaluator_model import EvalInputItem +from vuln_analysis.data_models.eval_input import EvalDataset + def parse_input(file_path: Path) -> EvalInput: """ @@ -13,15 +15,16 @@ def parse_input(file_path: Path) -> EvalInput: Note: Run this inside the vuln_analysis container, from the eval directory """ - # Load the test set file (validation should be done by Pydantic models before this function) + # Load and validate the test set file using Pydantic models with open(file_path, 'r', encoding='utf-8') as f: config_data = json.load(f) + dataset = EvalDataset.model_validate(config_data) # Extract containers - containers = config_data.get('containers', {}) + containers = dataset.containers # Get test set description from the file - test_set_description = config_data.get('test_set_description', 'Unknown test set') + test_set_description = dataset.test_set_description test_set_file = file_path.name # Extract filename from the file path print(f"Processing test set: {test_set_description} (file: {test_set_file})") @@ -29,8 +32,8 @@ def parse_input(file_path: Path) -> EvalInput: eval_items = [] for container_id, container_data in containers.items(): - container_image = container_data.get('container_image', {}) - ground_truth = container_data.get('ground_truth', []) + container_image = container_data.container_image.model_dump() + ground_truth = [gt.model_dump() for gt in container_data.ground_truth] # Create scan structure from ground truth scan_vulns = [] From fc91b674c93c2d6a2a2bc4fd1a3e88b7fc01ff17 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Fri, 10 Oct 2025 06:33:19 -0700 Subject: [PATCH 040/104] clean up config file --- src/vuln_analysis/configs/config-eval.yml | 58 +++++++++++++++++------ 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml index bbd6ed62e..218e4af45 100644 --- a/src/vuln_analysis/configs/config-eval.yml +++ b/src/vuln_analysis/configs/config-eval.yml @@ -15,11 +15,6 @@ general: use_uvloop: true - # telemetry: - # tracing: - # weave: #WIP - # _type: weave - # project: "cve-agent" functions: cve_generate_vdbs: @@ -65,7 +60,7 @@ functions: - Container Image Developer Guide QA System # - Lexical Search Container Image Code QA System # Uncomment to enable lexical search - Internet Search - max_concurrency: null + max_concurrency: 5 max_iterations: 10 prompt_examples: false replace_exceptions: true @@ -92,6 +87,10 @@ llms: temperature: 0.0 max_tokens: 2000 top_p: 0.01 + do_auto_retry: True + max_retries: 5 + retry_on_status_codes: [429, 500, 502, 503, 504] + retry_on_errors: ["Too Many Requests", "Internal Server Error", "Bad Gateway", "Service Unavailable", "Gateway Time-out", "Gateway Timeout"] code_vdb_retriever_llm: _type: nim base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} @@ -99,6 +98,10 @@ llms: temperature: 0.0 max_tokens: 2000 top_p: 0.01 + do_auto_retry: True + max_retries: 5 + retry_on_status_codes: [429, 500, 502, 503, 504] + retry_on_errors: ["Too Many Requests", "Internal Server Error", "Bad Gateway", "Service Unavailable", "Gateway Time-out", "Gateway Timeout"] doc_vdb_retriever_llm: _type: nim base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} @@ -106,6 +109,10 @@ llms: temperature: 0.0 max_tokens: 2000 top_p: 0.01 + do_auto_retry: True + max_retries: 5 + retry_on_status_codes: [429, 500, 502, 503, 504] + retry_on_errors: ["Too Many Requests", "Internal Server Error", "Bad Gateway", "Service Unavailable", "Gateway Time-out", "Gateway Timeout"] cve_agent_executor_llm: _type: nim base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} @@ -113,6 +120,10 @@ llms: temperature: 0.0 max_tokens: 2000 top_p: 0.01 + do_auto_retry: True + max_retries: 5 + retry_on_status_codes: [429, 500, 502, 503, 504] + retry_on_errors: ["Too Many Requests", "Internal Server Error", "Bad Gateway", "Service Unavailable", "Gateway Time-out", "Gateway Timeout"] summarize_llm: _type: nim base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} @@ -120,6 +131,10 @@ llms: temperature: 0.0 max_tokens: 1024 top_p: 0.01 + do_auto_retry: True + max_retries: 5 + retry_on_status_codes: [429, 500, 502, 503, 504] + retry_on_errors: ["Too Many Requests", "Internal Server Error", "Bad Gateway", "Service Unavailable", "Gateway Time-out", "Gateway Timeout"] justify_llm: _type: nim base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} @@ -127,6 +142,10 @@ llms: temperature: 0.0 max_tokens: 1024 top_p: 0.01 + do_auto_retry: True + max_retries: 5 + retry_on_status_codes: [429, 500, 502, 503, 504] + retry_on_errors: ["Too Many Requests", "Internal Server Error", "Bad Gateway", "Service Unavailable", "Gateway Time-out", "Gateway Timeout"] embedders: nim_embedder: @@ -135,6 +154,10 @@ embedders: model_name: ${EMBEDDER_MODEL_NAME:-nvidia/nv-embedqa-e5-v5} truncate: END max_batch_size: 128 + do_auto_retry: True + max_retries: 5 + retry_on_status_codes: [429, 500, 502, 503, 504] + retry_on_errors: ["Too Many Requests", "Internal Server Error", "Bad Gateway", "Service Unavailable", "Gateway Time-out", "Gateway Timeout"] workflow: _type: cve_agent @@ -154,17 +177,24 @@ eval: dataset: _type: custom file_path: data/eval_datasets/common-test-set.json - container_ids: ["morpheus-23.11", "morpheus-24.03"] function: vuln_analysis.eval.parse_eval_input.parse_input evaluators: status_accuracy: - _type: status_accuracy + _type: accuracy + field: status + multiple_reps: False + conflict_policy: strict # options: strict, keep_first, lenient label_accuracy: - _type: label_accuracy - # Uncomment to enable response similarity and consistency evaluators - # response_similarity: - # _type: response_similarity - # aggregation: "average" #Options: average, min, max - # consistency: + _type: accuracy + field: label + multiple_reps: False + conflict_policy: strict + + # Uncomment to enable response consistency evaluators + # status_consistency: + # _type: consistency + # field: status + # label_consistency: # _type: consistency + # field: label From e015abbf063c248d8cc98589812648ac30a0439e Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Mon, 13 Oct 2025 10:50:56 -0700 Subject: [PATCH 041/104] added robust parsing to checklist generation output. added max_concurency limiters to other stages of pipeline --- README.md | 21 +++++++++++++- src/vuln_analysis/functions/cve_checklist.py | 19 ++++++++---- src/vuln_analysis/functions/cve_justify.py | 12 ++++++-- src/vuln_analysis/functions/cve_summarize.py | 17 +++++++++-- .../utils/checklist_prompt_generator.py | 29 +++++++++++++++---- 5 files changed, 81 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 4ddc06d72..f8e7f4ce4 100644 --- a/README.md +++ b/README.md @@ -546,6 +546,7 @@ The configuration defines how the workflow operates, including functions, LLMs, - `skip`: If true, skips this check and runs all CVEs through the agent. Useful when the input CVEs to the workflow have been validated by a dedicated vulnerability scanner. Default is false. - Core LLM engine functions: - `cve_checklist`: Generates tailored, context-sensitive task checklist for impact analysis. + - `max_concurrency`: Controls the maximum number of concurrent requests to the LLM. Default is `null`, which doesn't limit concurrency. - `Container Image Code QA System`: Retriever tool used by `cve_agent_executor` to query source code vector database. - `Container Image Developer Guide QA System`: Retriever tool used by `cve_agent_executor` to query documentation vector database. - `Lexical Search Container Image Code QA System`: Lexical search tool used by `cve_agent_executor` to search source code. This tool is an alternative to `Container Image Code QA System` and can be useful for very large code bases that take a long time to embed as a vector database. Disabled by default, enable by uncommenting the tool in `cve_agent_executor`. @@ -562,8 +563,10 @@ The configuration defines how the workflow operates, including functions, LLMs, - `verbose`: Set to true for verbose output. Default is `false`. - `cve_summarize`: Generates concise, human-readable summarization paragraph from agent results. - `llm_name`: Name of LLM (`summarize_llm`) configured in `llms` section. + - `max_concurrency`: Controls the maximum number of concurrent requests to the LLM. Default is `null`, which doesn't limit concurrency. - `cve_justify`: Assigns justification label and reason to each CVE based on summary. - `llm_name`: Name of LLM (`justify_llm`) configured in `llms` section. + - `max_concurrency`: Controls the maximum number of concurrent requests to the LLM. Default is `null`, which doesn't limit concurrency. - Postprocessing/Output functions - `cve_file_output`: Outputs workflow results to a file. - `file_path`: Defines the path to the file where the output will be saved. @@ -590,6 +593,7 @@ The configuration defines how the workflow operates, including functions, LLMs, 6. **Evaluations and Profiling** (`eval`): The `eval` section contains the evaluation settings for the workflow. Refer to [Evaluating NVIDIA Agent Intelligence Toolkit Workflows](https://docs.nvidia.com/nemo/agent-toolkit/latest/workflows/evaluate.html) for more information about NeMo Agent toolkit built-in evaluators as well as the plugin system to add custom evaluators. The CVE workflow uses the `eval` section to configure a profiler that uses the NeMo Agent toolkit evaluation system to collect usage statistics and store them to the local file system. You can find more information about NeMo Agent toolkit profiling and performance monitoring [here](https://docs.nvidia.com/nemo/agent-toolkit/latest/workflows/profiler.html). - `general.output_dir`: Defines the path to the directory where profiling results will be saved. - `general.dataset`: Defines file path and format of dataset used to run profiling. + - `evaluators`: Custom evaluators defined in `eval/evaluators`. See [Evaluation](#evaluation) for more details on each custom evaluator and their configurations. - `profiler`: The profiler for this workflow is configured with the following options. - `token_uniqueness_forecast`: Compute inter query token uniqueness - `workflow_runtime_forecast`: Compute expected workflow runtime @@ -783,6 +787,9 @@ You can configure the policy for handling any conflicting ground truth inputs in You can write similar evaluators to calculate any desired custom metrics; read more at the NeMo Agent Toolkit documentation for [custom evaluators](https://docs.nvidia.com/nemo/agent-toolkit/1.2/reference/evaluate.html#adding-custom-evaluators). +**Note: Depending on the size of the test set, and how many evaluation [repetitions](#running-multiple-evaluations) are run, you may have to lower the maximum concurrency to avoid `429: Too Many Requests` errors. See [Troubleshooting](#429-errors---too-many-requests) for details.** + + ### Inputting Evaluation Test Sets Ground truth datasets should be put in the `data/eval_datasets/` directory. Each eval test set should be a separate json, with the following structure: @@ -964,11 +971,20 @@ Error requesting [1/10]: (Retry 0.1 sec) https://services.nvd.nist.gov/rest/json ##### 429 Errors - Too Many Requests -429 errors can occur when your requests exceed the rate limit for the model. Try setting the `cve_agent_executor.max_concurrency` in the [config.yml](src/vuln_analysis/configs/config.yml) to a low value such as 5 to reduce the rate of requests. +429 errors can occur when your requests exceed the rate limit for the model. These errors most commonly occur during the `cve_agent_executor` part of the workflow. Try setting the `cve_agent_executor.max_concurrency` in the [config.yml](src/vuln_analysis/configs/config.yml) to a low value such as 5 to reduce the rate of requests. ``` Exception: [429] Too Many Requests ``` +When running a large input dataset, or running evaluation with multiple repetitions, other parts of the workflow can also run into 429 errors. Aside from `cve_agent_executor.max_concurrency`, the following pipeline pieces also have `max_concurrency` limiters that you can set in the config file: + + - `eval.general.max_concurrency`: Controls how many complete evaluation runs can execute simultaneously. Each evaluation run executes the entire workflow from start to finish on a single source code input (so if your test set contains 10 source code repositories, you will have 10 evaluation runs, and this setting determines how many of those can run in parallel). + - `functions.cve_checklist.max_concurrency`: Controls maximum concurrency of the checklist creation part of the workflow, within a single evaluation run. + - `functions.cve_summarize.max_concurrency`: Controls maximum concurrency of the summarization part of the workflow, within a single evaluation run. + - `functions.cve_justify.max_concurrency`: Controls maximum concurrency of the justification part of the workflow, within a single evaluation run. + +To identify which workflow stage is triggering the 429 error, search the error output for these keywords: `agent_executor`, `checklist`, `summarize`, or `justify`. Once you find which stage is failing, adjust the corresponding `max_concurrency` setting for that specific part of the workflow. + #### Authentication errors Authentication errors will occur if required API key(s) are invalid or have not been set as environment variables as described in [Set up the environment file](#set-up-the-environment-file). For example, the following error will occur if `NVIDIA_API_KEY` is not properly set: @@ -984,6 +1000,9 @@ Note that exporting the required environment variables in a container shell will If you run out of credits for the NVIDIA API Catalog, you will need to obtain more credits to continue using the API. Please contact your NVIDIA representative to get more credits added. +### Mac Workarounds +Development on macOS is supported on a best-effort basis and may require workarounds. One current known issue is that running the project docker container causes a `ModuleNotFoundError: No module named 'numpy.distutils'` related to faiss. To work around this, in the `Dockerfile` change the `ARG PYTHON_VERSION=3.12` line to `ARG PYTHON_VERSION=3.11`. The full issue is tracked [here](https://github.com/NVIDIA-AI-Blueprints/vulnerability-analysis/issues/115). + ## Testing and validation Test-driven development is essential for building reliable LLM-based agentic systems, especially when deploying or scaling them in production environments. diff --git a/src/vuln_analysis/functions/cve_checklist.py b/src/vuln_analysis/functions/cve_checklist.py index fba78ce62..9d2cb4146 100644 --- a/src/vuln_analysis/functions/cve_checklist.py +++ b/src/vuln_analysis/functions/cve_checklist.py @@ -32,6 +32,8 @@ class CVEChecklistToolConfig(FunctionBaseConfig, name="cve_checklist"): Defines a function that generates tailored, context-sensitive task checklist for impact analysis. """ llm_name: str = Field(description="The LLM model to use") + max_concurrency: int | None = Field( + ge=1, default=None, description="The maximum number of concurrent checklist generation invocations.") prompt: str | None = Field( default=None, description= @@ -47,13 +49,20 @@ async def cve_checklist(config: CVEChecklistToolConfig, builder: Builder): from vuln_analysis.utils.checklist_prompt_generator import generate_checklist llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + semaphore = asyncio.Semaphore(config.max_concurrency) if config.max_concurrency else None async def generate_checklist_for_cve(cve_intel): - - checklist = await generate_checklist(prompt=config.prompt, - llm=llm, - input_dict=cve_intel, - enable_llm_list_parsing=False) + if semaphore: + async with semaphore: + checklist = await generate_checklist(prompt=config.prompt, + llm=llm, + input_dict=cve_intel, + enable_llm_list_parsing=False) + else: + checklist = await generate_checklist(prompt=config.prompt, + llm=llm, + input_dict=cve_intel, + enable_llm_list_parsing=False) checklist = await _parse_list([checklist]) diff --git a/src/vuln_analysis/functions/cve_justify.py b/src/vuln_analysis/functions/cve_justify.py index ed08267f3..3e338af51 100644 --- a/src/vuln_analysis/functions/cve_justify.py +++ b/src/vuln_analysis/functions/cve_justify.py @@ -29,8 +29,11 @@ class CVEJustifyToolConfig(FunctionBaseConfig, name="cve_justify"): """ Defines a function that assigns justification label and reason to each CVE based on summary. - """ +""" llm_name: str = Field(description="The LLM model to use") + max_concurrency: int | None = Field(ge=1, + default=None, + description="The maximum number of concurrent justification invocations.") @register_function(config_type=CVEJustifyToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) @@ -47,9 +50,14 @@ async def cve_justify(config: CVEJustifyToolConfig, builder: Builder): prompt = PromptTemplate(input_variables=["summary"], template=jp.JUSTIFICATION_PROMPT) chain = prompt | llm + semaphore = asyncio.Semaphore(config.max_concurrency) if config.max_concurrency else None async def justify_cve(summary): - justification_text = await chain.ainvoke({"summary": summary}) + if semaphore: + async with semaphore: + justification_text = await chain.ainvoke({"summary": summary}) + else: + justification_text = await chain.ainvoke({"summary": summary}) return justification_text.content async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: diff --git a/src/vuln_analysis/functions/cve_summarize.py b/src/vuln_analysis/functions/cve_summarize.py index 2aea14103..852634724 100644 --- a/src/vuln_analysis/functions/cve_summarize.py +++ b/src/vuln_analysis/functions/cve_summarize.py @@ -33,6 +33,9 @@ class CVESummarizeToolConfig(FunctionBaseConfig, name="cve_summarize"): Defines a function that generates concise, human-readable summarization paragraph from agent results. """ llm_name: str = Field(description="The LLM model to use") + max_concurrency: int | None = Field(ge=1, + default=None, + description="The maximum number of concurrent summarization invocations.") @register_function(config_type=CVESummarizeToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) @@ -46,11 +49,19 @@ async def cve_summarize(config: CVESummarizeToolConfig, builder: Builder): llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) prompt = PromptTemplate(input_variables=["response"], template=SUMMARY_PROMPT) chain = prompt | llm + semaphore = asyncio.Semaphore(config.max_concurrency) if config.max_concurrency else None async def summarize_cve(results): - response = '\n'.join( - [get_checklist_item_string(idx + 1, checklist_item) for idx, checklist_item in enumerate(results[1])]) - final_summary = await chain.ainvoke({"response": response}) + if semaphore: + async with semaphore: + response = '\n'.join([ + get_checklist_item_string(idx + 1, checklist_item) for idx, checklist_item in enumerate(results[1]) + ]) + final_summary = await chain.ainvoke({"response": response}) + else: + response = '\n'.join( + [get_checklist_item_string(idx + 1, checklist_item) for idx, checklist_item in enumerate(results[1])]) + final_summary = await chain.ainvoke({"response": response}) return final_summary.content diff --git a/src/vuln_analysis/utils/checklist_prompt_generator.py b/src/vuln_analysis/utils/checklist_prompt_generator.py index 6eb7abfd5..d93178548 100644 --- a/src/vuln_analysis/utils/checklist_prompt_generator.py +++ b/src/vuln_analysis/utils/checklist_prompt_generator.py @@ -14,7 +14,9 @@ # limitations under the License. import ast +import json import logging +import re from jinja2 import Template from langchain_core.language_models.base import BaseLanguageModel @@ -66,15 +68,30 @@ async def _parse_list(text: list[str]) -> list[list[str]]: # Remove newline characters that can cause incorrect string escaping in the next step x = x.replace("\n", "") - # Ensure backslashes are escaped - x = x.replace("\\", "\\\\") - # Try to do some very basic string cleanup to fix unescaped quotes x = attempt_fix_list_string(x) - # Only proceed if the input is a valid Python literal - # This isn't really dangerous, literal_eval only evaluates a small subset of python - current = ast.literal_eval(x) + # Handle backslash sequences that break JSON parsing + x = x.replace('\\\\..\\\\', '\\..\\') # Fix \\..\\ patterns + x = x.replace('\\\\"', '\\"') # Fix \\" patterns + + # Try JSON parsing first + try: + current = json.loads(x) + except json.JSONDecodeError: + # Use regex to extract list items manually + pattern = r'\"([^\"]*(?:\\.[^\"]*)*)\"' + matches = re.findall(pattern, x) + if matches: + # Clean up the matches by removing extra escaping + current = [] + for match in matches: + cleaned = match.replace('\\\\', '\\') + current.append(cleaned) + else: + # Fall back to ast.literal_eval + x = x.replace("\\", "\\\\") + current = ast.literal_eval(x) # Ensure that the parsed data is a list if not isinstance(current, list): From d755b8e6895a19bd54cc4f78bdb45651e2e5c7bb Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Mon, 13 Oct 2025 12:48:29 -0700 Subject: [PATCH 042/104] add accidentally removed .tmp files back --- .tmp/.gitkeep | 0 .../CVE-2023-36632.md | 394 ++++++++++++++++++ 2 files changed, 394 insertions(+) create mode 100644 .tmp/.gitkeep create mode 100644 .tmp/vulnerability_markdown_reports/CVE-2023-36632.md diff --git a/.tmp/.gitkeep b/.tmp/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/.tmp/vulnerability_markdown_reports/CVE-2023-36632.md b/.tmp/vulnerability_markdown_reports/CVE-2023-36632.md new file mode 100644 index 000000000..0708cbb9a --- /dev/null +++ b/.tmp/vulnerability_markdown_reports/CVE-2023-36632.md @@ -0,0 +1,394 @@ + + +# Vulnerability Analysis Report for CVE-2023-36632 +> **Container Analyzed:** `nvcr.io/nvidia/morpheus/morpheus:23.11-runtime` + + +> **SBOM Info:** `type='file' file_path='data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom'` + + +> **Status:** Exploitable +## CVE-2023-36632 +> GHSA-ID: GHSA-gv66-v8c8-v69c + +`Disputed`[source](https://nvd.nist.gov/vuln/detail/CVE-2023-36632) + + + The legacy email.utils.parseaddr function in Python through 3.11.4 allows attackers to trigger "RecursionError: maximum recursion depth exceeded while calling a Python object" via a crafted argument. This argument is plausibly an untrusted value from an application's input data that was supposed to contain a name and an e-mail address. NOTE: email.utils.parseaddr is categorized as a Legacy API in the documentation of the Python email package. Applications should instead use the email.parser.BytesParser or email.parser.Parser class. [references](#ref) + + +### Severity and CVSS Score +| Source | Severity | CVSS Score | Vector | Published At | Last Updated | +|---|---|---|---|---|---| +| [**GHSA**](https://github.com/advisories/GHSA-gv66-v8c8-v69c) |UNKNOWN|N/A|N/A|2023-06-25|2023-11-06 +| [**NVD**](https://nvd.nist.gov/vuln/detail/CVE-2023-36632) |HIGH|7.5|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H|2023-06-25|2024-11-21| +| [**RHSA**](https://access.redhat.com/security/cve/CVE-2023-36632) |N/A|N/A|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N|2023-06-25|N/A| +| [**Ubuntu**](https://ubuntu.com/security/CVE-2023-36632) |HIGH|7.5|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H|2023-06-25|2024-07-24| + + +### EPSS Score +| EPSS | Percentile | +| --- | --- | +| 0.00121 | 0.47904 | + +## Vulnerability Analysis + +### Summary (Exploitable) +Based on the provided Checklist and Findings, the CVE is exploitable. Specifically, the application does not handle exceptions like `RecursionError` when processing input data (Checklist Item 1), and the Docker container's error handling mechanisms are not explicitly designed to handle `RecursionError` exceptions (Checklist Item 2). Additionally, there is a potential source of uncontrolled recursion in the `ProducerConsumerQueue` class due to the use of a recursive lock (Checklist Item 3). These findings suggest that the CVE is exploitable, as the application and Docker container do not have adequate measures in place to prevent or handle `RecursionError` exceptions. + +### Justification + + +>label: vulnerable + +The analysis concludes that the CVE is exploitable due to the application's and Docker container's inadequate handling of `RecursionError` exceptions and the presence of a potential source of uncontrolled recursion, indicating that the package is vulnerable to the CVE and needs to be patched. + +--- +### Vulnerable SBOM Dependencies + + +| SBOM Name | SBOM Version | Dependency Name | Dependency Version | Relation | System | +| --- | --- | --- | --- | --- | --- | +| python | 3.10.12 | python | 3.10.12 | SELF | binary | +| python | 3.10.13 | python | 3.10.13 | SELF | binary | +### Checklist +1. [Assess Input Data Handling](#checklist-step-1) + + 1. [Docker Container Code QA System](#checklist-step-1.1) + 2. [Docker Container Code QA System](#checklist-step-1.2) + 3. [Docker Container Developer Guide QA System](#checklist-step-1.3) +2. [Review Error Handling Mechanisms](#checklist-step-2) + + 1. [Docker Container Code QA System](#checklist-step-2.1) + 2. [Docker Container Developer Guide QA System](#checklist-step-2.2) + 3. [Internet Search](#checklist-step-2.3) +3. [Inspect Code for Uncontrolled Recursion](#checklist-step-3) + + 1. [Docker Container Code QA System](#checklist-step-3.1) + 2. [Docker Container Code QA System](#checklist-step-3.2) + +## Checklist Details + +## Step 1 : Assess Input Data Handling + + +> **Input**: *Assess Input Data Handling: Evaluate how the application handles exceptions like `RecursionError`, as the exploit triggers this specific error. Check if the application processes potentially untrusted input data that could include crafted arguments designed to exploit this vulnerability.* + +> **Response**: *The application does not handle exceptions like `RecursionError` when processing input data, but it does handle `RecursionError` in the logging part of the application. Additionally, the application processes input data, but it's unclear whether this input data is potentially untrusted or could include crafted arguments designed to exploit a vulnerability. The application attempts to validate and sanitize input data, but it's still unclear whether this is sufficient to prevent exploitation. The purpose and functionality of the Docker container, and how it handles input data, is not explicitly explained in the provided context.* + +### Step 1.1 : *Docker Container Code QA System* + + +#### Action Log +
Thought: To assess the input data handling of the application, I need to understand how the application processes input data and if it has any mechanisms in place to handle exceptions like `RecursionError`. I also need to check if the application processes potentially untrusted input data that could include crafted arguments designed to exploit this vulnerability.
+
+Action: Docker Container Code QA System
+Action Input: Does the application handle exceptions like `RecursionError` when processing input data?
+
+ + +#### Tool Input +
Does the application handle exceptions like `RecursionError` when processing input data?
+
+ + +#### Tool Output +
No, the application does not handle exceptions like `RecursionError` when processing input data. According to the provided context, `RecursionError` is only handled in the logging part of the application, specifically when writing log messages. It is not handled in the input data processing part.
+ + Source Documents + + | ID | Type | source |... content_type |... language | Page Content | +| --- | --- | --- | --- | --- | --- | +| None | Document | morpheus/service/vdb/milvus_vector_db_service.py | functions_classes | python |
View ContentReturns ------- typing.Any The search result, which can vary depending on the query and options. Raises ------ RuntimeError If an error occurs during the search operation. If query argument is `None` and `data` keyword argument doesn't exist. If `data` keyword arguement is `None`. """ logger.debug("Searching in collection: %s, query=%s, kwargs=%s", self.\_name, query, kwargs) return self.\_client.query(collection\_name=self.\_name, filter=query, \*\*kwargs)
| +| None | Document | morpheus/stages/input/appshield_source_stage.py | functions_classes | python |
View Contentself.\_encoding = encoding self.\_input\_count = None self.\_watcher = DirectoryWatcher(input\_glob=input\_glob, watch\_directory=watch\_directory, max\_files=max\_files, sort\_glob=sort\_glob, recursive=recursive, queue\_max\_size=queue\_max\_size, batch\_timeout=batch\_timeout) @property def name(self) -> str: return "from-appshield" @property def input\_count(self) -> int: """Return None for no max intput count""" return self.\_input\_count def supports\_cpp\_node(self): return False
| +| None | Document | morpheus/stages/input/appshield_source_stage.py | functions_classes | python |
View ContentIf true the list of files matching `input\_glob` will be processed in sorted order. recursive : bool, default = True If true, events will be emitted for the files in subdirectories matching `input\_glob`. queue\_max\_size : int, default = 128 Maximum queue size to hold the file paths to be processed that match `input\_glob`. batch\_timeout : float, default = 5.0 Timeout to retrieve batch messages from the queue. encoding : str, default = latin1 Encoding to read a file. """
| +| None | Document | morpheus/utils/logger.py | functions_classes | python |
View Contentfile = self.\_stderr if is\_error else self.\_stdout color\_kwargs = self.\_determine\_color(record.levelno) with tqdm.external\_write\_mode(file=file, nolock=False): # Write the message click.echo(click.style(msg, \*\*color\_kwargs), file=file, err=is\_error) self.flush() # See issue 36272 https://bugs.python.org/issue36272 except (KeyboardInterrupt, SystemExit, RecursionError): # noqa raise except Exception: # pylint: disable=broad-except self.handleError(record)
| + + + + +[back to top](#checklist-toc) + +### Step 1.2 : *Docker Container Code QA System* + + +#### Action Log +
Thought: The application does not handle exceptions like `RecursionError` when processing input data. However, it does handle `RecursionError` in the logging part of the application. This suggests that the application may not be robust against crafted arguments designed to exploit this vulnerability.
+
+Action: Docker Container Code QA System
+Action Input: Does the application process potentially untrusted input data that could include crafted arguments designed to exploit this vulnerability?
+
+ + +#### Tool Input +
Does the application process potentially untrusted input data that could include crafted arguments designed to exploit this vulnerability?
+
+ + +#### Tool Output +
Based on the provided context, it appears that the application does process input data, specifically `user_inputs` and `input_names`, which are used to create mappings and validate input names. However, it's unclear whether this input data is potentially untrusted or could include crafted arguments designed to exploit a vulnerability.
+
+The tests `ProcessInputNamesMismatchMultipleMappings` and `ProcessInputNamesMismatchSingleMapping` suggest that the application is designed to handle invalid input and throw exceptions when input names do not match. This implies that the application is attempting to validate and sanitize input data, but it's still unclear whether this is sufficient to prevent exploitation.
+
+Therefore, the answer to the question is: I don't know.
+ + Source Documents + + | ID | Type | source |... content_type |... language | Page Content | +| --- | --- | --- | --- | --- | --- | +| None | Document | morpheus/_lib/tests/llm/test_utils.cpp | functions_classes | cpp |
View ContentTEST\_F(TestLLMUtils, ProcessInputNamesMismatchMultipleMappings) { auto user\_inputs = llm::user\_input\_mappings\_t{{"/ext1", "input1"}, {"/ext2", "input3"}}; auto input\_names = std::vector<std::string>{"input1", "input2"}; EXPECT\_THROW(llm::process\_input\_names(user\_inputs, input\_names), std::invalid\_argument); }
| +| None | Document | morpheus/_lib/include/morpheus/messages/memory/inference_memory_nlp.hpp | functions_classes | cpp |
View Contentstruct InferenceMemoryNLPInterfaceProxy : public InferenceMemoryInterfaceProxy { /\*\* \* @brief Create and initialize an InferenceMemoryNLP object, and return a shared pointer to the result \* \* @param count : Message count in inference memory object \* @param input\_ids : The token-ids for each string padded with 0s to max\_length \* @param input\_mask : The mask for token-ids result where corresponding positions identify valid token-id values \* @param seq\_ids : Ids used to index from an inference input to a message. Necessary since there can be more inference inputs than messages (i.e., if some messages get broken into multiple inference requests) \* @return std::shared\_ptr<InferenceMemoryNLP> \*/
| +| None | Document | morpheus/_lib/tests/llm/test_utils.cpp | functions_classes | cpp |
View ContentTEST\_F(TestLLMUtils, ProcessInputNamesMismatchSingleMapping) { auto user\_inputs = llm::user\_input\_mappings\_t{{"/ext1", "input2"}}; auto input\_names = std::vector<std::string>{"input1"}; EXPECT\_THROW(llm::process\_input\_names(user\_inputs, input\_names), std::invalid\_argument); }
| +| None | Document | morpheus/_lib/messages/module.cpp | functions_classes | cpp |
View Content&InferenceMemoryNLPInterfaceProxy::set\_input\_mask) .def\_property( "seq\_ids", &InferenceMemoryNLPInterfaceProxy::get\_seq\_ids, &InferenceMemoryNLPInterfaceProxy::set\_seq\_ids);
| + + + + +[back to top](#checklist-toc) + +### Step 1.3 : *Docker Container Developer Guide QA System* + + +#### Action Log +
Thought: The application does not handle exceptions like `RecursionError` when processing input data, but it does handle `RecursionError` in the logging part of the application. Additionally, the application processes input data, but it's unclear whether this input data is potentially untrusted or could include crafted arguments designed to exploit a vulnerability. The application attempts to validate and sanitize input data, but it's still unclear whether this is sufficient to prevent exploitation.
+
+Action: Docker Container Developer Guide QA System
+Action Input: What is the purpose and functionality of the Docker container, and how does it handle input data?
+
+ + +#### Tool Input +
What is the purpose and functionality of the Docker container, and how does it handle input data?
+
+ + +#### Tool Output +
Unfortunately, the provided context does not explicitly explain the purpose and functionality of the Docker container or how it handles input data. The context mentions that Docker containers can be used to get started with Morpheus, and that pre-built Docker containers can be used or built from source, but it does not provide further details on their purpose or functionality.
+
+However, based on the general knowledge of Docker containers, it can be inferred that the Docker container is likely used to provide a self-contained environment for running the Morpheus pipeline, including the data processing stages and modules such as the `file_batcher`. But without more specific information, it's difficult to provide a more detailed answer.
+
+So, the answer is: I don't know.
+ + Source Documents + + | ID | Type | source | Page Content | +| --- | --- | --- | --- | +| None | Document | examples/llm/rag/README.md |
View Content#### Data Input Points The pipeline has multiple data input avenues: 1. \*\*User Uploaded Documents\*\*: Raw documents provided by users for further processing. 2. \*\*Streaming Event Logs\*\*: Logs that are streamed in real-time. 3. \*\*Streaming Data Feeds\*\*: Continuous streams of data that could be dynamic in nature. 4. \*\*RSS Threat Intel Feeds\*\*: RSS-based feeds that might focus on threat intelligence. 5. \*\*LLM Input Query\*\*: Queries that are inputted for processing by the Large Language Model. #### Data Processing Stages The ingested data traverses several processing stages:
| +| None | Document | docs/source/index.rst |
View ContentGetting Started --------------- Using Morpheus ^^^^^^^^^^^^^^ \* :doc:`getting\_started` - Using pre-built Docker containers, building Docker containers from source, and fetching models and datasets \* :doc:`basics/overview` - Brief overview of the command line interface \* :doc:`basics/building\_a\_pipeline` - Introduction to building a pipeline using the command line interface \* :doc:`Morpheus Examples <examples>` - Example pipelines using both the Python API and command line interface \* :doc:`Pretrained Models <models\_and\_datasets>` - Pretrained models with corresponding training, validation scripts, and datasets \* :doc:`Developer Guides <developer\_guide/guides>` - Covers extending Morpheus with custom stages
| +| None | Document | docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md |
View ContentThe `file\_batcher` module is a component that is responsible for loading input files, filtering out files older than the specified time window, and grouping the remaining files by periods that fall within the time window. This module offers configurability for parameters such as batching options, cache directory, file type, filtering null values, data schema, and the timestamp column name. The `file\_batcher` module processes control messages, validates them, and generates a list of files with their timestamps. The module then groups files by the given period, creates control messages for each batch, and sends them downstream for further processing. A node function is used to handle the processing of control messages, and input and output ports are registered to integrate the module into
| +| None | Document | docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md |
View Contentprocessing. A node function is used to handle the processing of control messages, and input and output ports are registered to integrate the module into the data processing pipeline seamlessly.
| + + + + +## Step 2 : Review Error Handling Mechanisms + + +> **Input**: *Review Error Handling Mechanisms: Since the vendor's perspective is that this is neither a vulnerability nor a bug, but rather an expected behavior when limits are exceeded, review the application's error handling mechanisms to ensure they can properly handle `RecursionError` exceptions without causing unintended behavior or crashes.* + +> **Response**: *The Docker container's error handling mechanisms are not explicitly designed to handle `RecursionError` exceptions. However, the container's code does include some error handling mechanisms, such as raising `ValueError` exceptions in certain cases and catching `NotThisMethod` exceptions. To handle `RecursionError` exceptions, it is recommended to ensure that recursive functions have a terminating condition and to avoid exceeding the maximum recursion depth. Additionally, exceptions can be handled using try-except blocks.* + +### Step 2.1 : *Docker Container Code QA System* + + +#### Action Log +
Thought: To review the error handling mechanisms of the Docker container, I need to understand how the container handles `RecursionError` exceptions. I should first check if the container's code or any of its dependencies use a function or component that could potentially raise a `RecursionError`.
+
+Action: Docker Container Code QA System
+Action Input: Does the Docker container's code or any of its dependencies use a function or component that could raise a `RecursionError`?
+
+ + +#### Tool Input +
Does the Docker container's code or any of its dependencies use a function or component that could raise a `RecursionError`?
+
+ + +#### Tool Output +
I don't know.
+ + Source Documents + + | ID | Type | source |... content_type |... language | Page Content | +| --- | --- | --- | --- | --- | --- | +| None | Document | morpheus/service/vdb/milvus_vector_db_service.py | functions_classes | python |
View Content- 'partition\_name' (str, optional): Required when dropping a specific partition within a collection. Specifies the partition name to be dropped. - 'field\_name' (str, optional): Required when dropping an index within a collection. Specifies the field name for which the index is created. - 'index\_name' (str, optional): Required when dropping an index within a collection. Specifies the name of the index to be dropped. Raises ------ ValueError If mandatory arguments are missing or if the provided 'collection' value is invalid. """ logger.debug("Dropping collection: %s, kwargs=%s", name, kwargs)
| +| None | Document | versioneer.py | functions_classes | python |
View Contentfrom\_vcs\_f = handlers.get("pieces\_from\_vcs") if from\_vcs\_f: try: pieces = from\_vcs\_f(cfg.tag\_prefix, root, verbose) ver = render(pieces, cfg.style) if verbose: print("got version from VCS %s" % ver) return ver except NotThisMethod: pass try: if cfg.parentdir\_prefix: ver = versions\_from\_parentdir(cfg.parentdir\_prefix, root, verbose) if verbose: print("got version from parentdir %s" % ver) return ver except NotThisMethod: pass if verbose: print("unable to compute version")
| +| None | Document | models/training-tuning-scripts/root-cause-models/root_cause_bert.py | simplified_code | python |
View Content# See the License for the specific language governing permissions and # limitations under the License. """ Example Usage: python root\_cause\_bert.py \ --trainingdata ../../datasets/training-data/root-cause-training-data.csv \ --unseenerrors ../../datasets/training-data/root-cause-unseen-errors.csv """
| +| None | Document | morpheus/utils/execution_chain.py | functions_classes | python |
View ContentReturns ------- Any The result of the last function in the chain. Raises ------ Exception If the execution fails at any function in the chain. """ local\_state = {} local\_state.update(kwargs) try: returned\_state = None for function in self.functions: signature = inspect.signature(function) func\_kwargs = {key: value for key, value in local\_state.items() if key in signature.parameters} returned\_state = function(\*\*func\_kwargs) local\_state.update(returned\_state)
| + + + + +[back to top](#checklist-toc) + +### Step 2.2 : *Docker Container Developer Guide QA System* + + +#### Action Log +
Thought: The Docker Container Code QA System was unable to find any specific information about the container's code or its dependencies using a function or component that could raise a `RecursionError`. However, it did return some information about error handling in the container's code. For example, in the `morpheus/service/vdb/milvus_vector_db_service.py` file, it appears that the `drop_collection` function raises a `ValueError` if mandatory arguments are missing or if the provided 'collection' value is invalid. Additionally, in the `versioneer.py` file, it seems that the `from_vcs_f` function catches a `NotThisMethod` exception and passes it. Furthermore, in the `models/training-tuning-scripts/root-cause-models/root_cause_bert.py` file, there is no obvious error handling mechanism. Lastly, in the `morpheus/utils/execution_chain.py` file, the `execute` function raises an `Exception` if the execution fails at any function in the chain.
+
+Action: Docker Container Developer Guide QA System
+Action Input: What is the purpose and functionality of the Docker container's error handling mechanisms?
+
+ + +#### Tool Input +
What is the purpose and functionality of the Docker container's error handling mechanisms?
+
+ + +#### Tool Output +
Unfortunately, I don't know the answer to this question. The provided context does not mention Docker containers or their error handling mechanisms. It appears to be discussing a model and pipeline for root cause analysis, but does not provide information about Docker containers.
+ + Source Documents + + | ID | Type | source | Page Content | +| --- | --- | --- | --- | +| None | Document | docs/source/developer_guide/guides/2_real_world_phishing.md |
View Contentnecessary validation checks in the constructor, and raising an exception in the case of mis-configuration. This allows us to fail early rather than after the pipeline has started.
| +| None | Document | models/model-cards/root-cause-analysis-model-card.md |
View Content\* The model is primarily designed for testing purposes and serves as a small pre-trained model specifically used to evaluate and validate the Root Cause Analysis pipeline. This model is an example of customized transformer-based root cause analysis. It can be used for pipeline testing purposes. It needs to be re-trained for specific root cause analysis or predictive maintenance needs with the fine-tuning scripts in the repo. The hyperparameters can be optimised to adjust to get the best results with another dataset. The aim is to get the model to predict some false positives that could be previously unknown error types. Users can use this root cause analysis approach with other log types too. If they have known failures in their logs, they can use them to train along with ordinary logs
| +| None | Document | docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md |
View Contentprocessing. A node function is used to handle the processing of control messages, and input and output ports are registered to integrate the module into the data processing pipeline seamlessly.
| +| None | Document | docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md |
View ContentThe pipeline comprises a series of interconnected modules designed to create a versatile split processing pipeline. This design enables the reception and processing of control messages to perform tasks such as inference against observed events using either generic or user-specific models. Additionally, the pipeline can train new models based on aggregated or predefined training data, offering a more adaptable and user-friendly experience. ### Overview
| + + + + +[back to top](#checklist-toc) + +### Step 2.3 : *Internet Search* + + +#### Action Log +
Thought: The Docker Container Developer Guide QA System was unable to find any specific information about the Docker container's error handling mechanisms. However, it did return some information about error handling in general. For example, in the `docs/source/developer_guide/guides/2_real_world_phishing.md` file, it appears that the pipeline has necessary validation checks in the constructor and raises an exception in the case of mis-configuration. Additionally, in the `models/model-cards/root-cause-analysis-model-card.md` file, it seems that the model is designed for testing purposes and can be used for pipeline testing purposes. Furthermore, in the `docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md` file, the pipeline comprises a series of interconnected modules designed to create a versatile split processing pipeline.
+
+Action: Internet Search
+Action Input: How to handle RecursionError exceptions in Python
+
+ + +#### Tool Input +
How to handle RecursionError exceptions in Python
+
+ + +#### Tool Output +
- The Python RecursionError is an exception that occurs when the maximum recursion depth is exceeded. Learn how to fix it.
+- I have a reference to a function within an error. That is to say, if an error occurs, then the code re-references the function it is entered it runs again.
+- In this article, we will elucidate the Recursionerror In Python through examples, and we will also explore potential approaches to resolve this issue.
+- Make sure you recursive Python function has a terminating condition. If there is one, then consider the size of the data structure input to the ...
+- If an exception occurs during execution of the try clause, the exception may be handled by an except clause. If the exception is not handled by an except clause ...
+- Working on a bot that will write out text from a .CSV file at random, line by line as I run it, until all lines on that csv have been printed out.
+- I have an issue which I believe is caused by hitting the recursion limit inside of a getattr call. However, I have no idea how to debug this.
+- User code can raise built-in exceptions. This can be used to test an exception handler or to report an error condition “just like” the situation in which the ...
+- In this article, we shall look at why this error occurs and how to handle it in Python. To understand this, we need to first look at tail ...
+- The recursion limit is set too high. I think the problem is that fixRecursionLimit assumes that one python stack entry is no larger than 50 empty js function ...
+ + + +## Step 3 : Inspect Code for Uncontrolled Recursion + + +> **Input**: *Inspect Code for Uncontrolled Recursion: Although the vulnerable package version is already verified to be installed, inspect the application code to identify any instances of uncontrolled recursion that could be exacerbated by the `email.utils.parseaddr` function's behavior, potentially leading to resource exhaustion or crashes.* + +> **Response**: *The Docker container does not appear to be vulnerable to uncontrolled recursion due to the `email.utils.parseaddr` function, as it is not used in the application code. However, there is a potential source of uncontrolled recursion in the `ProducerConsumerQueue` class due to the use of a recursive lock.* + +### Step 3.1 : *Docker Container Code QA System* + + +#### Action Log +
Thought: To answer this question, I need to inspect the application code within the Docker container to identify any instances of uncontrolled recursion that could be exacerbated by the `email.utils.parseaddr` function's behavior. I should use the Docker Container Code QA System to search for any usage of the `email.utils.parseaddr` function and analyze the surrounding code for potential recursion issues.
+
+Action: Docker Container Code QA System
+Action Input: Does the application code use the `email.utils.parseaddr` function, and if so, are there any instances of uncontrolled recursion in the surrounding code?
+
+ + +#### Tool Input +
Does the application code use the `email.utils.parseaddr` function, and if so, are there any instances of uncontrolled recursion in the surrounding code?
+
+ + +#### Tool Output +
No, the provided code does not use the `email.utils.parseaddr` function. Additionally, there is no indication of uncontrolled recursion in the surrounding code. The code appears to be related to file watching, data processing, and logging, but does not involve email parsing or recursive function calls that could lead to uncontrolled recursion.
+ + Source Documents + + | ID | Type | source |... content_type |... language | Page Content | +| --- | --- | --- | --- | --- | --- | +| None | Document | morpheus/service/vdb/milvus_vector_db_service.py | functions_classes | python |
View ContentReturns ------- typing.Any The search result, which can vary depending on the query and options. Raises ------ RuntimeError If an error occurs during the search operation. If query argument is `None` and `data` keyword argument doesn't exist. If `data` keyword arguement is `None`. """ logger.debug("Searching in collection: %s, query=%s, kwargs=%s", self.\_name, query, kwargs) return self.\_client.query(collection\_name=self.\_name, filter=query, \*\*kwargs)
| +| None | Document | morpheus/stages/input/appshield_source_stage.py | functions_classes | python |
View Contentself.\_encoding = encoding self.\_input\_count = None self.\_watcher = DirectoryWatcher(input\_glob=input\_glob, watch\_directory=watch\_directory, max\_files=max\_files, sort\_glob=sort\_glob, recursive=recursive, queue\_max\_size=queue\_max\_size, batch\_timeout=batch\_timeout) @property def name(self) -> str: return "from-appshield" @property def input\_count(self) -> int: """Return None for no max intput count""" return self.\_input\_count def supports\_cpp\_node(self): return False
| +| None | Document | models/training-tuning-scripts/dfp-models/dfp-feature-selection-demo.ipynb | | |
View Content"... ... \n", "3234 External security challenge was not satisfied. \n", "3235 NaN \n", "3236 NaN \n", "3237 NaN \n", "3238 NaN \n", "\n", " properties.clientAppUsed \\\n", "0 Mobile Apps and Desktop clients \n", "1 Browser \n", "2 Mobile Apps and Desktop clients \n", "3 Mobile Apps and Desktop clients \n", "4 Mobile Apps and Desktop clients \n", "... ... \n",
| +| None | Document | morpheus/stages/input/azure_source_stage.py | functions_classes | python |
View ContentIf true the list of files matching `input\_glob` will be processed in sorted order. recursive: bool, default = True If true, events will be emitted for the files in subdirectories that match `input\_glob`. queue\_max\_size: int, default = 128 Maximum queue size to hold the file paths to be processed that match `input\_glob`. batch\_timeout: float, default = 5.0 Timeout to retrieve batch messages from the queue. """
| + + + + +[back to top](#checklist-toc) + +### Step 3.2 : *Docker Container Code QA System* + + +#### Action Log +
Thought: Based on the observation from the Docker Container Code QA System, it appears that the application code does not use the `email.utils.parseaddr` function, and there is no indication of uncontrolled recursion in the surrounding code. This suggests that the vulnerability related to uncontrolled recursion in the `email.utils.parseaddr` function may not be applicable to this Docker container.
+
+However, to further confirm this, I would like to investigate if there are any other functions or components in the application code that could potentially lead to uncontrolled recursion.
+
+Action: Docker Container Code QA System
+Action Input: Are there any other functions or components in the application code that could potentially lead to uncontrolled recursion?
+
+ + +#### Tool Input +
Are there any other functions or components in the application code that could potentially lead to uncontrolled recursion?
+
+ + +#### Tool Output +
Based on the provided code, there is one potential source of uncontrolled recursion: the use of a recursive lock (`threading.RLock`) in the `ProducerConsumerQueue` class. Recursive locks can lead to uncontrolled recursion if not used carefully, as they allow a thread to acquire the lock multiple times without releasing it.
+
+However, in this specific implementation, the recursive lock is used to prevent reentrant deadlocks, which suggests that the developer is aware of the potential risks and has taken steps to mitigate them.
+
+There are no other obvious sources of uncontrolled recursion in the provided code. The `DirectoryWatcher` class is not shown in the code snippet, but it's possible that it could contain recursive functions or methods that could lead to uncontrolled recursion. Without more information, it's impossible to say for sure.
+
+In general, it's always a good idea to carefully review code that uses recursive functions or locks to ensure that they are used safely and correctly.
+ + Source Documents + + | ID | Type | source |... content_type |... language | Page Content | +| --- | --- | --- | --- | --- | --- | +| None | Document | morpheus/controllers/monitor_controller.py | functions_classes | python |
View Contentif isinstance(log\_level, LogLevels): # pylint: disable=isinstance-second-argument-not-valid-type log\_level = log\_level.value self.\_log\_level = log\_level self.\_enabled = None # defined on first call to \_is\_enabled @property def delayed\_start(self) -> bool: return self.\_delayed\_start @property def progress(self) -> tqdm: return self.\_progress def is\_enabled(self) -> bool: """ Returns a boolean indicating whether or not the logger is enabled. """ if self.\_enabled is None: self.\_enabled = logger.isEnabledFor(self.\_log\_level) return self.\_enabled
| +| None | Document | morpheus/utils/producer_consumer_queue.py | functions_classes | python |
View Contentclass ProducerConsumerQueue(queue.Queue, typing.Generic[\_T]): """ Custom queue.Queue implementation which supports closing and uses recursive locks Parameters ---------- maxsize : int Maximum size of queue. If maxsize is <= 0, the queue size is infinite. """ def \_\_init\_\_(self, maxsize: int = 0) -> None: super().\_\_init\_\_(maxsize=maxsize) # Use a recursive lock here to prevent reentrant deadlocks self.mutex = threading.RLock() self.not\_empty = threading.Condition(self.mutex) self.not\_full = threading.Condition(self.mutex) self.all\_tasks\_done = threading.Condition(self.mutex) self.\_is\_closed = False
| +| None | Document | morpheus/stages/input/appshield_source_stage.py | functions_classes | python |
View Contentself.\_encoding = encoding self.\_input\_count = None self.\_watcher = DirectoryWatcher(input\_glob=input\_glob, watch\_directory=watch\_directory, max\_files=max\_files, sort\_glob=sort\_glob, recursive=recursive, queue\_max\_size=queue\_max\_size, batch\_timeout=batch\_timeout) @property def name(self) -> str: return "from-appshield" @property def input\_count(self) -> int: """Return None for no max intput count""" return self.\_input\_count def supports\_cpp\_node(self): return False
| +| None | Document | morpheus/utils/directory_watcher.py | functions_classes | python |
View Contentmax\_files: int Max number of files to read. Useful for debugging to limit startup time. Default value of -1 is unlimited. sort\_glob : bool If true the list of files matching `input\_glob` will be processed in sorted order. recursive: bool If true, events will be emitted for the files in subdirectories matching `input\_glob`. queue\_max\_size: int Maximum queue size to hold the file paths to be processed that match `input\_glob`. batch\_timeout: float Timeout to retrieve batch messages from the queue. """
| + + + + +[back to top](#checklist-toc) + +--- +## References +Here are key references for further details on this vulnerability: +- https://www.cve.org/CVERecord?id=CVE-2023-36632 +- https://github.com/python/cpython/issues/103800 +- https://github.com/Daybreak2019/PoC_python3.9_Vul/blob/main/RecursionError-email.utils.parseaddr.py +- https://docs.python.org/3/library/email.html +- https://docs.python.org/3/library/email.utils.html +- https://github.com/advisories/GHSA-gv66-v8c8-v69c +- https://nvd.nist.gov/vuln/detail/CVE-2023-36632 + +[back to top](#cve-intro) From be4b26f553b357602c9fab37735179efc92fadf5 Mon Sep 17 00:00:00 2001 From: Katherine Huang Date: Mon, 13 Oct 2025 12:51:56 -0700 Subject: [PATCH 043/104] update accidentally removed CVE markdown report to be same as original version --- .../CVE-2023-36632.md | 74 +++++++++---------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/.tmp/vulnerability_markdown_reports/CVE-2023-36632.md b/.tmp/vulnerability_markdown_reports/CVE-2023-36632.md index 0708cbb9a..0e20a23ea 100644 --- a/.tmp/vulnerability_markdown_reports/CVE-2023-36632.md +++ b/.tmp/vulnerability_markdown_reports/CVE-2023-36632.md @@ -23,13 +23,13 @@ limitations under the License. > **Status:** Exploitable -## CVE-2023-36632 +## CVE-2023-36632 > GHSA-ID: GHSA-gv66-v8c8-v69c `Disputed`[source](https://nvd.nist.gov/vuln/detail/CVE-2023-36632) - The legacy email.utils.parseaddr function in Python through 3.11.4 allows attackers to trigger "RecursionError: maximum recursion depth exceeded while calling a Python object" via a crafted argument. This argument is plausibly an untrusted value from an application's input data that was supposed to contain a name and an e-mail address. NOTE: email.utils.parseaddr is categorized as a Legacy API in the documentation of the Python email package. Applications should instead use the email.parser.BytesParser or email.parser.Parser class. [references](#ref) + The legacy email.utils.parseaddr function in Python through 3.11.4 allows attackers to trigger "RecursionError: maximum recursion depth exceeded while calling a Python object" via a crafted argument. This argument is plausibly an untrusted value from an application's input data that was supposed to contain a name and an e-mail address. NOTE: email.utils.parseaddr is categorized as a Legacy API in the documentation of the Python email package. Applications should instead use the email.parser.BytesParser or email.parser.Parser class. [references](#ref) ### Severity and CVSS Score @@ -39,7 +39,7 @@ limitations under the License. | [**NVD**](https://nvd.nist.gov/vuln/detail/CVE-2023-36632) |HIGH|7.5|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H|2023-06-25|2024-11-21| | [**RHSA**](https://access.redhat.com/security/cve/CVE-2023-36632) |N/A|N/A|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N|2023-06-25|N/A| | [**Ubuntu**](https://ubuntu.com/security/CVE-2023-36632) |HIGH|7.5|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H|2023-06-25|2024-07-24| - + ### EPSS Score | EPSS | Percentile | @@ -51,7 +51,7 @@ limitations under the License. ### Summary (Exploitable) Based on the provided Checklist and Findings, the CVE is exploitable. Specifically, the application does not handle exceptions like `RecursionError` when processing input data (Checklist Item 1), and the Docker container's error handling mechanisms are not explicitly designed to handle `RecursionError` exceptions (Checklist Item 2). Additionally, there is a potential source of uncontrolled recursion in the `ProducerConsumerQueue` class due to the use of a recursive lock (Checklist Item 3). These findings suggest that the CVE is exploitable, as the application and Docker container do not have adequate measures in place to prevent or handle `RecursionError` exceptions. -### Justification +### Justification >label: vulnerable @@ -59,7 +59,7 @@ Based on the provided Checklist and Findings, the CVE is exploitable. Specifical The analysis concludes that the CVE is exploitable due to the application's and Docker container's inadequate handling of `RecursionError` exceptions and the presence of a potential source of uncontrolled recursion, indicating that the package is vulnerable to the CVE and needs to be patched. --- -### Vulnerable SBOM Dependencies +### Vulnerable SBOM Dependencies | SBOM Name | SBOM Version | Dependency Name | Dependency Version | Relation | System | @@ -94,7 +94,7 @@ The analysis concludes that the CVE is exploitable due to the application's and ### Step 1.1 : *Docker Container Code QA System* -#### Action Log +#### Action Log 
Thought: To assess the input data handling of the application, I need to understand how the application processes input data and if it has any mechanisms in place to handle exceptions like `RecursionError`. I also need to check if the application processes potentially untrusted input data that could include crafted arguments designed to exploit this vulnerability.
 
 Action: Docker Container Code QA System
@@ -102,15 +102,15 @@ Action Input: Does the application handle exceptions like `RecursionError` when
-#### Tool Input +#### Tool Input 
Does the application handle exceptions like `RecursionError` when processing input data?
-#### Tool Output +#### Tool Output 
No, the application does not handle exceptions like `RecursionError` when processing input data. According to the provided context, `RecursionError` is only handled in the logging part of the application, specifically when writing log messages. It is not handled in the input data processing part.
- Source Documents + Source Documents | ID | Type | source |... content_type |... language | Page Content | | --- | --- | --- | --- | --- | --- | @@ -127,7 +127,7 @@ Action Input: Does the application handle exceptions like `RecursionError` when ### Step 1.2 : *Docker Container Code QA System* -#### Action Log +#### Action Log 
Thought: The application does not handle exceptions like `RecursionError` when processing input data. However, it does handle `RecursionError` in the logging part of the application. This suggests that the application may not be robust against crafted arguments designed to exploit this vulnerability.
 
 Action: Docker Container Code QA System
@@ -135,19 +135,19 @@ Action Input: Does the application process potentially untrusted input data that
-#### Tool Input +#### Tool Input 
Does the application process potentially untrusted input data that could include crafted arguments designed to exploit this vulnerability?
-#### Tool Output +#### Tool Output 
Based on the provided context, it appears that the application does process input data, specifically `user_inputs` and `input_names`, which are used to create mappings and validate input names. However, it's unclear whether this input data is potentially untrusted or could include crafted arguments designed to exploit a vulnerability.
 
 The tests `ProcessInputNamesMismatchMultipleMappings` and `ProcessInputNamesMismatchSingleMapping` suggest that the application is designed to handle invalid input and throw exceptions when input names do not match. This implies that the application is attempting to validate and sanitize input data, but it's still unclear whether this is sufficient to prevent exploitation.
 
 Therefore, the answer to the question is: I don't know.
- Source Documents + Source Documents | ID | Type | source |... content_type |... language | Page Content | | --- | --- | --- | --- | --- | --- | @@ -164,7 +164,7 @@ Therefore, the answer to the question is: I don't know. ### Step 1.3 : *Docker Container Developer Guide QA System* -#### Action Log +#### Action Log 
Thought: The application does not handle exceptions like `RecursionError` when processing input data, but it does handle `RecursionError` in the logging part of the application. Additionally, the application processes input data, but it's unclear whether this input data is potentially untrusted or could include crafted arguments designed to exploit a vulnerability. The application attempts to validate and sanitize input data, but it's still unclear whether this is sufficient to prevent exploitation.
 
 Action: Docker Container Developer Guide QA System
@@ -172,19 +172,19 @@ Action Input: What is the purpose and functionality of the Docker container, and
-#### Tool Input +#### Tool Input 
What is the purpose and functionality of the Docker container, and how does it handle input data?
-#### Tool Output +#### Tool Output 
Unfortunately, the provided context does not explicitly explain the purpose and functionality of the Docker container or how it handles input data. The context mentions that Docker containers can be used to get started with Morpheus, and that pre-built Docker containers can be used or built from source, but it does not provide further details on their purpose or functionality.
 
 However, based on the general knowledge of Docker containers, it can be inferred that the Docker container is likely used to provide a self-contained environment for running the Morpheus pipeline, including the data processing stages and modules such as the `file_batcher`. But without more specific information, it's difficult to provide a more detailed answer.
 
 So, the answer is: I don't know.
- Source Documents + Source Documents | ID | Type | source | Page Content | | --- | --- | --- | --- | @@ -206,7 +206,7 @@ So, the answer is: I don't know. ### Step 2.1 : *Docker Container Code QA System* -#### Action Log +#### Action Log 
Thought: To review the error handling mechanisms of the Docker container, I need to understand how the container handles `RecursionError` exceptions. I should first check if the container's code or any of its dependencies use a function or component that could potentially raise a `RecursionError`.
 
 Action: Docker Container Code QA System
@@ -214,15 +214,15 @@ Action Input: Does the Docker container's code or any of its dependencies use a
-#### Tool Input +#### Tool Input 
Does the Docker container's code or any of its dependencies use a function or component that could raise a `RecursionError`?
-#### Tool Output +#### Tool Output 
I don't know.
- Source Documents + Source Documents | ID | Type | source |... content_type |... language | Page Content | | --- | --- | --- | --- | --- | --- | @@ -239,7 +239,7 @@ Action Input: Does the Docker container's code or any of its dependencies use a ### Step 2.2 : *Docker Container Developer Guide QA System* -#### Action Log +#### Action Log 
Thought: The Docker Container Code QA System was unable to find any specific information about the container's code or its dependencies using a function or component that could raise a `RecursionError`. However, it did return some information about error handling in the container's code. For example, in the `morpheus/service/vdb/milvus_vector_db_service.py` file, it appears that the `drop_collection` function raises a `ValueError` if mandatory arguments are missing or if the provided 'collection' value is invalid. Additionally, in the `versioneer.py` file, it seems that the `from_vcs_f` function catches a `NotThisMethod` exception and passes it. Furthermore, in the `models/training-tuning-scripts/root-cause-models/root_cause_bert.py` file, there is no obvious error handling mechanism. Lastly, in the `morpheus/utils/execution_chain.py` file, the `execute` function raises an `Exception` if the execution fails at any function in the chain.
 
 Action: Docker Container Developer Guide QA System
@@ -247,15 +247,15 @@ Action Input: What is the purpose and functionality of the Docker container's er
-#### Tool Input +#### Tool Input 
What is the purpose and functionality of the Docker container's error handling mechanisms?
-#### Tool Output +#### Tool Output 
Unfortunately, I don't know the answer to this question. The provided context does not mention Docker containers or their error handling mechanisms. It appears to be discussing a model and pipeline for root cause analysis, but does not provide information about Docker containers.
- Source Documents + Source Documents | ID | Type | source | Page Content | | --- | --- | --- | --- | @@ -272,7 +272,7 @@ Action Input: What is the purpose and functionality of the Docker container's er ### Step 2.3 : *Internet Search* -#### Action Log +#### Action Log 
Thought: The Docker Container Developer Guide QA System was unable to find any specific information about the Docker container's error handling mechanisms. However, it did return some information about error handling in general. For example, in the `docs/source/developer_guide/guides/2_real_world_phishing.md` file, it appears that the pipeline has necessary validation checks in the constructor and raises an exception in the case of mis-configuration. Additionally, in the `models/model-cards/root-cause-analysis-model-card.md` file, it seems that the model is designed for testing purposes and can be used for pipeline testing purposes. Furthermore, in the `docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md` file, the pipeline comprises a series of interconnected modules designed to create a versatile split processing pipeline.
 
 Action: Internet Search
@@ -280,12 +280,12 @@ Action Input: How to handle RecursionError exceptions in Python
-#### Tool Input +#### Tool Input 
How to handle RecursionError exceptions in Python
-#### Tool Output +#### Tool Output 
- The Python RecursionError is an exception that occurs when the maximum recursion depth is exceeded. Learn how to fix it.
 - I have a reference to a function within an error. That is to say, if an error occurs, then the code re-references the function it is entered it runs again.
 - In this article, we will elucidate the Recursionerror In Python through examples, and we will also explore potential approaches to resolve this issue.
@@ -309,7 +309,7 @@ Action Input: How to handle RecursionError exceptions in Python
 ### Step 3.1 : *Docker Container Code QA System*
 
 
-#### Action Log
+#### Action Log 
 
Thought: To answer this question, I need to inspect the application code within the Docker container to identify any instances of uncontrolled recursion that could be exacerbated by the `email.utils.parseaddr` function's behavior. I should use the Docker Container Code QA System to search for any usage of the `email.utils.parseaddr` function and analyze the surrounding code for potential recursion issues.
 
 Action: Docker Container Code QA System
@@ -317,15 +317,15 @@ Action Input: Does the application code use the `email.utils.parseaddr` function
  

 
 
-#### Tool Input
+#### Tool Input 
 
Does the application code use the `email.utils.parseaddr` function, and if so, are there any instances of uncontrolled recursion in the surrounding code?
  

 
 
-#### Tool Output
+#### Tool Output 
 
No, the provided code does not use the `email.utils.parseaddr` function. Additionally, there is no indication of uncontrolled recursion in the surrounding code. The code appears to be related to file watching, data processing, and logging, but does not involve email parsing or recursive function calls that could lead to uncontrolled recursion.

 
- Source Documents
+ Source Documents 
 
  | ID | Type | source |... content_type |... language | Page Content |
 | --- | --- | --- | --- | --- | --- |
@@ -342,7 +342,7 @@ Action Input: Does the application code use the `email.utils.parseaddr` function
 ### Step 3.2 : *Docker Container Code QA System*
 
 
-#### Action Log
+#### Action Log 
 
Thought: Based on the observation from the Docker Container Code QA System, it appears that the application code does not use the `email.utils.parseaddr` function, and there is no indication of uncontrolled recursion in the surrounding code. This suggests that the vulnerability related to uncontrolled recursion in the `email.utils.parseaddr` function may not be applicable to this Docker container.
 
 However, to further confirm this, I would like to investigate if there are any other functions or components in the application code that could potentially lead to uncontrolled recursion.
@@ -352,12 +352,12 @@ Action Input: Are there any other functions or components in the application cod
  

 
 
-#### Tool Input
+#### Tool Input 
 
Are there any other functions or components in the application code that could potentially lead to uncontrolled recursion?
  

 
 
-#### Tool Output
+#### Tool Output 
 
Based on the provided code, there is one potential source of uncontrolled recursion: the use of a recursive lock (`threading.RLock`) in the `ProducerConsumerQueue` class. Recursive locks can lead to uncontrolled recursion if not used carefully, as they allow a thread to acquire the lock multiple times without releasing it.
 
 However, in this specific implementation, the recursive lock is used to prevent reentrant deadlocks, which suggests that the developer is aware of the potential risks and has taken steps to mitigate them.
@@ -366,7 +366,7 @@ There are no other obvious sources of uncontrolled recursion in the provided cod
 
 In general, it's always a good idea to carefully review code that uses recursive functions or locks to ensure that they are used safely and correctly.

 
- Source Documents
+ Source Documents 
 
  | ID | Type | source |... content_type |... language | Page Content |
 | --- | --- | --- | --- | --- | --- |
@@ -381,7 +381,7 @@ In general, it's always a good idea to carefully review code that uses recursive
 [back to top](#checklist-toc)
 
 ---
-## References 
+## References   
 Here are key references for further details on this vulnerability:
 - https://www.cve.org/CVERecord?id=CVE-2023-36632
 - https://github.com/python/cpython/issues/103800

From 93be0506d46de8bff6d5d2f2954a9f56253b5d9e Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 13 Oct 2025 12:54:17 -0700
Subject: [PATCH 044/104] moved markdown report back to original dir

---
 .../CVE-2023-36632.md                                             | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename .tmp/vulnerability_markdown_reports/{ => vulnerability_reports_morpheus_23_11-runtime}/CVE-2023-36632.md (100%)

diff --git a/.tmp/vulnerability_markdown_reports/CVE-2023-36632.md b/.tmp/vulnerability_markdown_reports/vulnerability_reports_morpheus_23_11-runtime/CVE-2023-36632.md
similarity index 100%
rename from .tmp/vulnerability_markdown_reports/CVE-2023-36632.md
rename to .tmp/vulnerability_markdown_reports/vulnerability_reports_morpheus_23_11-runtime/CVE-2023-36632.md

From 881087fa4a4464bb3cf85b9d32919ba5bbddf743 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 13 Oct 2025 12:57:55 -0700
Subject: [PATCH 045/104] fix formatting of function description

---
 src/vuln_analysis/functions/cve_justify.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/vuln_analysis/functions/cve_justify.py b/src/vuln_analysis/functions/cve_justify.py
index 3e338af51..3d3de2cf2 100644
--- a/src/vuln_analysis/functions/cve_justify.py
+++ b/src/vuln_analysis/functions/cve_justify.py
@@ -29,7 +29,7 @@
 class CVEJustifyToolConfig(FunctionBaseConfig, name="cve_justify"):
     """
     Defines a function that assigns justification label and reason to each CVE based on summary.
-"""
+    """
     llm_name: str = Field(description="The LLM model to use")
     max_concurrency: int | None = Field(ge=1,
                                         default=None,

From 583875dd0ec4484bb680346a041d74b7e6ae01c9 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Fri, 17 Oct 2025 11:34:38 -0700
Subject: [PATCH 046/104] add license to all files

---
 src/vuln_analysis/data_models/eval_input.py | 15 +++++++++++++++
 src/vuln_analysis/eval/parse_eval_input.py  | 15 +++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/src/vuln_analysis/data_models/eval_input.py b/src/vuln_analysis/data_models/eval_input.py
index 3433f7c8f..ea5f6106a 100644
--- a/src/vuln_analysis/data_models/eval_input.py
+++ b/src/vuln_analysis/data_models/eval_input.py
@@ -1,3 +1,18 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from typing import Literal
 
 from pydantic import BaseModel
diff --git a/src/vuln_analysis/eval/parse_eval_input.py b/src/vuln_analysis/eval/parse_eval_input.py
index cdc37abe4..874c10eef 100644
--- a/src/vuln_analysis/eval/parse_eval_input.py
+++ b/src/vuln_analysis/eval/parse_eval_input.py
@@ -1,3 +1,18 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import json
 from pathlib import Path
 

From 9565d6547002b19cffde668516505567095b3b16 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Fri, 17 Oct 2025 12:08:10 -0700
Subject: [PATCH 047/104] minor formatting fixes

---
 src/vuln_analysis/configs/config-eval.yml     |  2 +-
 src/vuln_analysis/eval/evaluators/accuracy.py | 40 +++++++++----------
 .../eval/evaluators/consistency.py            | 29 +++++++-------
 src/vuln_analysis/register.py                 |  3 +-
 4 files changed, 36 insertions(+), 38 deletions(-)

diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml
index 218e4af45..f6a42dfc4 100644
--- a/src/vuln_analysis/configs/config-eval.yml
+++ b/src/vuln_analysis/configs/config-eval.yml
@@ -184,7 +184,7 @@ eval:
       _type: accuracy
       field: status
       multiple_reps: False
-      conflict_policy: strict # options: strict, keep_first, lenient
+      conflict_policy: strict  # Options: strict, keep_first, lenient
     label_accuracy:
       _type: accuracy
       field: label
diff --git a/src/vuln_analysis/eval/evaluators/accuracy.py b/src/vuln_analysis/eval/evaluators/accuracy.py
index 27480b38d..73fbe6058 100644
--- a/src/vuln_analysis/eval/evaluators/accuracy.py
+++ b/src/vuln_analysis/eval/evaluators/accuracy.py
@@ -31,6 +31,25 @@
 _STATUS_MAP = {"TRUE": "AFFECTED", "FALSE": "NOT AFFECTED", "UNKNOWN": "UNKNOWN"}
 
 
+class AccuracyEvaluatorConfig(EvaluatorBaseConfig, name="accuracy"):
+    """Configuration for unified accuracy evaluator"""
+    field: str = "status"  # "status" or "label"
+    multiple_reps: bool = False
+    conflict_policy: str = "keep_first"  # "strict", "keep_first", or "lenient"
+
+
+@register_evaluator(config_type=AccuracyEvaluatorConfig)
+async def accuracy_evaluator(config: AccuracyEvaluatorConfig, builder: EvalBuilder):
+    """Register unified accuracy evaluator"""
+    evaluator = AccuracyEvaluator(builder.get_max_concurrency(),
+                                  field=config.field,
+                                  multiple_reps=config.multiple_reps,
+                                  conflict_policy=config.conflict_policy)
+    yield EvaluatorInfo(config=config,
+                        evaluate_fn=evaluator.evaluate,
+                        description=f"{config.field.title()} Accuracy Evaluator")
+
+
 class AccuracyEvaluator:
     '''Configurable accuracy evaluator for either the status or label field.'''
 
@@ -56,7 +75,7 @@ def _preprocess_ground_truth(ground_truth_list, field: str, policy: str):
             - keep_first: take the first encountered answer
             - lenient: any answer in ground truth is considered acceptable
             """
-            # Collect ground truthanswers per vuln_id in encounter order
+            # Collect ground truth answers per vuln_id in encounter order
             encountered_order = {}
             answers_per_id = {}
             for idx, gt in enumerate(ground_truth_list):
@@ -200,22 +219,3 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict, int, int]:
         ]
 
         return EvalOutput(average_score=final_output, eval_output_items=eval_output_items)
-
-
-class AccuracyEvaluatorConfig(EvaluatorBaseConfig, name="accuracy"):
-    """Configuration for unified accuracy evaluator"""
-    field: str = "status"  # "status" or "label"
-    multiple_reps: bool = False
-    conflict_policy: str = "keep_first"  # "strict", "keep_first", or "lenient"
-
-
-@register_evaluator(config_type=AccuracyEvaluatorConfig)
-async def accuracy_evaluator(config: AccuracyEvaluatorConfig, builder: EvalBuilder):
-    """Register unified accuracy evaluator"""
-    evaluator = AccuracyEvaluator(builder.get_max_concurrency(),
-                                  field=config.field,
-                                  multiple_reps=config.multiple_reps,
-                                  conflict_policy=config.conflict_policy)
-    yield EvaluatorInfo(config=config,
-                        evaluate_fn=evaluator.evaluate,
-                        description=f"{config.field.title()} Accuracy Evaluator")
diff --git a/src/vuln_analysis/eval/evaluators/consistency.py b/src/vuln_analysis/eval/evaluators/consistency.py
index dcf26305c..5690b6728 100644
--- a/src/vuln_analysis/eval/evaluators/consistency.py
+++ b/src/vuln_analysis/eval/evaluators/consistency.py
@@ -28,6 +28,20 @@
 from tqdm import tqdm
 
 
+class ConsistencyEvaluatorConfig(EvaluatorBaseConfig, name="consistency"):
+    """Configuration for consistency evaluator"""
+    field: str = "label"  # "label" or "status"
+
+
+@register_evaluator(config_type=ConsistencyEvaluatorConfig)
+async def consistency_evaluator(config: ConsistencyEvaluatorConfig, builder: EvalBuilder):
+    """Register consistency evaluator"""
+    evaluator = ConsistencyEvaluator(builder.get_max_concurrency(), field=config.field)
+    yield EvaluatorInfo(config=config,
+                        evaluate_fn=evaluator.evaluate,
+                        description=f"{config.field.title()} Consistency Evaluator")
+
+
 class ConsistencyEvaluator:
     '''Evaluator class for measuring consistency of the specified field via the agreement probability metric ∑p², where p is the probability of each unique value'''
 
@@ -104,18 +118,3 @@ async def extract_run_data(item):
         ]
 
         return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items)
-
-
-# Register custom evaluator in NAT
-class ConsistencyEvaluatorConfig(EvaluatorBaseConfig, name="consistency"):
-    """Configuration for consistency evaluator"""
-    field: str = "label"  # "label" or "status"
-
-
-@register_evaluator(config_type=ConsistencyEvaluatorConfig)
-async def consistency_evaluator(config: ConsistencyEvaluatorConfig, builder: EvalBuilder):
-    """Register consistency evaluator"""
-    evaluator = ConsistencyEvaluator(builder.get_max_concurrency(), field=config.field)
-    yield EvaluatorInfo(config=config,
-                        evaluate_fn=evaluator.evaluate,
-                        description=f"{config.field.title()} Consistency Evaluator")
diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py
index 8322067c4..9c55cbc81 100644
--- a/src/vuln_analysis/register.py
+++ b/src/vuln_analysis/register.py
@@ -28,10 +28,9 @@
 from vuln_analysis.data_models.input import AgentMorpheusInput
 from vuln_analysis.data_models.output import AgentMorpheusOutput
 from vuln_analysis.data_models.state import AgentMorpheusEngineState
-# Import evaluator registrations
+# pylint: disable=unused-import
 from vuln_analysis.eval.evaluators import accuracy
 from vuln_analysis.eval.evaluators import consistency
-# pylint: disable=unused-import
 from vuln_analysis.functions import cve_agent
 from vuln_analysis.functions import cve_check_vuln_deps
 from vuln_analysis.functions import cve_checklist

From 881be7659db26c071af3abfe530bee98c836dc39 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Fri, 17 Oct 2025 12:30:53 -0700
Subject: [PATCH 048/104] remove separated morpheus test sets, renamed combined
 test set to eval_dataset, removed eval datasets directory

---
 README.md                                     |  4 +-
 src/vuln_analysis/configs/config-eval.yml     |  2 +-
 ...common-test-set.json => eval_dataset.json} |  0
 .../morpheus-23.11-test-set.json              | 71 -----------------
 .../morpheus-24.03-test-set.json              | 76 -------------------
 5 files changed, 3 insertions(+), 150 deletions(-)
 rename src/vuln_analysis/data/{eval_datasets/common-test-set.json => eval_dataset.json} (100%)
 delete mode 100644 src/vuln_analysis/data/eval_datasets/morpheus-23.11-test-set.json
 delete mode 100644 src/vuln_analysis/data/eval_datasets/morpheus-24.03-test-set.json

diff --git a/README.md b/README.md
index f8e7f4ce4..20917a2f5 100644
--- a/README.md
+++ b/README.md
@@ -807,10 +807,10 @@ containers:
 
 In `config-eval.yml`, the `file_path` field should specify the specific test set to use.
 
-An example evaluation experiment has been set up to run the `test-set-common` test set on the morpheus_23.11-runtime and morpheus_24.03-runtime images. Note that in order to run a test set across multiple containers, each container must contain exactly the same test set of CVEs present in every other container (ie. a container cannot contain only a subset of the full test set).
+An example evaluation experiment has been set up to run the `eval_dataset` test set on the morpheus_23.11-runtime and morpheus_24.03-runtime images. Note that in order to run a test set across multiple containers, each container must contain exactly the same test set of CVEs present in every other container (ie. a container cannot contain only a subset of the full test set).
 
 ```yml
-file_path: data/eval_datasets/common-test-set.json
+file_path: data/eval_dataset.json
 ```
 
 You can also configure the evaluation output directory using the `output_dir` variable.
diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml
index f6a42dfc4..cf91b4131 100644
--- a/src/vuln_analysis/configs/config-eval.yml
+++ b/src/vuln_analysis/configs/config-eval.yml
@@ -176,7 +176,7 @@ eval:
     output_dir: ./.tmp/evaluators
     dataset:
         _type: custom
-        file_path: data/eval_datasets/common-test-set.json
+        file_path: data/eval_dataset.json
         function: vuln_analysis.eval.parse_eval_input.parse_input
 
   evaluators:
diff --git a/src/vuln_analysis/data/eval_datasets/common-test-set.json b/src/vuln_analysis/data/eval_dataset.json
similarity index 100%
rename from src/vuln_analysis/data/eval_datasets/common-test-set.json
rename to src/vuln_analysis/data/eval_dataset.json
diff --git a/src/vuln_analysis/data/eval_datasets/morpheus-23.11-test-set.json b/src/vuln_analysis/data/eval_datasets/morpheus-23.11-test-set.json
deleted file mode 100644
index 0eb118a56..000000000
--- a/src/vuln_analysis/data/eval_datasets/morpheus-23.11-test-set.json
+++ /dev/null
@@ -1,71 +0,0 @@
-{
-    "test_set_description": "Test set for morpheus-23.11 container",
-    "containers": {
-        "morpheus-23.11": {
-            "container_image": {
-                "name": "nvcr.io/nvidia/morpheus/morpheus",
-                "tag": "23.11-runtime",
-                "source_info": [
-                    {
-                        "type": "code",
-                        "git_repo": "https://github.com/nv-morpheus/Morpheus.git",
-                        "ref": "branch-23.11",
-                        "include": [
-                            "**/*.cpp",
-                            "**/*.cu",
-                            "**/*.cuh",
-                            "**/*.h",
-                            "**/*.hpp",
-                            "**/*.ipynb",
-                            "**/*.py",
-                            "**/*Dockerfile"
-                        ],
-                        "exclude": [
-                            "tests/**/*"
-                        ]
-                    },
-                    {
-                        "type": "doc",
-                        "git_repo": "https://github.com/nv-morpheus/Morpheus.git",
-                        "ref": "branch-23.11",
-                        "include": [
-                            "**/*.md",
-                            "docs/**/*.rst"
-                        ]
-                    }
-                ],
-                "sbom_info": {
-                    "_type": "file",
-                    "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom"
-                }
-            },
-            "ground_truth": [
-                {
-                    "vuln_id": "GHSA-3f63-hfp8-52jq",
-                    "status": "NOT AFFECTED",
-                    "label": "code_not_reachable"
-                },
-                {
-                    "vuln_id": "CVE-2023-36632",
-                    "status": "AFFECTED",
-                    "label": "vulnerable"
-                },
-                {
-                    "vuln_id": "GHSA-cxfr-5q3r-2rc2",
-                    "status": "AFFECTED",
-                    "label": "vulnerable"
-                },
-                {
-                    "vuln_id": "GHSA-554w-xh4j-8w64",
-                    "status": "AFFECTED",
-                    "label": "vulnerable"
-                },
-                {
-                    "vuln_id": "CVE-2023-51767",
-                    "status": "AFFECTED",
-                    "label": "vulnerable"
-                }
-            ]
-        }
-    }
-}
diff --git a/src/vuln_analysis/data/eval_datasets/morpheus-24.03-test-set.json b/src/vuln_analysis/data/eval_datasets/morpheus-24.03-test-set.json
deleted file mode 100644
index dbdef0b0e..000000000
--- a/src/vuln_analysis/data/eval_datasets/morpheus-24.03-test-set.json
+++ /dev/null
@@ -1,76 +0,0 @@
-{
-    "test_set_description": "Test set for morpheus-24.03 container (toy data)",
-    "containers": {
-        "morpheus-24.03": {
-            "container_image": {
-                "name": "nvcr.io/nvidia/morpheus/morpheus",
-                "tag": "v24.03.02-runtime",
-                "source_info": [
-                    {
-                        "type": "code",
-                        "git_repo": "https://github.com/nv-morpheus/Morpheus.git",
-                        "ref": "v24.03.02",
-                        "include": [
-                            "**/*.cpp",
-                            "**/*.cu",
-                            "**/*.cuh",
-                            "**/*.h",
-                            "**/*.hpp",
-                            "**/*.ipynb",
-                            "**/*.py",
-                            "**/*Dockerfile"
-                        ],
-                        "exclude": [
-                            "tests/**/*"
-                        ]
-                    },
-                    {
-                        "type": "doc",
-                        "git_repo": "https://github.com/nv-morpheus/Morpheus.git",
-                        "ref": "v24.03.02",
-                        "include": [
-                            "**/*.md",
-                            "docs/**/*.rst"
-                        ]
-                    }
-                ],
-                "sbom_info": {
-                    "_type": "file",
-                    "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v24.03.02-runtime.sbom"
-                }
-            },
-            "ground_truth": [
-                {
-                    "vuln_id": "GHSA-3f63-hfp8-52jq",
-                    "status": "NOT AFFECTED",
-                    "label": "code_not_reachable"
-                },
-                {
-                    "vuln_id": "CVE-2023-50782",
-                    "status": "AFFECTED",
-                    "label": "vulnerable"
-                },
-                {
-                    "vuln_id": "CVE-2023-36632",
-                    "status": "AFFECTED",
-                    "label": "vulnerable"
-                },
-                {
-                    "vuln_id": "CVE-2023-43804",
-                    "status": "NOT AFFECTED",
-                    "label": "code_not_reachable"
-                },
-                {
-                    "vuln_id": "GHSA-cxfr-5q3r-2rc2",
-                    "status": "AFFECTED",
-                    "label": "vulnerable"
-                },
-                {
-                    "vuln_id": "GHSA-554w-xh4j-8w64",
-                    "status": "AFFECTED",
-                    "label": "vulnerable"
-                }
-            ]
-        }
-    }
-}

From 99986f3ebd1bc3bebb4668b60a0c7c80afc981d1 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Fri, 17 Oct 2025 12:41:53 -0700
Subject: [PATCH 049/104] change eval data model names to be more descriptive

---
 src/vuln_analysis/data_models/eval_input.py | 6 +++---
 src/vuln_analysis/eval/parse_eval_input.py  | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/vuln_analysis/data_models/eval_input.py b/src/vuln_analysis/data_models/eval_input.py
index ea5f6106a..5a8822bfd 100644
--- a/src/vuln_analysis/data_models/eval_input.py
+++ b/src/vuln_analysis/data_models/eval_input.py
@@ -55,14 +55,14 @@ def validate_vuln_id(cls, v: str) -> str:
         return v
 
 
-class ContainerEval(BaseModel):
+class AgentMorpheusEvalInputItem(BaseModel):
     container_image: ImageInfoInput
     ground_truth: list[GroundTruthItem]
 
 
-class EvalDataset(BaseModel):
+class AgentMorpheusEvalDataset(BaseModel):
     test_set_description: str = Field(default="Unknown test set")
-    containers: dict[str, ContainerEval]
+    containers: dict[str, AgentMorpheusEvalInputItem]
 
     @model_validator(mode="after")
     def validate_consistent_vuln_sets(self):
diff --git a/src/vuln_analysis/eval/parse_eval_input.py b/src/vuln_analysis/eval/parse_eval_input.py
index 874c10eef..b412c1c71 100644
--- a/src/vuln_analysis/eval/parse_eval_input.py
+++ b/src/vuln_analysis/eval/parse_eval_input.py
@@ -19,7 +19,7 @@
 from nat.eval.evaluator.evaluator_model import EvalInput
 from nat.eval.evaluator.evaluator_model import EvalInputItem
 
-from vuln_analysis.data_models.eval_input import EvalDataset
+from vuln_analysis.data_models.eval_input import AgentMorpheusEvalDataset
 
 
 def parse_input(file_path: Path) -> EvalInput:
@@ -33,7 +33,7 @@ def parse_input(file_path: Path) -> EvalInput:
     # Load and validate the test set file using Pydantic models
     with open(file_path, 'r', encoding='utf-8') as f:
         config_data = json.load(f)
-    dataset = EvalDataset.model_validate(config_data)
+    dataset = AgentMorpheusEvalDataset.model_validate(config_data)
 
     # Extract containers
     containers = dataset.containers

From 3bc0140b365bc23095d6bf37a5b37d236ef51505 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Fri, 17 Oct 2025 13:33:21 -0700
Subject: [PATCH 050/104] accuracy evaluator for multiple reps outputs
 DataFrame.describe() metrics

---
 src/vuln_analysis/eval/evaluators/accuracy.py | 32 +++++++++----------
 .../visualizations/box_and_whisker_plot.py    | 17 +++++++---
 2 files changed, 28 insertions(+), 21 deletions(-)

diff --git a/src/vuln_analysis/eval/evaluators/accuracy.py b/src/vuln_analysis/eval/evaluators/accuracy.py
index 73fbe6058..e5dea04df 100644
--- a/src/vuln_analysis/eval/evaluators/accuracy.py
+++ b/src/vuln_analysis/eval/evaluators/accuracy.py
@@ -17,6 +17,7 @@
 import json
 import statistics as stats
 
+import pandas as pd
 from nat.builder.builder import EvalBuilder
 from nat.builder.evaluator import EvaluatorInfo
 from nat.cli.register_workflow import register_evaluator
@@ -189,24 +190,21 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict, int, int]:
         if self.multiple_reps:
             scores_sequential = list(all_averages)
             if scores_sequential:
-                scores_sequential.sort()
-                # Calculating average by averaging each EvalOutputItem's score since for multiple reps, each EvalOutputItem has the same number of CVEs
-                mean_value = round(sum(scores_sequential) / len(scores_sequential), 2)
-                min_value = round(scores_sequential[0], 2)
-                max_value = round(scores_sequential[-1], 2)
-                try:
-                    q1, med, q3 = stats.quantiles(scores_sequential, n=4, method="inclusive")
-                    q1 = round(q1, 2)
-                    med = round(med, 2)
-                    q3 = round(q3, 2)
-                except Exception:  # Fallback for very small samples that may fail to compute quantiles
-                    med = round(stats.median(scores_sequential), 2)
-                    q1 = med
-                    q3 = med
-                box_and_whiskers_values = [mean_value, min_value, q1, med, q3, max_value]
+                series = pd.Series(scores_sequential)
+                desc = series.describe()  # count, mean, std, min, 25%, 50%, 75%, max
+                # Round values and make count an int
+                final_output = {k: (int(v) if k == "count" else round(float(v), 2)) for k, v in desc.items()}
             else:
-                box_and_whiskers_values = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
-            final_output = box_and_whiskers_values
+                final_output = {
+                    "count": 0,
+                    "mean": 0.0,
+                    "std": 0.0,
+                    "min": 0.0,
+                    "25%": 0.0,
+                    "50%": 0.0,
+                    "75%": 0.0,
+                    "max": 0.0,
+                }
         else:  # Single rep
             # Calculating weighted average (each EvalOutputItem's score is weighted by its number of CVEs)
             total_correct_sum = sum(total_correct)
diff --git a/src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py b/src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py
index fb3c700b2..496e3c480 100644
--- a/src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py
+++ b/src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py
@@ -37,12 +37,21 @@ def load_box_values(path: str) -> tuple[list[float], str]:
         data = json.load(f)
 
     avg = data.get("average_score")
-    if not isinstance(avg, list) or len(avg) != 6:
-        raise ValueError(
-            f"File '{path}' does not contain box/whisker values. Expecting list of 6 numbers in 'average_score'.")
+    if not isinstance(avg, dict):
+        raise ValueError(f"File '{path}' does not contain pandas describe() format. Expecting dict in 'average_score'.")
+
+    # Extract values from pandas describe() format: [mean, min, q1, median, q3, max]
+    values = [
+        float(avg.get("mean", 0)),
+        float(avg.get("min", 0)),
+        float(avg.get("25%", 0)),
+        float(avg.get("50%", 0)),
+        float(avg.get("75%", 0)),
+        float(avg.get("max", 0))
+    ]
 
     label = os.path.splitext(os.path.basename(path))[0]
-    return [float(x) for x in avg], label
+    return values, label
 
 
 def main() -> int:

From 5b48a31b8af931e986662d5f2732ecaba6fcae1f Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Fri, 17 Oct 2025 13:48:08 -0700
Subject: [PATCH 051/104] add parent dir for profile and eval datasets

---
 src/vuln_analysis/data/{ => eval_datasets}/eval_dataset.json      | 0
 .../data/{ => profiler_datasets}/profiler_dataset.json            | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename src/vuln_analysis/data/{ => eval_datasets}/eval_dataset.json (100%)
 rename src/vuln_analysis/data/{ => profiler_datasets}/profiler_dataset.json (100%)

diff --git a/src/vuln_analysis/data/eval_dataset.json b/src/vuln_analysis/data/eval_datasets/eval_dataset.json
similarity index 100%
rename from src/vuln_analysis/data/eval_dataset.json
rename to src/vuln_analysis/data/eval_datasets/eval_dataset.json
diff --git a/src/vuln_analysis/data/profiler_dataset.json b/src/vuln_analysis/data/profiler_datasets/profiler_dataset.json
similarity index 100%
rename from src/vuln_analysis/data/profiler_dataset.json
rename to src/vuln_analysis/data/profiler_datasets/profiler_dataset.json

From 5855cc6de87cdbe744cd1c381f17eb33e0511d39 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Fri, 17 Oct 2025 13:58:18 -0700
Subject: [PATCH 052/104] readme update on eval dataset file location

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 20917a2f5..04703cc5b 100644
--- a/README.md
+++ b/README.md
@@ -791,7 +791,7 @@ You can write similar evaluators to calculate any desired custom metrics; read m
 
 
 ### Inputting Evaluation Test Sets
-Ground truth datasets should be put in the `data/eval_datasets/` directory. Each eval test set should be a separate json, with the following structure:
+Ground truth datasets should be put in the `data/eval_datasets/` directory. Each JSON file represents an experiment, which can use multiple containers. The structure is as follows:
 
 ```
 test-set-description
@@ -807,10 +807,10 @@ containers:
 
 In `config-eval.yml`, the `file_path` field should specify the specific test set to use.
 
-An example evaluation experiment has been set up to run the `eval_dataset` test set on the morpheus_23.11-runtime and morpheus_24.03-runtime images. Note that in order to run a test set across multiple containers, each container must contain exactly the same test set of CVEs present in every other container (ie. a container cannot contain only a subset of the full test set).
+An example evaluation experiment has been set up to run `eval_dataset` which runs test sets of CVEs on the morpheus_23.11-runtime and morpheus_24.03-runtime images.
 
 ```yml
-file_path: data/eval_dataset.json
+file_path: data/eval_datasets/eval_dataset.json
 ```
 
 You can also configure the evaluation output directory using the `output_dir` variable.

From bc83a42238e99049e148b661ce2ecc6d02779bde Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Fri, 17 Oct 2025 14:17:58 -0700
Subject: [PATCH 053/104] added required dataset_id field for test sets (and
 kept optional dataset_description field)

---
 src/vuln_analysis/configs/config-eval.yml            |  2 +-
 .../data/eval_datasets/eval_dataset.json             |  3 ++-
 src/vuln_analysis/data_models/eval_input.py          |  3 ++-
 src/vuln_analysis/eval/parse_eval_input.py           | 12 +++++++-----
 4 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml
index cf91b4131..4d9cb39f0 100644
--- a/src/vuln_analysis/configs/config-eval.yml
+++ b/src/vuln_analysis/configs/config-eval.yml
@@ -176,7 +176,7 @@ eval:
     output_dir: ./.tmp/evaluators
     dataset:
         _type: custom
-        file_path: data/eval_dataset.json
+        file_path: data/eval_datasets/eval_dataset.json
         function: vuln_analysis.eval.parse_eval_input.parse_input
 
   evaluators:
diff --git a/src/vuln_analysis/data/eval_datasets/eval_dataset.json b/src/vuln_analysis/data/eval_datasets/eval_dataset.json
index adea2fb1a..fe73b8cd6 100644
--- a/src/vuln_analysis/data/eval_datasets/eval_dataset.json
+++ b/src/vuln_analysis/data/eval_datasets/eval_dataset.json
@@ -1,4 +1,5 @@
-{   "test_set_description": "Common test set for all containers",
+{   "dataset_id": "common-test-set",
+    "dataset_description": "Experiment using two containers",
     "containers": {
         "morpheus-23.11": {
             "container_image": {
diff --git a/src/vuln_analysis/data_models/eval_input.py b/src/vuln_analysis/data_models/eval_input.py
index 5a8822bfd..4962f3c4c 100644
--- a/src/vuln_analysis/data_models/eval_input.py
+++ b/src/vuln_analysis/data_models/eval_input.py
@@ -61,7 +61,8 @@ class AgentMorpheusEvalInputItem(BaseModel):
 
 
 class AgentMorpheusEvalDataset(BaseModel):
-    test_set_description: str = Field(default="Unknown test set")
+    dataset_id: str
+    dataset_description: str | None = None
     containers: dict[str, AgentMorpheusEvalInputItem]
 
     @model_validator(mode="after")
diff --git a/src/vuln_analysis/eval/parse_eval_input.py b/src/vuln_analysis/eval/parse_eval_input.py
index b412c1c71..d039d0fb9 100644
--- a/src/vuln_analysis/eval/parse_eval_input.py
+++ b/src/vuln_analysis/eval/parse_eval_input.py
@@ -38,10 +38,11 @@ def parse_input(file_path: Path) -> EvalInput:
     # Extract containers
     containers = dataset.containers
 
-    # Get test set description from the file
-    test_set_description = dataset.test_set_description
+    # Get dataset metadata from the file
+    dataset_id = dataset.dataset_id
+    dataset_description = dataset.dataset_description
     test_set_file = file_path.name  # Extract filename from the file path
-    print(f"Processing test set: {test_set_description} (file: {test_set_file})")
+    print(f"Processing dataset: {dataset_id} (file: {test_set_file})")
 
     # Create evaluation items for each container
     eval_items = []
@@ -94,7 +95,8 @@ def parse_input(file_path: Path) -> EvalInput:
                                       "question": question_string,
                                       "answer": answer_string,
                                       "test_set_file": test_set_file,
-                                      "test_set_description": test_set_description,
+                                      "dataset_id": dataset_id,
+                                      "dataset_description": dataset_description,
                                       "container_id": container_id,
                                       "ground_truth": ground_truth
                                   })
@@ -105,7 +107,7 @@ def parse_input(file_path: Path) -> EvalInput:
         print("No valid evaluation items created")
         return EvalInput(eval_input_items=[])
 
-    print(f"Created {len(eval_items)} evaluation items for test set '{test_set_description}'")
+    print(f"Created {len(eval_items)} evaluation items for dataset '{dataset_id}'")
     return EvalInput(eval_input_items=eval_items)
 
 

From c6b57abb604e9db921793093685ca14c44432775 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Sun, 19 Oct 2025 09:01:50 -0700
Subject: [PATCH 054/104] update eval input data model validation to use
 predefined label/status mappings from justification_parser

---
 src/vuln_analysis/data_models/eval_input.py | 57 +++++++++++++++------
 1 file changed, 40 insertions(+), 17 deletions(-)

diff --git a/src/vuln_analysis/data_models/eval_input.py b/src/vuln_analysis/data_models/eval_input.py
index 4962f3c4c..96a1c2110 100644
--- a/src/vuln_analysis/data_models/eval_input.py
+++ b/src/vuln_analysis/data_models/eval_input.py
@@ -21,31 +21,27 @@
 from pydantic import model_validator
 
 from vuln_analysis.data_models.input import ImageInfoInput
+from vuln_analysis.utils.justification_parser import JustificationParser
 from vuln_analysis.utils.string_utils import is_valid_cve_id
 from vuln_analysis.utils.string_utils import is_valid_ghsa_id
 
-EvalLabel = Literal[
-    "false_positive",
-    "code_not_present",
-    "code_not_reachable",
-    "requires_configuration",
-    "requires_dependency",
-    "requires_environment",
-    "protected_by_compiler",
-    "protected_at_runtime",
-    "protected_at_perimeter",
-    "protected_by_mitigating_control",
-    "uncertain",
-    "vulnerable",
-]
+# Use label/status mappings from JustificationParser
+ALLOWED_LABELS: tuple[str, ...] = tuple(JustificationParser.JUSTIFICATION_TO_AFFECTED_STATUS_MAP.keys())
 
-EvalStatus = Literal["AFFECTED", "NOT AFFECTED", "UNKNOWN"]
+# Map justification status (TRUE/FALSE/UNKNOWN) to eval dataset statuses (AFFECTED/NOT AFFECTED/UNKNOWN)
+_JUST_STATUS_TO_EVAL_STATUS: dict[str, str] = {
+    "TRUE": "AFFECTED",
+    "FALSE": "NOT AFFECTED",
+    "UNKNOWN": "UNKNOWN",
+}
+
+ALLOWED_STATUSES: tuple[str, ...] = tuple(_JUST_STATUS_TO_EVAL_STATUS.values())
 
 
 class GroundTruthItem(BaseModel):
     vuln_id: str
-    status: EvalStatus | None = None
-    label: EvalLabel | None = None
+    status: str
+    label: str
 
     @field_validator("vuln_id")
     @classmethod
@@ -54,6 +50,33 @@ def validate_vuln_id(cls, v: str) -> str:
             raise ValueError(f"{v} is not a valid CVE ID or GHSA ID.")
         return v
 
+    @field_validator("label")
+    @classmethod
+    def validate_label(cls, v: str) -> str:
+        if v not in ALLOWED_LABELS:
+            raise ValueError(f"{v} is not a valid justification label. Allowed: {sorted(ALLOWED_LABELS)}")
+        return v
+
+    @field_validator("status")
+    @classmethod
+    def validate_status(cls, v: str) -> str:
+        if v not in ALLOWED_STATUSES:
+            raise ValueError(f"{v} is not a valid status. Allowed: {sorted(ALLOWED_STATUSES)}")
+        return v
+
+    @model_validator(mode="after")
+    def validate_label_status_consistency(self):
+        # Expected status from label via mapping from JustificationParser
+        expected_status = JustificationParser.JUSTIFICATION_TO_AFFECTED_STATUS_MAP[self.label]
+        expected_status_transformed = _JUST_STATUS_TO_EVAL_STATUS[expected_status]
+
+        if self.status != expected_status_transformed:
+            raise ValueError(
+                f"Label/status mismatch: label '{self.label}' implies '{expected_status_transformed}' but got '{self.status}'."
+            )
+
+        return self
+
 
 class AgentMorpheusEvalInputItem(BaseModel):
     container_image: ImageInfoInput

From 0e6777a36634dd40449a0bb8e91177b9a54d7664 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Sun, 19 Oct 2025 09:11:17 -0700
Subject: [PATCH 055/104] minor: remove dead code

---
 src/vuln_analysis/eval/evaluators/accuracy.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/vuln_analysis/eval/evaluators/accuracy.py b/src/vuln_analysis/eval/evaluators/accuracy.py
index e5dea04df..1d4507b7d 100644
--- a/src/vuln_analysis/eval/evaluators/accuracy.py
+++ b/src/vuln_analysis/eval/evaluators/accuracy.py
@@ -76,15 +76,12 @@ def _preprocess_ground_truth(ground_truth_list, field: str, policy: str):
             - keep_first: take the first encountered answer
             - lenient: any answer in ground truth is considered acceptable
             """
-            # Collect ground truth answers per vuln_id in encounter order
-            encountered_order = {}
+            # Collect ground truth answers per vuln_id
             answers_per_id = {}
-            for idx, gt in enumerate(ground_truth_list):
+            for gt in ground_truth_list:
                 vuln_id = gt.get("vuln_id")
                 if not vuln_id:
                     continue
-                if vuln_id not in encountered_order:
-                    encountered_order[vuln_id] = idx
                 if field == "status":
                     ans = gt.get("status")
                 else:
@@ -105,7 +102,6 @@ def _preprocess_ground_truth(ground_truth_list, field: str, policy: str):
 
             if policy == "keep_first":
                 # Keep first encountered answer per vuln_id
-                # Use encountered order of ground truth items
                 for vuln_id, answers in answers_per_id.items():
                     first_answer = answers[0]
                     acceptable_map[vuln_id] = {first_answer}

From 4b502ecd28cd97302fcdb4a56e8b8ce8da7873d2 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Sun, 19 Oct 2025 09:18:15 -0700
Subject: [PATCH 056/104] minor: implementation fixes to accuracy evaluator
 (keeps same logic)

---
 src/vuln_analysis/eval/evaluators/accuracy.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/vuln_analysis/eval/evaluators/accuracy.py b/src/vuln_analysis/eval/evaluators/accuracy.py
index 1d4507b7d..ad873e36b 100644
--- a/src/vuln_analysis/eval/evaluators/accuracy.py
+++ b/src/vuln_analysis/eval/evaluators/accuracy.py
@@ -82,10 +82,7 @@ def _preprocess_ground_truth(ground_truth_list, field: str, policy: str):
                 vuln_id = gt.get("vuln_id")
                 if not vuln_id:
                     continue
-                if field == "status":
-                    ans = gt.get("status")
-                else:
-                    ans = gt.get("label")
+                ans = gt.get(field)
                 if ans is None:
                     continue
                 answers_per_id.setdefault(vuln_id, [])
@@ -109,8 +106,7 @@ def _preprocess_ground_truth(ground_truth_list, field: str, policy: str):
 
             # lenient: any of the distinct answers are acceptable
             for vuln_id, answers in answers_per_id.items():
-                distinct = list(dict.fromkeys(answers))
-                acceptable_map[vuln_id] = set(distinct)
+                acceptable_map[vuln_id] = set(answers)
             return acceptable_map
 
         async def process_item(item):

From f14845f9394421376ff34ae7a7142a6d2a457a04 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Sun, 19 Oct 2025 09:30:11 -0700
Subject: [PATCH 057/104] minor: consistency evaluator implementation change,
 logic same

---
 src/vuln_analysis/eval/evaluators/consistency.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/vuln_analysis/eval/evaluators/consistency.py b/src/vuln_analysis/eval/evaluators/consistency.py
index 5690b6728..049d2a2a9 100644
--- a/src/vuln_analysis/eval/evaluators/consistency.py
+++ b/src/vuln_analysis/eval/evaluators/consistency.py
@@ -79,17 +79,16 @@ async def extract_run_data(item):
                 field_value = cve_result.get('justification', {}).get(self.field, 'MISSING')
 
                 if vuln_id not in vuln_responses:
-                    vuln_responses[vuln_id] = {'values': []}
+                    vuln_responses[vuln_id] = []
 
-                vuln_responses[vuln_id]['values'].append(field_value)
+                vuln_responses[vuln_id].append(field_value)
 
         # Calculate consistency score per CVE
         consistencies_dict = {}  # Stores consistency scores mapped to CVE
         consistencies = []  # Stores all consistency scores in an array
 
-        for vuln_id, responses in vuln_responses.items():
+        for vuln_id, values in vuln_responses.items():
             # Calculate consistency for the specified field
-            values = responses['values']
             if len(values) > 1:
                 # Consistency metric is defined as the sum of squared probabilities
                 value_counts = Counter(values)

From 9482c0f7c8e8d801ff3b5cc1d1e1f3981e805986 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Sun, 19 Oct 2025 09:44:06 -0700
Subject: [PATCH 058/104] minor: move example usage of script to docstring

---
 .../visualizations/box_and_whisker_plot.py    | 22 ++++++++++---------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py b/src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py
index 496e3c480..7e1c3befd 100644
--- a/src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py
+++ b/src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py
@@ -18,14 +18,17 @@
 import os
 
 import matplotlib.pyplot as plt
+"""
+Box-and-whisker plot utility for evaluator outputs.
+
+Example usage:
 
-EXAMPLE_USAGE = """
-python3 src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py \
-  .tmp/evaluators/llama8b-experiment/status_accuracy_output.json \
-  .tmp/evaluators/llama70b-experiment/status_accuracy_output.json \
-  --title "Status Accuracy: llama8b vs llama70b" \
-  --save src/vuln_analysis/eval/visualizations/plot.png \
-  --labels llama8b llama70b
+    python3 src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py \
+      .tmp/evaluators/llama8b-experiment/status_accuracy_output.json \
+      .tmp/evaluators/llama70b-experiment/status_accuracy_output.json \
+      --title "Status Accuracy: llama8b vs llama70b" \
+      --save src/vuln_analysis/eval/visualizations/plot.png \
+      --labels llama8b llama70b
 """
 
 
@@ -55,10 +58,9 @@ def load_box_values(path: str) -> tuple[list[float], str]:
 
 
 def main() -> int:
-    parser = argparse.ArgumentParser(
-        description=
+    parser = argparse.ArgumentParser(description=(
         "Plot box-and-whisker charts from evaluator output JSON files (average_score = [mean, min, q1, median, q3, max])."
-    )
+    ))
     parser.add_argument("files", nargs="+", help="Paths to output files")
     parser.add_argument("--title", default="Accuracy Distribution Across Runs", help="Chart title.")
     parser.add_argument("--save",

From 2a274e98d3a41f5fecb263773c2cd8c5ddfe6d96 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Sun, 19 Oct 2025 10:01:15 -0700
Subject: [PATCH 059/104] add docstrings to explain custom input parser script

---
 src/vuln_analysis/eval/parse_eval_input.py | 29 ++++++++++++++++++----
 1 file changed, 24 insertions(+), 5 deletions(-)

diff --git a/src/vuln_analysis/eval/parse_eval_input.py b/src/vuln_analysis/eval/parse_eval_input.py
index d039d0fb9..cc461b46f 100644
--- a/src/vuln_analysis/eval/parse_eval_input.py
+++ b/src/vuln_analysis/eval/parse_eval_input.py
@@ -24,11 +24,30 @@
 
 def parse_input(file_path: Path) -> EvalInput:
     """
-    To run just this script outside of the full nat pipeline:
-    python3 parse_eval_input.py [test_set_file].json --preview -o [your output file].json
-    Ex: python3 parse_eval_input.py ../data/eval_datasets/common-test-set.json --preview -o my_eval_string.json
-
-    Note: Run this inside the vuln_analysis container, from the eval directory
+    Transform a human-readable eval dataset in the eval_datasets/ directory into a series of EvalInputItems
+    that the NAT evaluation harness accepts. Each EvalInputItem corresponds to a container.
+    EvalInputItem `id` is the container name, `question` encodes the source code and input CVEs,
+    and `answer` encodes the CVE ground truth mapping.
+
+    Usage (outside of the full NAT pipeline):
+        python3 parse_eval_input.py [test_set_file].json --preview -o [your output file].json
+        Example:
+        python3 parse_eval_input.py ../data/eval_datasets/common-test-set.json --preview -o my_eval_string.json
+    Run this inside the vuln_analysis container, from the eval directory.
+
+    Example output:
+        [
+            {
+                "id": "morpheus-23.11",
+                "question": "{\"image\":{\"name\":\"nvcr.io/nvidia/morpheus/morpheus\",\"tag\":\"23.11-runtime\",\"source_info\":[{\"type\":\"code\",\"git_repo\":\"https://github.com/nv-morpheus/Morpheus.git\",\"ref\":\"branch-23.11\",\"include\":[\"**/*.cpp\",\"**/*.cu\",\"**/*.cuh\",\"**/*.h\",\"**/*.hpp\",\"**/*.ipynb\",\"**/*.py\",\"**/*Dockerfile\"],\"exclude\":[\"tests/**/*\"]},{\"type\":\"doc\",\"git_repo\":\"https://github.com/nv-morpheus/Morpheus.git\",\"ref\":\"branch-23.11\",\"include\":[\"**/*.md\",\"docs/**/*.rst\"]}],\"sbom_info\":{\"_type\":\"file\",\"file_path\":\"data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom\"}},\"scan\":{\"vulns\":[{\"vuln_id\":\"GHSA-3f63-hfp8-52jq\"},{\"vuln_id\":\"CVE-2023-36632\"}]}}",
+                "answer": "{\"GHSA-3f63-hfp8-52jq\":\"NOT AFFECTED\",\"GHSA-3f63-hfp8-52jq_label\":\"code_not_reachable\",\"CVE-2023-36632\":\"AFFECTED\",\"CVE-2023-36632_label\":\"vulnerable\"}"
+            },
+            {
+                "id": "morpheus-24.03",
+                "question": "{\"image\":{\"name\":\"nvcr.io/nvidia/morpheus/morpheus\",\"tag\":\"v24.03.02-runtime\",\"source_info\":[{\"type\":\"code\",\"git_repo\":\"https://github.com/nv-morpheus/Morpheus.git\",\"ref\":\"v24.03.02\",\"include\":[\"**/*.cpp\",\"**/*.cu\",\"**/*.cuh\",\"**/*.h\",\"**/*.hpp\",\"**/*.ipynb\",\"**/*.py\",\"**/*Dockerfile\"],\"exclude\":[\"tests/**/*\"]},{\"type\":\"doc\",\"git_repo\":\"https://github.com/nv-morpheus/Morpheus.git\",\"ref\":\"v24.03.02\",\"include\":[\"**/*.md\",\"docs/**/*.rst\"]}],\"sbom_info\":{\"_type\":\"file\",\"file_path\":\"data/sboms/nvcr.io/nvidia/morpheus/morpheus:v24.03.02-runtime.sbom\"}},\"scan\":{\"vulns\":[{\"vuln_id\":\"GHSA-3f63-hfp8-52jq\"},{\"vuln_id\":\"CVE-2023-36632\"}]}}",
+                "answer": "{\"GHSA-3f63-hfp8-52jq\":\"NOT AFFECTED\",\"GHSA-3f63-hfp8-52jq_label\":\"code_not_reachable\",\"CVE-2023-36632\":\"AFFECTED\",\"CVE-2023-36632_label\":\"vulnerable\"}"
+            }
+        ]
     """
     # Load and validate the test set file using Pydantic models
     with open(file_path, 'r', encoding='utf-8') as f:

From b0b8a0b532d6fa992943be05c770ce0f395982c2 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Sun, 19 Oct 2025 10:06:06 -0700
Subject: [PATCH 060/104] readme typo fix

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 04703cc5b..148e61c1e 100644
--- a/README.md
+++ b/README.md
@@ -777,7 +777,7 @@ nat eval --config_file=configs/config-eval.yml
 
 This will write the files `status_accuracy_output.json` and `label_accuracy_output.json` to the configured output directory (`.tmp/evaluators` in this example), containing accuracy scores for status and label respectively. Scores range from 0 to 1, with 0 indicating no CVEs were categorized correctly, and 1 indicating all CVEs were categorized correctly.
 
-Under the hood, the source code of the evaluator being run can be found at `eval/evaluators/accuracy.py`; this custom evaluator is registered in the NeMo Agent Toolkit as an evaluator of type `_accuracy`.  In `config-eval.yml`, under the `evaluators` section, we have defined two evaluations to be run: `status_accuracy` and `label_accuracy`. However, the underlying code for these evaluations is the same. In other words, they use the same evaluator, which accepts either `status` or `label` as a configurable field input.
+Under the hood, the source code of the evaluator being run can be found at `eval/evaluators/accuracy.py`; this custom evaluator is registered in the NeMo Agent Toolkit as an evaluator of type `accuracy`.  In `config-eval.yml`, under the `evaluators` section, we have defined two evaluations to be run: `status_accuracy` and `label_accuracy`. However, the underlying code for these evaluations is the same. In other words, they use the same evaluator, which accepts either `status` or `label` as a configurable field input.
 
 In certain instances, a provided ground-truth dataset may be self-conflicting if it is made of a compilation of answers from different sources. For example, a CVE may be listed twice as both "vulnerable" and "not vulnerable" with different justifications. To handle such cases, we provide three levels of strictness for judging accuracy:
 - `strict`: Any CVE with self-conflicting ground truth will be omitted and not contribute towards evaluation results.
@@ -837,7 +837,7 @@ python3 src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py \
   --labels my-experiment1 my-experiment2
 ```
 
-You can list an arbitary number of .json files output by the `_accuracy` type of evaluator, in order to compare experiment run distributions side by side. Additionally, you should specify the chart title, the save location, and the X axis labels (the label for each box & whisker plot). The output will look something like the plot below.
+You can list an arbitary number of .json files output by the `accuracy` type of evaluator, in order to compare experiment run distributions side by side. Additionally, you should specify the chart title, the save location, and the X axis labels (the label for each box & whisker plot). The output will look something like the plot below.
 
 ![example plot](src/vuln_analysis/eval/visualizations/example_plot.png)
 

From 958caa5a884a604008cb4c38e2501ae90a01aa28 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Sun, 19 Oct 2025 10:46:39 -0700
Subject: [PATCH 061/104] renamed accuracy evaluator's duplicate handling
 policy config options

---
 README.md                                     |  8 ++---
 src/vuln_analysis/configs/config-eval.yml     |  4 +--
 src/vuln_analysis/eval/evaluators/accuracy.py | 29 ++++++++++++-------
 3 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/README.md b/README.md
index 148e61c1e..8fa005bee 100644
--- a/README.md
+++ b/README.md
@@ -779,11 +779,11 @@ This will write the files `status_accuracy_output.json` and `label_accuracy_outp
 
 Under the hood, the source code of the evaluator being run can be found at `eval/evaluators/accuracy.py`; this custom evaluator is registered in the NeMo Agent Toolkit as an evaluator of type `accuracy`.  In `config-eval.yml`, under the `evaluators` section, we have defined two evaluations to be run: `status_accuracy` and `label_accuracy`. However, the underlying code for these evaluations is the same. In other words, they use the same evaluator, which accepts either `status` or `label` as a configurable field input.
 
-In certain instances, a provided ground-truth dataset may be self-conflicting if it is made of a compilation of answers from different sources. For example, a CVE may be listed twice as both "vulnerable" and "not vulnerable" with different justifications. To handle such cases, we provide three levels of strictness for judging accuracy:
-- `strict`: Any CVE with self-conflicting ground truth will be omitted and not contribute towards evaluation results.
+In certain instances, a provided ground-truth dataset may contain duplicate CVEs with potentially conflicting answers from different sources. For example, a CVE may be listed twice as both "vulnerable" and "not vulnerable" with different justifications. To handle such cases, we provide three policies for handling duplicates:
+- `drop_all`: Any CVE with conflicting ground truth will be omitted and not contribute towards evaluation results.
 - `keep_first`: Use the first encountered instance of a CVE as the truth, disregard all other instances.
-- `lenient`: Any answer listed in the ground truth for a CVE is considered valid (eg. both "vulnerable" and "not vulnerable" would be scored as correct if the ground truth listed both).
-You can configure the policy for handling any conflicting ground truth inputs in the `conflict_policy` field.
+- `keep_all`: Any answer listed in the ground truth for a CVE is considered valid (eg. both "vulnerable" and "not vulnerable" would be scored as correct if the ground truth listed both).
+You can configure the policy for handling any duplicate ground truth inputs in the `duplicates_policy` field.
 
 You can write similar evaluators to calculate any desired custom metrics; read more at the NeMo Agent Toolkit documentation for [custom evaluators](https://docs.nvidia.com/nemo/agent-toolkit/1.2/reference/evaluate.html#adding-custom-evaluators).
 
diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml
index 4d9cb39f0..7ea16457e 100644
--- a/src/vuln_analysis/configs/config-eval.yml
+++ b/src/vuln_analysis/configs/config-eval.yml
@@ -184,12 +184,12 @@ eval:
       _type: accuracy
       field: status
       multiple_reps: False
-      conflict_policy: strict  # Options: strict, keep_first, lenient
+      duplicates_policy: drop_all  # Options: drop_all, keep_first, keep_all
     label_accuracy:
       _type: accuracy
       field: label
       multiple_reps: False
-      conflict_policy: strict
+      duplicates_policy: drop_all
 
     # Uncomment to enable response consistency evaluators
     # status_consistency:
diff --git a/src/vuln_analysis/eval/evaluators/accuracy.py b/src/vuln_analysis/eval/evaluators/accuracy.py
index ad873e36b..0d532bd28 100644
--- a/src/vuln_analysis/eval/evaluators/accuracy.py
+++ b/src/vuln_analysis/eval/evaluators/accuracy.py
@@ -16,6 +16,7 @@
 import asyncio
 import json
 import statistics as stats
+from typing import Literal
 
 import pandas as pd
 from nat.builder.builder import EvalBuilder
@@ -36,7 +37,13 @@ class AccuracyEvaluatorConfig(EvaluatorBaseConfig, name="accuracy"):
     """Configuration for unified accuracy evaluator"""
     field: str = "status"  # "status" or "label"
     multiple_reps: bool = False
-    conflict_policy: str = "keep_first"  # "strict", "keep_first", or "lenient"
+    duplicates_policy: Literal["drop_all", "keep_first", "keep_all"] = "keep_first"
+    """Policy for handling duplicate CVEs in ground truth.
+
+    - drop_all: Only CVEs with single unique answers are kept; conflicting CVEs are excluded from evaluation
+    - keep_first: The prediction will be considered a true positive if it matches the FIRST ground truth value
+    - keep_all: The prediction will be considered a true positive if it matches ANY of the ground truth values
+    """
 
 
 @register_evaluator(config_type=AccuracyEvaluatorConfig)
@@ -45,7 +52,7 @@ async def accuracy_evaluator(config: AccuracyEvaluatorConfig, builder: EvalBuild
     evaluator = AccuracyEvaluator(builder.get_max_concurrency(),
                                   field=config.field,
                                   multiple_reps=config.multiple_reps,
-                                  conflict_policy=config.conflict_policy)
+                                  duplicates_policy=config.duplicates_policy)
     yield EvaluatorInfo(config=config,
                         evaluate_fn=evaluator.evaluate,
                         description=f"{config.field.title()} Accuracy Evaluator")
@@ -58,23 +65,23 @@ def __init__(self,
                  max_concurrency: int,
                  field: str = "status",
                  multiple_reps: bool = False,
-                 conflict_policy: str = "keep_first"):
+                 duplicates_policy: Literal["drop_all", "keep_first", "keep_all"] = "keep_first"):
         self.max_concurrency = max_concurrency
         self.semaphore = asyncio.Semaphore(self.max_concurrency)
         # field must be either "status" or "label"
         self.field = field
         self.multiple_reps = multiple_reps
-        # conflict_policy: "strict" | "keep_first" | "lenient"
-        self.conflict_policy = conflict_policy
+        # duplicates_policy: "drop_all" | "keep_first" | "keep_all"
+        self.duplicates_policy = duplicates_policy
 
     async def evaluate(self, eval_input: EvalInput) -> EvalOutput:
 
         def _preprocess_ground_truth(ground_truth_list, field: str, policy: str):
             """Build a mapping of vuln_id -> acceptable answer set based on policy.
 
-            - strict: only vuln_ids with a single unique answer are kept
+            - drop_all: only vuln_ids with a single unique answer are kept
             - keep_first: take the first encountered answer
-            - lenient: any answer in ground truth is considered acceptable
+            - keep_all: any answer in ground truth is considered acceptable
             """
             # Collect ground truth answers per vuln_id
             answers_per_id = {}
@@ -90,7 +97,7 @@ def _preprocess_ground_truth(ground_truth_list, field: str, policy: str):
 
             acceptable_map = {}
 
-            if policy == "strict":
+            if policy == "drop_all":
                 for vuln_id, answers in answers_per_id.items():
                     distinct = list(dict.fromkeys(answers))  # preserve order, unique
                     if len(distinct) == 1:
@@ -104,7 +111,7 @@ def _preprocess_ground_truth(ground_truth_list, field: str, policy: str):
                     acceptable_map[vuln_id] = {first_answer}
                 return acceptable_map
 
-            # lenient: any of the distinct answers are acceptable
+            # keep_all: any of the distinct answers are acceptable
             for vuln_id, answers in answers_per_id.items():
                 acceptable_map[vuln_id] = set(answers)
             return acceptable_map
@@ -112,7 +119,7 @@ def _preprocess_ground_truth(ground_truth_list, field: str, policy: str):
         async def process_item(item):
             # Build policy-aware acceptable answers map from the original ground truth list
             original_ground_truth = item.full_dataset_entry.get("ground_truth", [])
-            acceptable_answers_map = _preprocess_ground_truth(original_ground_truth, self.field, self.conflict_policy)
+            acceptable_answers_map = _preprocess_ground_truth(original_ground_truth, self.field, self.duplicates_policy)
             pipeline_result = json.loads(item.output_obj)["output"]
 
             per_item_accuracies = []
@@ -131,7 +138,7 @@ async def process_item(item):
                     pred = justification_label
                     acceptable_set = acceptable_answers_map.get(vuln_id)
 
-                # If this vuln_id was filtered out by the conflict policy (e.g., strict), skip entirely
+                # If this vuln_id was filtered out by the duplicates policy (e.g., drop_all), skip entirely
                 if acceptable_set is None:
                     continue
 

From 43fc8839791adf8d10ab3aad2d4883725e0c7407 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Sun, 19 Oct 2025 11:01:07 -0700
Subject: [PATCH 062/104] make evaluator config fields use typing.Literal and
 pydantic Fields

---
 src/vuln_analysis/eval/evaluators/accuracy.py    | 15 ++++++---------
 src/vuln_analysis/eval/evaluators/consistency.py |  5 ++++-
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/vuln_analysis/eval/evaluators/accuracy.py b/src/vuln_analysis/eval/evaluators/accuracy.py
index 0d532bd28..a8528d022 100644
--- a/src/vuln_analysis/eval/evaluators/accuracy.py
+++ b/src/vuln_analysis/eval/evaluators/accuracy.py
@@ -16,6 +16,7 @@
 import asyncio
 import json
 import statistics as stats
+import typing
 from typing import Literal
 
 import pandas as pd
@@ -28,6 +29,7 @@
 from nat.eval.evaluator.evaluator_model import EvalOutput
 from nat.eval.evaluator.evaluator_model import EvalOutputItem
 from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry
+from pydantic import Field
 from tqdm import tqdm
 
 _STATUS_MAP = {"TRUE": "AFFECTED", "FALSE": "NOT AFFECTED", "UNKNOWN": "UNKNOWN"}
@@ -35,15 +37,10 @@
 
 class AccuracyEvaluatorConfig(EvaluatorBaseConfig, name="accuracy"):
     """Configuration for unified accuracy evaluator"""
-    field: str = "status"  # "status" or "label"
-    multiple_reps: bool = False
-    duplicates_policy: Literal["drop_all", "keep_first", "keep_all"] = "keep_first"
-    """Policy for handling duplicate CVEs in ground truth.
-
-    - drop_all: Only CVEs with single unique answers are kept; conflicting CVEs are excluded from evaluation
-    - keep_first: The prediction will be considered a true positive if it matches the FIRST ground truth value
-    - keep_all: The prediction will be considered a true positive if it matches ANY of the ground truth values
-    """
+    field: typing.Literal["status", "label"] = Field(default="status", description="The field to evaluate accuracy on.")
+    multiple_reps: bool = Field(default=False, description="Whether to evaluate multiple reps of the same input.")
+    duplicates_policy: typing.Literal["drop_all", "keep_first", "keep_all"] = Field(
+        default="keep_first", description="Policy for handling duplicate CVEs in ground truth.")
 
 
 @register_evaluator(config_type=AccuracyEvaluatorConfig)
diff --git a/src/vuln_analysis/eval/evaluators/consistency.py b/src/vuln_analysis/eval/evaluators/consistency.py
index 049d2a2a9..d2f2f5028 100644
--- a/src/vuln_analysis/eval/evaluators/consistency.py
+++ b/src/vuln_analysis/eval/evaluators/consistency.py
@@ -15,6 +15,7 @@
 
 import asyncio
 import json
+import typing
 from collections import Counter
 
 from nat.builder.builder import EvalBuilder
@@ -25,12 +26,14 @@
 from nat.eval.evaluator.evaluator_model import EvalOutput
 from nat.eval.evaluator.evaluator_model import EvalOutputItem
 from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry
+from pydantic import Field
 from tqdm import tqdm
 
 
 class ConsistencyEvaluatorConfig(EvaluatorBaseConfig, name="consistency"):
     """Configuration for consistency evaluator"""
-    field: str = "label"  # "label" or "status"
+    field: typing.Literal["label", "status"] = Field(default="label",
+                                                     description="The field to evaluate consistency on.")
 
 
 @register_evaluator(config_type=ConsistencyEvaluatorConfig)

From 9a5dda03499be22b678de3e9006a597944af08c7 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Sun, 19 Oct 2025 11:32:59 -0700
Subject: [PATCH 063/104] removed constraint that every container input needs
 same test set of CVEs

---
 src/vuln_analysis/data_models/eval_input.py   | 26 -------------------
 src/vuln_analysis/eval/evaluators/accuracy.py | 14 ++++++----
 2 files changed, 9 insertions(+), 31 deletions(-)

diff --git a/src/vuln_analysis/data_models/eval_input.py b/src/vuln_analysis/data_models/eval_input.py
index 96a1c2110..9954a5f70 100644
--- a/src/vuln_analysis/data_models/eval_input.py
+++ b/src/vuln_analysis/data_models/eval_input.py
@@ -87,29 +87,3 @@ class AgentMorpheusEvalDataset(BaseModel):
     dataset_id: str
     dataset_description: str | None = None
     containers: dict[str, AgentMorpheusEvalInputItem]
-
-    @model_validator(mode="after")
-    def validate_consistent_vuln_sets(self):
-        # Ensure all containers provide the same set of vuln_ids
-        container_to_set: dict[str, set[str]] = {}
-        for container_id, container in self.containers.items():
-            vuln_ids = {gt.vuln_id for gt in container.ground_truth}
-            container_to_set[container_id] = vuln_ids
-
-        if not container_to_set:
-            return self
-
-        sets = list(container_to_set.values())
-        base = sets[0]
-        mismatches = []
-        for cid, s in container_to_set.items():
-            if s != base:
-                missing = sorted(list(base - s))
-                extra = sorted(list(s - base))
-                mismatches.append((cid, missing, extra))
-
-        if mismatches:
-            details = "; ".join(f"{cid}: missing={missing} extra={extra}" for cid, missing, extra in mismatches)
-            raise ValueError(f"All containers must contain the same set of vuln_id entries. Mismatches: {details}")
-
-        return self
diff --git a/src/vuln_analysis/eval/evaluators/accuracy.py b/src/vuln_analysis/eval/evaluators/accuracy.py
index a8528d022..3012b43cd 100644
--- a/src/vuln_analysis/eval/evaluators/accuracy.py
+++ b/src/vuln_analysis/eval/evaluators/accuracy.py
@@ -182,6 +182,11 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict, int, int]:
 
         all_averages, all_reasonings, total_items, total_correct = zip(*results) if results else ([], [], [], [])
 
+        # Calculate overall accuracy across all CVEs regardless of container or repetition
+        total_correct_sum = sum(total_correct)
+        total_items_sum = sum(total_items)
+        overall_accuracy = round(total_correct_sum / total_items_sum, 2) if total_items_sum > 0 else 0.0
+
         # Choose output based on config: return box & whiskers values if multiple_reps=True, else return a single avg
         if self.multiple_reps:
             scores_sequential = list(all_averages)
@@ -190,10 +195,12 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict, int, int]:
                 desc = series.describe()  # count, mean, std, min, 25%, 50%, 75%, max
                 # Round values and make count an int
                 final_output = {k: (int(v) if k == "count" else round(float(v), 2)) for k, v in desc.items()}
+                # Override mean with overall accuracy across all CVEs
+                final_output["mean"] = overall_accuracy
             else:
                 final_output = {
                     "count": 0,
-                    "mean": 0.0,
+                    "mean": overall_accuracy,
                     "std": 0.0,
                     "min": 0.0,
                     "25%": 0.0,
@@ -202,10 +209,7 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict, int, int]:
                     "max": 0.0,
                 }
         else:  # Single rep
-            # Calculating weighted average (each EvalOutputItem's score is weighted by its number of CVEs)
-            total_correct_sum = sum(total_correct)
-            total_items_sum = sum(total_items)
-            final_output = round(total_correct_sum / total_items_sum, 2) if total_items_sum > 0 else 0.0
+            final_output = overall_accuracy
 
         eval_output_items = [
             EvalOutputItem(id=item.id, score=score, reasoning=reasoning)

From 3285389fa77d17e48e734795ff5d55e3008eb868 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Sun, 19 Oct 2025 13:02:23 -0700
Subject: [PATCH 064/104] revert checklist prompt generator (moved robust
 parsing version to another PR)

---
 .../utils/checklist_prompt_generator.py       | 29 ++++---------------
 1 file changed, 6 insertions(+), 23 deletions(-)

diff --git a/src/vuln_analysis/utils/checklist_prompt_generator.py b/src/vuln_analysis/utils/checklist_prompt_generator.py
index d93178548..6eb7abfd5 100644
--- a/src/vuln_analysis/utils/checklist_prompt_generator.py
+++ b/src/vuln_analysis/utils/checklist_prompt_generator.py
@@ -14,9 +14,7 @@
 # limitations under the License.
 
 import ast
-import json
 import logging
-import re
 
 from jinja2 import Template
 from langchain_core.language_models.base import BaseLanguageModel
@@ -68,30 +66,15 @@ async def _parse_list(text: list[str]) -> list[list[str]]:
             # Remove newline characters that can cause incorrect string escaping in the next step
             x = x.replace("\n", "")
 
+            # Ensure backslashes are escaped
+            x = x.replace("\\", "\\\\")
+
             # Try to do some very basic string cleanup to fix unescaped quotes
             x = attempt_fix_list_string(x)
 
-            # Handle backslash sequences that break JSON parsing
-            x = x.replace('\\\\..\\\\', '\\..\\')  # Fix \\..\\ patterns
-            x = x.replace('\\\\"', '\\"')  # Fix \\" patterns
-
-            # Try JSON parsing first
-            try:
-                current = json.loads(x)
-            except json.JSONDecodeError:
-                # Use regex to extract list items manually
-                pattern = r'\"([^\"]*(?:\\.[^\"]*)*)\"'
-                matches = re.findall(pattern, x)
-                if matches:
-                    # Clean up the matches by removing extra escaping
-                    current = []
-                    for match in matches:
-                        cleaned = match.replace('\\\\', '\\')
-                        current.append(cleaned)
-                else:
-                    # Fall back to ast.literal_eval
-                    x = x.replace("\\", "\\\\")
-                    current = ast.literal_eval(x)
+            # Only proceed if the input is a valid Python literal
+            # This isn't really dangerous, literal_eval only evaluates a small subset of python
+            current = ast.literal_eval(x)
 
             # Ensure that the parsed data is a list
             if not isinstance(current, list):

From 31ab517e98131e88ab11ba83a29575aafade4392 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 20 Oct 2025 00:49:37 -0700
Subject: [PATCH 065/104] refactor consistency evaluator to categorize by
 container. Add erroring if reps=1

---
 .../eval/evaluators/consistency.py            | 129 +++++++++++-------
 1 file changed, 79 insertions(+), 50 deletions(-)

diff --git a/src/vuln_analysis/eval/evaluators/consistency.py b/src/vuln_analysis/eval/evaluators/consistency.py
index d2f2f5028..42520c2fe 100644
--- a/src/vuln_analysis/eval/evaluators/consistency.py
+++ b/src/vuln_analysis/eval/evaluators/consistency.py
@@ -54,69 +54,98 @@ def __init__(self, max_concurrency: int, field: str = "label"):
         self.field = field  # "label" or "status"
 
     async def evaluate(self, eval_input: EvalInput) -> EvalOutput:
-        '''Evaluate consistency across multiple runs'''
+        '''Evaluate consistency across multiple runs per CVE, broken down by container'''
 
         async def extract_run_data(item):
             """Extract data from a single evaluation run"""
             async with self.semaphore:
                 evaluation_run = json.loads(item.output_obj)['output']
                 pbar.update(1)
-                return evaluation_run
+                return evaluation_run, item
 
         try:
             tqdm_position = TqdmPositionRegistry.claim()
             pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Consistency", position=tqdm_position)
 
             # Extract data from all evaluation runs in parallel
-            all_runs = await asyncio.gather(*[extract_run_data(item) for item in eval_input.eval_input_items])
+            all_runs_with_items = await asyncio.gather(
+                *[extract_run_data(item) for item in eval_input.eval_input_items])
         finally:
             pbar.close()
             TqdmPositionRegistry.release(tqdm_position)
 
-        # Now aggregate ALL results across ALL runs to calculate consistency per CVE
-        vuln_responses = {}
-        for evaluation_run in all_runs:
-            for cve_result in evaluation_run:
-                vuln_id = cve_result['vuln_id']
-                # Measure consistency on the specified field
-                field_value = cve_result.get('justification', {}).get(self.field, 'MISSING')
-
-                if vuln_id not in vuln_responses:
-                    vuln_responses[vuln_id] = []
-
-                vuln_responses[vuln_id].append(field_value)
-
-        # Calculate consistency score per CVE
-        consistencies_dict = {}  # Stores consistency scores mapped to CVE
-        consistencies = []  # Stores all consistency scores in an array
-
-        for vuln_id, values in vuln_responses.items():
-            # Calculate consistency for the specified field
-            if len(values) > 1:
-                # Consistency metric is defined as the sum of squared probabilities
-                value_counts = Counter(values)
-                n = len(values)
-                probs = [count / n for count in value_counts.values()]
-                consistency = sum(p * p for p in probs)
-            else:
-                consistency = 1.0  # Perfect consistency if there is only one response
-
-            consistencies_dict[vuln_id] = round(consistency, 2)
-            consistencies.append(round(consistency, 2))
-
-        # Calculate average consistency
-        avg_consistency = sum(consistencies) / len(consistencies) if consistencies else 1.0
-        avg_score = round(avg_consistency, 2)
-
-        # Create output with per-CVE scores
-        output = {
-            f"{self.field}_consistencies": consistencies_dict,
-            "total_cves": len(vuln_responses),
-        }
-
-        # Create a single dummy EvalOutputItem to store reasoning
-        eval_output_items = [
-            EvalOutputItem(id=f"{self.field.title()} Consistency per CVE", score=avg_score, reasoning=output)
-        ]
-
-        return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items)
+        # Group runs by container (extract container name from item id)
+        container_runs = {}
+        for evaluation_run, item in all_runs_with_items:
+            container_name = item.id.split('_rep')[0]
+
+            if container_name not in container_runs:
+                container_runs[container_name] = []
+            container_runs[container_name].append(evaluation_run)
+
+        # Validate that each container has multiple reps
+        for container_name, runs in container_runs.items():
+            if len(runs) <= 1:
+                raise ValueError(
+                    f"Consistency evaluator requires multiple runs per container. Container '{container_name}' has only {len(runs)} run(s). Consistency cannot be measured with a single run per container."
+                )
+
+        # Calculate consistency for each container
+        eval_output_items = []
+        all_cve_consistencies = []  # Collect all CVE consistency scores across all containers
+
+        for container_name, runs in container_runs.items():
+            # Aggregate results across all reps for this container
+            vuln_responses = {}
+            for evaluation_run in runs:
+                for cve_result in evaluation_run:  # Group by CVE
+                    vuln_id = cve_result['vuln_id']
+                    # Measure consistency on the specified field
+                    field_value = cve_result.get('justification', {}).get(self.field, 'MISSING')
+
+                    if vuln_id not in vuln_responses:
+                        vuln_responses[vuln_id] = []
+
+                    vuln_responses[vuln_id].append(field_value)
+
+            # Calculate consistency score per CVE for this container
+            consistencies_dict = {}  # Stores consistency scores mapped to CVE
+            consistencies = []  # Stores all consistency scores in an array
+
+            for vuln_id, values in vuln_responses.items():
+                if len(values) > 1:
+                    value_counts = Counter(values)
+                    n = len(values)
+                    probs = [count / n for count in value_counts.values()]
+                    consistency = sum(p * p for p in probs)
+                else:
+                    consistency = 1.0  # Perfect consistency if there is only one response
+
+                consistencies_dict[vuln_id] = round(consistency, 2)
+                consistencies.append(round(consistency, 2))
+
+            # Add all CVE consistency scores from this container to the global collection
+            all_cve_consistencies.extend(consistencies)
+
+            # Calculate average consistency for this container
+            container_avg = sum(consistencies) / len(consistencies) if consistencies else 1.0
+            container_score = round(container_avg, 2)
+
+            # Create output with per-CVE scores for this container
+            output = {
+                f"{self.field}_consistencies": consistencies_dict,
+                "total_cves": len(vuln_responses),
+                "num_reps": len(runs),
+            }
+
+            # Create EvalOutputItem for this container
+            eval_output_items.append(
+                EvalOutputItem(id=f"{self.field.title()} Consistency per CVE - {container_name}",
+                               score=container_score,
+                               reasoning=output))
+
+        # Calculate overall average across all CVEs from all containers
+        overall_avg = sum(all_cve_consistencies) / len(all_cve_consistencies) if all_cve_consistencies else 1.0
+        overall_score = round(overall_avg, 2)
+
+        return EvalOutput(average_score=overall_score, eval_output_items=eval_output_items)

From 1d7b6cf479fdff00e45bce76500540259091e0b3 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 20 Oct 2025 01:36:36 -0700
Subject: [PATCH 066/104] infer multiple reps instead of having manual config
 option in accuracy evaluator

---
 src/vuln_analysis/configs/config-eval.yml     |  2 --
 src/vuln_analysis/eval/evaluators/accuracy.py | 18 ++++++++++++------
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml
index 7ea16457e..9b830e551 100644
--- a/src/vuln_analysis/configs/config-eval.yml
+++ b/src/vuln_analysis/configs/config-eval.yml
@@ -183,12 +183,10 @@ eval:
     status_accuracy:
       _type: accuracy
       field: status
-      multiple_reps: False
       duplicates_policy: drop_all  # Options: drop_all, keep_first, keep_all
     label_accuracy:
       _type: accuracy
       field: label
-      multiple_reps: False
       duplicates_policy: drop_all
 
     # Uncomment to enable response consistency evaluators
diff --git a/src/vuln_analysis/eval/evaluators/accuracy.py b/src/vuln_analysis/eval/evaluators/accuracy.py
index 3012b43cd..1c2e5e7c9 100644
--- a/src/vuln_analysis/eval/evaluators/accuracy.py
+++ b/src/vuln_analysis/eval/evaluators/accuracy.py
@@ -38,7 +38,6 @@
 class AccuracyEvaluatorConfig(EvaluatorBaseConfig, name="accuracy"):
     """Configuration for unified accuracy evaluator"""
     field: typing.Literal["status", "label"] = Field(default="status", description="The field to evaluate accuracy on.")
-    multiple_reps: bool = Field(default=False, description="Whether to evaluate multiple reps of the same input.")
     duplicates_policy: typing.Literal["drop_all", "keep_first", "keep_all"] = Field(
         default="keep_first", description="Policy for handling duplicate CVEs in ground truth.")
 
@@ -48,7 +47,6 @@ async def accuracy_evaluator(config: AccuracyEvaluatorConfig, builder: EvalBuild
     """Register unified accuracy evaluator"""
     evaluator = AccuracyEvaluator(builder.get_max_concurrency(),
                                   field=config.field,
-                                  multiple_reps=config.multiple_reps,
                                   duplicates_policy=config.duplicates_policy)
     yield EvaluatorInfo(config=config,
                         evaluate_fn=evaluator.evaluate,
@@ -61,13 +59,11 @@ class AccuracyEvaluator:
     def __init__(self,
                  max_concurrency: int,
                  field: str = "status",
-                 multiple_reps: bool = False,
                  duplicates_policy: Literal["drop_all", "keep_first", "keep_all"] = "keep_first"):
         self.max_concurrency = max_concurrency
         self.semaphore = asyncio.Semaphore(self.max_concurrency)
         # field must be either "status" or "label"
         self.field = field
-        self.multiple_reps = multiple_reps
         # duplicates_policy: "drop_all" | "keep_first" | "keep_all"
         self.duplicates_policy = duplicates_policy
 
@@ -187,8 +183,18 @@ async def wrapped_process(item: EvalInputItem) -> tuple[float, dict, int, int]:
         total_items_sum = sum(total_items)
         overall_accuracy = round(total_correct_sum / total_items_sum, 2) if total_items_sum > 0 else 0.0
 
-        # Choose output based on config: return box & whiskers values if multiple_reps=True, else return a single avg
-        if self.multiple_reps:
+        # Infer multiple reps by checking if any container has more than 1 rep
+        container_counts = {}
+        inferred_multiple_reps = False
+        for item in eval_input.eval_input_items:
+            container_name = item.id.split('_rep')[0]
+            container_counts[container_name] = container_counts.get(container_name, 0) + 1
+            if container_counts[container_name] > 1:
+                inferred_multiple_reps = True
+                break
+
+        # Choose output based on inferred reps: return statistical summary if multiple reps detected, else a single avg
+        if inferred_multiple_reps:
             scores_sequential = list(all_averages)
             if scores_sequential:
                 series = pd.Series(scores_sequential)

From 111a4fc99fe6ee738fd9a1c27a16a641dff7f51d Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 20 Oct 2025 02:12:29 -0700
Subject: [PATCH 067/104] readme updtes

---
 README.md | 27 +++++++++++----------------
 1 file changed, 11 insertions(+), 16 deletions(-)

diff --git a/README.md b/README.md
index 8fa005bee..e446c133e 100644
--- a/README.md
+++ b/README.md
@@ -779,22 +779,23 @@ This will write the files `status_accuracy_output.json` and `label_accuracy_outp
 
 Under the hood, the source code of the evaluator being run can be found at `eval/evaluators/accuracy.py`; this custom evaluator is registered in the NeMo Agent Toolkit as an evaluator of type `accuracy`.  In `config-eval.yml`, under the `evaluators` section, we have defined two evaluations to be run: `status_accuracy` and `label_accuracy`. However, the underlying code for these evaluations is the same. In other words, they use the same evaluator, which accepts either `status` or `label` as a configurable field input.
 
-In certain instances, a provided ground-truth dataset may contain duplicate CVEs with potentially conflicting answers from different sources. For example, a CVE may be listed twice as both "vulnerable" and "not vulnerable" with different justifications. To handle such cases, we provide three policies for handling duplicates:
+In certain instances, a provided ground-truth dataset may contain duplicate CVEs, with conflicting answers that originate from different sources. For example, a CVE may be listed twice in the dataset as both "AFFECTED" and "NOT AFFECTED" with different justifications. To handle such cases, we provide three policies for handling duplicates:
 - `drop_all`: Any CVE with conflicting ground truth will be omitted and not contribute towards evaluation results.
 - `keep_first`: Use the first encountered instance of a CVE as the truth, disregard all other instances.
 - `keep_all`: Any answer listed in the ground truth for a CVE is considered valid (eg. both "vulnerable" and "not vulnerable" would be scored as correct if the ground truth listed both).
-You can configure the policy for handling any duplicate ground truth inputs in the `duplicates_policy` field.
+You can configure the policy in the `duplicates_policy` field.
 
 You can write similar evaluators to calculate any desired custom metrics; read more at the NeMo Agent Toolkit documentation for [custom evaluators](https://docs.nvidia.com/nemo/agent-toolkit/1.2/reference/evaluate.html#adding-custom-evaluators).
 
-**Note: Depending on the size of the test set, and how many evaluation [repetitions](#running-multiple-evaluations) are run, you may have to lower the maximum concurrency to avoid `429: Too Many Requests` errors. See [Troubleshooting](#429-errors---too-many-requests) for details.**
+**Note: Depending on the size of your test set, and how many evaluation [repetitions](#running-multiple-evaluations) are being run, you may have to lower  maximum concurrency to avoid `429: Too Many Requests` errors. See [Troubleshooting](#429-errors---too-many-requests) for details.**
 
 
 ### Inputting Evaluation Test Sets
-Ground truth datasets should be put in the `data/eval_datasets/` directory. Each JSON file represents an experiment, which can use multiple containers. The structure is as follows:
+Ground truth datasets should be put in the `data/eval_datasets/` directory. Each JSON file represents an experiment, which can be run across multiple containers. The structure is as follows:
 
 ```
-test-set-description
+dataset_id
+dataset_description
 containers:
      containerA
           metadata
@@ -805,7 +806,7 @@ containers:
      ...
 ```
 
-In `config-eval.yml`, the `file_path` field should specify the specific test set to use.
+In `config-eval.yml`, configure the `file_path` field to specify the dataset to use. You can also configure the evaluation output directory using the `output_dir` variable.
 
 An example evaluation experiment has been set up to run `eval_dataset` which runs test sets of CVEs on the morpheus_23.11-runtime and morpheus_24.03-runtime images.
 
@@ -813,20 +814,16 @@ An example evaluation experiment has been set up to run `eval_dataset` which run
 file_path: data/eval_datasets/eval_dataset.json
 ```
 
-You can also configure the evaluation output directory using the `output_dir` variable.
-
 ### Running Multiple Evaluations
 
-Use the `--reps` flag to run evaluation multiple times on the same image and workflow. Note that **caching must be disabled first** (see [Disable Caching](#disable-caching) below).
+NeMo Agent Toolkit provides a `--reps` flag, which can be used to conveniently run an evaluation item multiple times. Note that **caching must be disabled first** (see [Disable Caching](#disable-caching) below) in order for evaluation repetitions to yield differing results.
 
 ```
 nat eval --config_file=configs/config-eval.yml --reps=3
 ```
 
 #### Visualizing Score Distribution Across Multiple Runs
-NeMo Agent Toolkit enables flexibility in defining evaluator output, and the ensuing consumption of this output (eg. uploading to observability platforms, or running a custom visualization script). In this repo, we have implemented an example to visualize the distribution of accuracy outputs after running multiple repetitions.
-
-To enable this, set the `multiple_reps` field to True under both the `status_accuracy` and `label_accuracy` evaluators. This configuration option will indicate to the evaluator that it should calculate and output the minimum, 1st quartile, median, 3rd quartile, and maximum accuracy values across all evaluation runs (instead of a single accuracy score). We have also implemented a script in `eval/visualizations/box_and_whisker_plot.py` that consumes this output and draws the box & whisker plot. The script can be run like so:
+NeMo Agent Toolkit enables flexibility in defining evaluation output format, as well as flexibility in consuming outputs (eg. uploading to observability platforms like Phoenix, or running a custom visualization script). In this example, we use a custom script in `eval/visualizations/box_and_whisker_plot.py` to plot the distribution of accuracy outputs after running multiple repetitions. The script reads from the evaluation results written at your configured output directory. It can be run like this:
 
 ```
 python3 src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py \
@@ -837,9 +834,7 @@ python3 src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py \
   --labels my-experiment1 my-experiment2
 ```
 
-You can list an arbitary number of .json files output by the `accuracy` type of evaluator, in order to compare experiment run distributions side by side. Additionally, you should specify the chart title, the save location, and the X axis labels (the label for each box & whisker plot). The output will look something like the plot below.
-
-![example plot](src/vuln_analysis/eval/visualizations/example_plot.png)
+You can specify any number of output files created by the `accuracy` evaluator. You can also specify chart title, save location, and X axis labels.
 
 
 #### Disable Caching
@@ -865,7 +860,7 @@ nat eval --config_file=configs/config-eval.yml --reps=3 --override llms.checklis
 ```
 
 #### Evaluation Across Multiple Runs
-Below is an example evaluation metric suitable for evaluating across multiple runs. Specifically, it measures consistency of results across repeated runs of the same input source code and CVE test set. It can be configured to measure consistency of the `status` or `label` field by configuring the `evaluators:consistency:field` field in the `config-eval.yml` file.
+In addition to gathering metrics from within an evaluation run, metrics can also be calculated from across runs. We provide an example of such a metric, which measures consistency of results across repeated runs. It can be configured to measure consistency of the `status` or `label` fields.
 
 Consistency is defined as: ∑p² per CVE, where p is the probability of each unique status or label. For example, if the labels for a CVE across 3 runs are `{vulnerable, code_not_reachable, code_not_reachable}`, then the probabilities become `{1/3, 2/3}`, so the consistency score is `(1/3)² + (2/3)² = 0.56`. A score of 1.0 means perfect consistency (all runs yielded exact same values). The final output is a per-CVE consistency score and an average consistency score.
 

From 53a7088ab30d9daf6c24ed18f4ccf7af72a5e222 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Wed, 29 Oct 2025 12:15:59 -0700
Subject: [PATCH 068/104] changed accuracy to be per rep, not per container

---
 src/vuln_analysis/eval/evaluators/accuracy.py | 194 ++++++++++--------
 1 file changed, 106 insertions(+), 88 deletions(-)

diff --git a/src/vuln_analysis/eval/evaluators/accuracy.py b/src/vuln_analysis/eval/evaluators/accuracy.py
index 1c2e5e7c9..348f20eac 100644
--- a/src/vuln_analysis/eval/evaluators/accuracy.py
+++ b/src/vuln_analysis/eval/evaluators/accuracy.py
@@ -15,7 +15,6 @@
 
 import asyncio
 import json
-import statistics as stats
 import typing
 from typing import Literal
 
@@ -109,40 +108,28 @@ def _preprocess_ground_truth(ground_truth_list, field: str, policy: str):
                 acceptable_map[vuln_id] = set(answers)
             return acceptable_map
 
-        async def process_item(item):
-            # Build policy-aware acceptable answers map from the original ground truth list
-            original_ground_truth = item.full_dataset_entry.get("ground_truth", [])
-            acceptable_answers_map = _preprocess_ground_truth(original_ground_truth, self.field, self.duplicates_policy)
-            pipeline_result = json.loads(item.output_obj)["output"]
-
-            per_item_accuracies = []
-            generated_answers = []
-            question = []
-
-            for result in pipeline_result:
-                vuln_id = result["vuln_id"]
-
-                if self.field == "status":
-                    justification_status = result.get("justification", {}).get("status", "MISSING")
-                    pred = _STATUS_MAP.get(justification_status, "MISSING")
-                    acceptable_set = acceptable_answers_map.get(vuln_id)
-                else:  # label
-                    justification_label = result.get("justification", {}).get("label", "MISSING")
-                    pred = justification_label
-                    acceptable_set = acceptable_answers_map.get(vuln_id)
-
-                # If this vuln_id was filtered out by the duplicates policy (e.g., drop_all), skip entirely
-                if acceptable_set is None:
-                    continue
-
-                question.append(vuln_id)
-                generated_answers.append(pred)
-                per_item_accuracies.append(float(pred in acceptable_set))
-
-            total_correct = sum(per_item_accuracies)
-            total_items = len(per_item_accuracies)
-
-            avg = total_correct / total_items if total_items > 0 else 0.0
+        def _extract_rep_number(item_id: str) -> str:
+            """Extract rep number from item id (e.g., 'container1_rep1' -> 'rep1')"""
+            if "_rep" in item_id:
+                return "rep" + item_id.split("_rep")[-1]
+            return "rep1"  # Default to rep1 if no rep suffix
+
+        def _group_by_rep(items: list[EvalInputItem]) -> dict[str, list[EvalInputItem]]:
+            """Group items by rep number"""
+            groups = {}
+            for item in items:
+                rep_id = _extract_rep_number(item.id)
+                groups.setdefault(rep_id, [])
+                groups[rep_id].append(item)
+            return groups
+
+        async def process_rep_group(items: list[EvalInputItem]):
+            """Process all containers in a single rep together"""
+            # Aggregate results across all containers in this rep
+            all_per_item_accuracies = []
+            acceptable_answers_map_combined = {}
+            test_set_file = "unknown"
+            container_data = {}
 
             if self.field == "status":
                 answer_key = "status_answer"
@@ -151,75 +138,106 @@ async def process_item(item):
                 answer_key = "label_answer"
                 gen_key = "generated_label_answer"
 
-            reasoning = {
-                "question": question,
-                answer_key: [sorted(list(acceptable_answers_map.get(q, set()))) for q in question],
-                gen_key: generated_answers,
-                "test_set_file": item.full_dataset_entry.get("test_set_file", "unknown"),
-            }
+            for item in items:
+                # Extract container name from item id (e.g., "container1_rep1" -> "container1")
+                container_name = item.id.split('_rep')[0] if '_rep' in item.id else item.id
+
+                # Build acceptable answers map from the original ground truth list
+                original_ground_truth = item.full_dataset_entry.get("ground_truth", [])
+                acceptable_answers_map = _preprocess_ground_truth(original_ground_truth, self.field, self.duplicates_policy)
+                acceptable_answers_map_combined.update(acceptable_answers_map)
+
+                pipeline_result = json.loads(item.output_obj)["output"]
+                test_set_file = item.full_dataset_entry.get("test_set_file", "unknown")
+
+                # Track data per container
+                container_questions = []
+                container_generated_answers = []
+
+                for result in pipeline_result:
+                    vuln_id = result["vuln_id"]
+
+                    if self.field == "status":
+                        justification_status = result.get("justification", {}).get("status", "MISSING")
+                        pred = _STATUS_MAP.get(justification_status, "MISSING")
+                        acceptable_set = acceptable_answers_map.get(vuln_id)
+                    else:  # label
+                        justification_label = result.get("justification", {}).get("label", "MISSING")
+                        pred = justification_label
+                        acceptable_set = acceptable_answers_map.get(vuln_id)
+
+                    # If this vuln_id was filtered out by the duplicates policy, skip
+                    if acceptable_set is None:
+                        continue
+
+                    container_questions.append(vuln_id)
+                    container_generated_answers.append(pred)
+                    all_per_item_accuracies.append(float(pred in acceptable_set))
+
+                # Output the data per-container
+                container_data[container_name] = {
+                    "question": container_questions,
+                    answer_key: [sorted(list(acceptable_answers_map_combined.get(q, set()))) for q in container_questions],
+                    gen_key: container_generated_answers,
+                }
+
+            total_correct = sum(all_per_item_accuracies)
+            total_items = len(all_per_item_accuracies)
+            avg = round(total_correct / total_items, 2) if total_items > 0 else 0.0
+            reasoning = {**container_data, "test_set_file": test_set_file}
 
             return avg, reasoning, total_items, total_correct
 
-        async def wrapped_process(item: EvalInputItem) -> tuple[float, dict, int, int]:
+        async def wrapped_process_rep(rep_id: str, items: list[EvalInputItem]) -> tuple[str, float, dict, int, int]:
             async with self.semaphore:
-                result = await process_item(item)
-                pbar.update(1)
-                return result
+                result = await process_rep_group(items)
+                pbar.update(len(items))
+                return (rep_id, *result)
+
+        # Group items by rep number
+        rep_groups = _group_by_rep(eval_input.eval_input_items)
 
         try:
             tqdm_position = TqdmPositionRegistry.claim()
             pbar = tqdm(total=len(eval_input.eval_input_items),
                         desc=f"Evaluating {self.field.title()} Accuracy",
                         position=tqdm_position)
-            results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items])
+            results = await asyncio.gather(*[wrapped_process_rep(rep_id, items) for rep_id, items in rep_groups.items()])
         finally:
             pbar.close()
             TqdmPositionRegistry.release(tqdm_position)
 
-        all_averages, all_reasonings, total_items, total_correct = zip(*results) if results else ([], [], [], [])
-
-        # Calculate overall accuracy across all CVEs regardless of container or repetition
-        total_correct_sum = sum(total_correct)
-        total_items_sum = sum(total_items)
-        overall_accuracy = round(total_correct_sum / total_items_sum, 2) if total_items_sum > 0 else 0.0
-
-        # Infer multiple reps by checking if any container has more than 1 rep
-        container_counts = {}
-        inferred_multiple_reps = False
-        for item in eval_input.eval_input_items:
-            container_name = item.id.split('_rep')[0]
-            container_counts[container_name] = container_counts.get(container_name, 0) + 1
-            if container_counts[container_name] > 1:
-                inferred_multiple_reps = True
-                break
-
-        # Choose output based on inferred reps: return statistical summary if multiple reps detected, else a single avg
-        if inferred_multiple_reps:
-            scores_sequential = list(all_averages)
-            if scores_sequential:
-                series = pd.Series(scores_sequential)
-                desc = series.describe()  # count, mean, std, min, 25%, 50%, 75%, max
-                # Round values and make count an int
-                final_output = {k: (int(v) if k == "count" else round(float(v), 2)) for k, v in desc.items()}
-                # Override mean with overall accuracy across all CVEs
-                final_output["mean"] = overall_accuracy
-            else:
-                final_output = {
-                    "count": 0,
-                    "mean": overall_accuracy,
-                    "std": 0.0,
-                    "min": 0.0,
-                    "25%": 0.0,
-                    "50%": 0.0,
-                    "75%": 0.0,
-                    "max": 0.0,
-                }
-        else:  # Single rep
-            final_output = overall_accuracy
+        # Unpack results: rep_id, avg, reasoning, total_items, total_correct
+        if results:
+            rep_ids, all_averages, all_reasonings, _, _ = zip(*results)
+        else:
+            rep_ids, all_averages, all_reasonings = [], [], []
+
+        # Output dict format with .describe() summary
+        scores_sequential = list(all_averages)
+        if scores_sequential:
+            series = pd.Series(scores_sequential)
+            desc = series.describe()  # count, mean, std, min, 25%, 50%, 75%, max
+            # Round values
+            final_output = {k: (int(v) if k == "count" else round(float(v), 2)) for k, v in desc.items()}
+            # Add the raw scores list
+            final_output["scores_per_rep"] = scores_sequential
+        else:
+            final_output = {
+                "count": 0,
+                "mean": 0.0,
+                "std": 0.0,
+                "min": 0.0,
+                "25%": 0.0,
+                "50%": 0.0,
+                "75%": 0.0,
+                "max": 0.0,
+                "scores_per_rep": [],
+            }
 
         eval_output_items = [
-            EvalOutputItem(id=item.id, score=score, reasoning=reasoning)
-            for item, score, reasoning in zip(eval_input.eval_input_items, all_averages, all_reasonings)
+            EvalOutputItem(id=rep_id, score=score, reasoning=reasoning)
+            for rep_id, score, reasoning in zip(rep_ids, all_averages, all_reasonings)
         ]
 
         return EvalOutput(average_score=final_output, eval_output_items=eval_output_items)

From 4391b9e1d5a0a7ec28791d2f51ebc98c58b61e49 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Wed, 29 Oct 2025 12:19:29 -0700
Subject: [PATCH 069/104] remove plot

---
 .../eval/visualizations/example_plot.png        | Bin 38224 -> 0 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 src/vuln_analysis/eval/visualizations/example_plot.png

diff --git a/src/vuln_analysis/eval/visualizations/example_plot.png b/src/vuln_analysis/eval/visualizations/example_plot.png
deleted file mode 100644
index 9303512e4c3a055081cff62721336a1e649b7357..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 38224
zcmeFZ2UJzrwk?WU79|GCfB^wZ1tlmHA}HBViW^B1C94RCgJg!l!2W5`gugCmRD=cTb~d1juUf8spzliKkWJS%G>?umev
z0w0?u=bj!doC$ko(HbhaL+2{v
zuTz7K0hLB|gDJ&&pX+$#_1&iYx@OjHc(p;R(*IX!_39WnG-XXy6yzu7wNhf#l}B3?)41|OU|Vf{n(aClByu2k1Fd-o1ObdhtVoMoMDv9j1!%
z^0P@=S+$sdHT;=wp%6Delu^x;#lCXHpGMnb;Pvw5%gOs5-%ghMA5@W(^ZMM?m1WT`
z)^1(OPK(SkZ5)_xwd}oPy1GlPVLe0QrHVj78h=<&XPUvS6DLkk*E2ISYm|DkPc1EW
zF4bWm5s!pT{Z9$;WsVljEA;pG(;~%3a!!&by}5p;?Z@XUge}@~ytHqn_jGkVeg6D;
zy2VZUdJc*6HyD}m!c>nsv{~PBo*79JF>iU>*{M&S>|
zll^&v$@yD0Zrm*`E$w;pCV6;g?4&IFVXc6!k~eSqryBJ&ut&GBvhwQd>z_J(IvyK?
zEVwP3H~S>GUKDTc>nO}O6A7;4(+^m*#K037IUJ8YeE4vDZ0wy{w%}>)XJ+rq1&R|D
zLdD`v-(wx0o>o6|=FgTa)0jp*m!z8NFJ?`t$4}l|_ufxKFYfvCW2;xM?jJ}
z&=)7dmySLQKWj-Tp7i+Ibzgb_F-iUW`M_@r3w8(&{hr(OlrCMmbbT1X)vLp)m)&mQ
z!@IOFUF$b21C1%+{`|TF)Pl2(*@Z4MSDR8V;-3h-9jcwHkG_nYsQ5FjLg%bQhYn4C
zy0t4_K3Ifl!v^&$S7ON;_2bj%VYO^b)^+~!f>A9+PjYr{E?%$D*?GFzc&(k^*vA()
zSVgIkkB(R-fBbl1c78s=x~qg$#4Pa1lP5af4*gVpM81)N2?ftvx5^tEHP*8puKDum
z6?XbNiLR|vF6oJ(k}m3qqUp_M&QtXoYY%j0nQCAjbUhZPoNgY}D>%O}H$}o8eQ+@0
z@#9~}a>9Sr?X)b2z~lJ8KG1Di8%gn|FOHS&%0F;~+I>{v399
zeUkQ(`q*>)V@Ig^awC^GC!y|M=r*(daJMFUQTB)8Eh?
zPt8_US?+q6VUTe0)&?F?QO%UX=||Z6(bujq;Bf}4#793MYT4Y&HmYQ8cj^s|SBde)
zypnj`fA2^~jpy2JOd-9x`SvE5^t^%ige}{*$KuPnHkHE8Be}h{LrsDKUAlQ~+YX$6
z9Bh$CL5lGW4-bEI*yM?zK}p|GbB0q%mgOt^`x4`4!iH0Gs&n$$CV$qjJg{>=puD>BkJJ}lR-rv$GxYDS^n?2ZA>&B1kd9AeY1X^L^
z0SZgXQv>zoLdorj^!gtUTNX|=>=DQg(R2K?dha_&N5?>eQg3AEprh7Lo-WoWX!MFN
zKHMu)Vb`duaX~yZNYo~J&AN3J4h6?={77f*&1%#WqRC;qj+<*_TeOFGTgr_rRT_u8
z8A(V=+Qs`@>zAfeD$ibDpXwe869)qgDuRP8iw5|TGHR1pZ)MM_D{g5YqDlW
ziQuB5^ulE?!F^(4E%b4|#1Olq+DF!Q&Gv>nn5jrFeX|MsBOU~DXR!0`DDqkC)Cudj4yX5J}MaoNbIXUCD=xn~z%tn3;c9i_Hc
z&AQpfg_PU8_
zl@fjN)y-M|yWZX+BEgks4k3qFCtt)P+FqYn_@JW5l53uzo<#MMHZ1eqbKlZSaHh@X
z{=>^yN);pR`4j7wu(a7HrfSa$(07E{?DFee%0^W0>3-+K-#WH1(;0m2+BMgJ_tHzV
ziZNYV>EhzWZogI0bKMu5>vF6OVurs=ObGF*C;nQp9T9!>sDE{oG`5b2rlzJsYe;CQ
zIeF~?U9yL+U%zfN_W6xMX?1R!j5k|PM7TzkT~F(sR#sMvh-;asm9ha#SicIozhViq0CN
z5~I+A@O$K}{{g|`rN#W)ckc?lfB)X-9gVAJB!ARC^9={X5Wh~&1)7hKJWz!}uX}%H
zZEYTzVJh}o|(3ZaMgD~25&Y)m;q
zcK2;BuL_|NtWI|E2`tCpHJkSZVr>KguLR=Vi$ce#3GTOQEKsbLehui)h53~bML2)-{h_cqhAQ%d7XC^nu!NTv)bk>zj+vT3RyS1m)vRD{zq-V0f&S++&-KhKRL+`0C
z^&0w>LPc(FYd35VIez?j50aTkhEe6A()+uPkbk7VeYx?}?BYq&S2x$oA3Ju<3Cq6X
z`YJ{_SJwi~XJ$brwUM_UUXE8!x{6$M6-Y)tMX#`;%#TlAM<-+!0JI|Z+*7tGuN^`q
z{nVvVkEKCwoB84F;Bb${W;SW}w%4Vl!3aaa+7`KQ(kl7&i`wW^OW^3VX0skF>E-u_
z{o0dKym$zS^o~K0u5)@sY^P5^d1!>)`zNQo=~)+F{TM*+a2`B}2C>mf#)ixD5D8DycKmwbT64PLT{6%44x=K95@UZny?3O|>c+dOswx$k
zstavVXSb16w0ZMpgH4+@RbYcv0QU67zG{Bw`^43Fn@FaJSyRo2M9n(6kfZO}q+FX`
zzj+hXoM9aN<%?D67wmh{9;do!`F*>0zZh!IZzN0G+qvlL=e05A3vagRAFh4lV^8oF
zuDmB|H{eVD(JXLGBcr{?sRBe6(>XRx??}CPa__r!>|!6x+Vkq)-z?qb(f0o4Sw)c$
zafd3-{rkmldLYF+Jv(WDgp3Id?LT+=D62=UB_w1?Ab!ayv
z(~9-L>Jt_gX36fk{!h<~#p6xLsB8Wf6roT#1Sh(Jro^bBz>7K7+U7j3`
zp%L}&3~C`a`ntO>%UC(sr}_!GN=9&YjQ!N(6%hB@2~Gfp6#?w?^VX)@qQrXJb5K%xT31cUPspqi!C?mfCj%&h5GgNip(AESdUsIJw!Bphq{bEXXs=1Se3B@x
zc=gP(bEL_iB;lB@n3$Mge%-w7l%8j;7Ue<`y;P6Er0kFeYwX>f{h!;Vs7ynz#ty<&_>Q|sms;+h6gblWzH&py$r?tHP2
z1v;W9)i1h_YggkL2e@ZmFzz~geF=ki#$T#Y#8H=O(SiLra2!1Nvi&JH$qbYezKpJ%Qw{BhCU{8{+<
zX%sjaNzJXTxi-D7t$EHIc?4j7TxjkQuX6iquN~=Tbv&~s~lEk4$#3KdUqY$(O)~mwuL$&I&f03Dr%pcx;wru
zT*4_m@@}ZbVbpeK>;j?@@cZ$_0M2_Hk}ZPzq$cTm|8y-|F_u;yGKJ2D;`xz$E(-E(
zc|DK6o22Q7*oRjelaqXXx5P-}Ra|XOkK)W7YN{~G==ayQ;CbD)^+2&RtMFA$3MtT%
zQ4hu#Qj2c#iin6b{yHx)Gt{iIjbFz~v>H#Ng!+uLaiT5aBxAtnNpqyvu9?mI6>>tt
z$3FvD(>Oz=+*(YkV-V=>amHz7UCkSQ)=FKU;h{mmB&m25!liW6?-j}k6n{(aa_a@i
zi0!hoVV3&(n8WqUi~QM_PJA!Gy9?xV5(esTLDZoIRA!OQmc
z&j2=f-%W{WHjNIww8~<~-YA_hL4B>7byY2fKGb^A+1WX4Dc3CwRrWB|T}4@$t<$u*
zx!Fm~BYhCSA2mYghK!f?UsF65&Z+b6pC4uwy!2a8<|?jRHB34$OG{<&#_Irh
zSy))2KyEF3o2Yi4d@n=q>e}1?`S^>}i?8kohoWYR1zLLqXoWi*L1o!MznfYeBC3*T
z!Xkqh&c?XHXTPHQZ@>LP>b-m01y6vCv~DrDaE=5=J9mf{
znUGSX6)I*&oAg*wZvj50CH3MBPLsC}cArYl=hkx>77n`W@Kf$L%|d6pB0ENt=s87pVY;&W;SUhyu58uXuAyV?S$=W4F>gz>SER(?6SuG!xgh0_5~Zc3XQ`6avsn=!5xqOB
zL&ba3b)4P{b$xWfc`Q00O7_W5Tyyo%3wpfa&*1
zkg=Q>%f0mG&dNrLiy4I6hw@DYJ(Qza%-AO$i?FTuP|H+LR3Z#lex
zC4T4b-8w`ezmSkbOlIXdDIuY{gA>$TykN#cY)mrYUV<&)_y~8JKU$=H@#012Z)2qb
z;^$ja!hU3A`wJelMwZ^^viLoOCBH
zKk?pKYO9i7bILP*txWDlK2IQO5Dy9zs{8NNt*zQ1MH>)^SU5P6D=Os7TQVtx6t*aw
zQup-qG@0ys?=7xUrtIjLvuE#Kc8^Zb7M_b1m&9!k_KH~L&UcSLV@HXu0mk7^FF_K4
z-NBtnIG9CT0{eHWrpV2u^hG`=9=I!gV7A*wk&}XTUOP~qAZ+#V7$G*8)~{DnQhKm*
z)hfS-4?p>fY@0SSaDUN+Ei6#*<#%KNfd$m@Sl20&eNXT69ylO}pjHDKrv9nS{XMq^
z!IECZW~cH0m_Uf>;g+oOi3waOR+2lp^;3vOD3#;!mYZ&EPrUc%8s~0ZPB$nMG-(J2
zyG)RY^TtYcs-ff|`v?^+)4FxGPB-(9mcO-ler*L46O*!$QFPJboGE2n%C{yD)ME2C
zp<~(c_oWw`(hhK$RE3BF!l_4;$WtghlV3s@KThB1L;Bz1=oz1?;gwV+Rb4GPqZ85F
zEITTH@GP^$musL3QKM3mf~Pmzuiq(RTkvX58C8-qP>|iXuht9LV{-^tWQIUctC1`w
zK3ee9Wh6I7wP&w1p1T2QeBi6g%Rko4(F63Ege|PIU%4;Lu{ej?&WQhZR;B8T)Ww&7
zoT#=rw=09^EWlR=GOZNC%`&Z68+BG!O7`0L{Tik%4;M65`o}GAG=ofuynFn4z4>j?
zl+3NSKr2NSWRKE)@ltY5rTn6y=M=@^{`Dfkx_55hey^e^$wh%W5VMtE=X~U|=2eR4
zhFY@)8p}Ivq%@56lMw+OutrZ=$XxVK;LMg-6=G8~lr1aXbu~0jzqqLDfYZ5vhq@^J
z0LQo~-n};o=@?iXvVUvhKz65l9l*Xo|L2Jquwj0jm?Sw=$vjryz(e$iyMosB?i{R7
z=>6L6&Uew)HnUuYftPh&B7i>em97S2iES`?Ve2PkPNKZfn5NodDveY?Dhb>6DS@z*
z2YRYQ9l}K+A=!SUP4h&OKoh2AisTg9B-SZP?PT)i30jOTm1oLlCtIA8^YxgMq4s{z
zFY(B{XW|1V3a-}kLKvyGc9|Wgm#d+wNw?T4`*Q>f_mhz9AN1$EZWZ3xIX`hDxyiD+
zETt_f`-NV7YF)f;{KPtR0=}sn%U%eIc=>C|(G#X%9+@QO
zU7&Kvd?|Cv$~qgs@6FJk_%!-kleoT^{ZQc3S9e~f3pD2BGjj>^e9e7!a!6N3=Hdfc+tj2ir)@K1~#wMa3
ziv=UaOYT76$L$6i>bL2;MgZBK1~-a4Ok2+M*-^2vu{tj%m5~b{atfQbe6UXC)*X_Q
zqoE$of;Mra;@(tquM?&tS1gLx+;br;lXd>!OGM$iPp@tXyG)yvQk3UYpZxUGPc>%+
z3cJ7Nb9dH{Y}vLg1{mqx_NUl9eVt2-+0lyO{FauMh>nA|_lI$(dV}uF8M@31M#*|S
z6Co^jtx)>rVME!0Fgf+xZ&%*j
zVVRY(xqIh&4O8D|&A0EL%4~DAvg>}k(QWL_HmEpnvKUv0y9flX*451jAJa{}aJ{Zq
zI$9xA>%yPBLzzR8Z2N$|3pYJ_^eBFtp5qJp*I7?po|zqOA&dj7HS_J0KreApy!jkw
z<3OhZjB`;Q2^!oqU}W2}r8}qWh})cPWte2EA$veayg0UZX;Ab2a93qeN(Ux$rs5Bx7n1E`AdwCtPBY;(gB=3&dJ3VX
zDs5OkT&nQ7xD{x-kK^J)N!cA#2^)`o`>#+`5Z)`S*P0!#c0Da4qy2fd^Rpv(Z%$o_
zW~ocB|6Dmb$<5sQ3HipZ{9rb>u&^3(uvXe7Ply)!L(bb~ocWay!4M2>3WVfovk)LBK1{q~mOgq@5%ft2K=9?umjk`LymUs7
zl+RYZLFTVEYkgKp2qnU;fuwG5bas9O@wDWe5w?X>^weaXqIgGdHsXK}jrKWEy=o@T
z2se5A_RQXQfBf-BAU04*h=fyZ$WiN;6`|sdI+*Ba$UxD6*$SP;M*#>_kbi`(e&Ei)
z_OZcgw3(YU$?#Zm7c!~kEOjwowQ^+z)UvgO6~m({8Q`!WiFmU10yR<(_yDztODW3A
z9s*Z+h~@=&<@EXX{t9pu_r^;zKSN?b7nb!4Gi;QgeNKNn!fv
z_3PhkzJ9qV=Fc28e6P_*dpZq&Cv#erp>?yo(V7?q8%@;>Kp46ia+yMTJ)wnV21{Q%zjTO
zIx^Sakm-8Bji&TBr9G{isvLQGU#e_M>h!K%yEbpz)|+WoOB0KG%Y
zGOo^C@=Fl9q66E7jO4;h*6ld-I&B0+*6wPn`S}xIW-U2`pg09VsC|S&ax)8yp`QD!
zVHcfl$QguL2`1EXoB6#-tdSK-m|Bj9dB@Gmxa6
z-NRa(I=WgU`pc+E<>-=5pk2n}JSmVGrC-491)+uuT9}^+#yoD`xzp5rmcEV5MI9Ke
z&D*yJ8h`uNVIAV*##+^}0(H;3mpN9Se#cu;N!~RtejH+DsPTDgD0$Dv+~uGi?|tVA
z5XE2aFxXIe*tDUdV74dN2xLG{Q<}jt(F)qB2kiZQ6l(DgFDK`GW&$BRyC;jQ*X5Kn
zwCoe+w*rdlCdegEy#YM!4tG}Gad(wt|P
zK=uYgX@b-EyZ!7_omQ{b?-LgnS7eu7nDiL~VQhAG^FFy>J-&_I5_Ox)6m=SpY%X$J
zrltGqV(#OLiivi0Ux&19(~?Obg!u=#D@2r(bOBcE*Nx)!1|bgKhN5x%_U#_vO(Os`
zqqZDtlma3OonzX84(!_|b*);>vv*LULEaf5UG#uh^&YNiUV7(;yicSw@G|*bM=aU`
zN5%Sv*k`zle@eP0*i$XNbXZDCDtAMP?t-I}Q|`C%BFr!0kn6^%Bi*0~<@4=_brhb0
z<1_*am>v0HdZ^g|dP#`4hsnHpvQB@x8TN7|V5#m%Ahk%Yz5af2aWQHpi--UR#J)95
zOknjOPB2M!V87kNUp~A%H$5C|+JuxQ4>-VNmLSyWHWRv6DQC{wzUB+!axiWuesFYf
z(A20ZBxv-w%()HM;KS$&bTF1)`dTW@v`(bAI{e5g#tjFocF?yOA${6pJ;hGpFV7Rt
z+&aDA*ijToKJMJpn-4FG+4eoyU&Cr3Co6krSqHEc4BW@XMQ070@c0U@Ej&lI(s=4JVGE~q;D-=NB3&D`=M~SKITL2xzESJ$xTNNTC&_o;WLbTB
zsWUbE^=lxOe*k^BoUb?>*e3Aa&LdlM4~pJ6chQ5IyqVC`-}N(u#88DUaI4;%Uk$fr
zS7+co$w*GtoaB#bP1F>!`TTlKz(o3$_j>|5TSYxktyf}!*xb|RIBAON-wA{t+eOLZ
zaBx(;J384T?2wYZeO}GF>2KrgU9D&ia@EmG!D2@m5gWQJ>
zDWg!WL-hLqbvU4_2oo^}#_TWxCLo+Fa?$wsxH4)PxOb8eRrNsS)P*{91X;S+JOU&B
zbX)7oHdeE3fC7%jLy+@)2i>S}2@r<@AVxgIic{n!a0h_e!g}Oeze}p^y~FDUl`+SG*FklAs=~F1x
zxhD%)B4N#Gf#g7LS(>*4RbQ(`zC4g-Es=IFup^~|pm~Hn5l2i6_z54B0>9t5VFw~?
z5YJo}B5K1ndfAR-uUtKI1Nw%*^o)!ts9Zd#VNrIIuXi=ahpb$=a{*87+uq1c9y&F*cC4;$*C;p1bNmW!9L>7mCN<-gu%o9ORX-}O%
z8GuV0xcpo(|6CA^C>87i=*4pbd<6dG!JqR1wL458Wp#BbcmZ|L9#f8VIG#Lr?%V)^
zugT%gPZ0aag}(m&J&s{*K~Vj~!+bh1qk2wsmH<08Qu|d7)gBa;Ai4+zzBqen&+YG!_^@R1D>GaT-be>eg+osd$QM_vLsQpjpb$N>)zWDNoH?LoJ54?>l
zfYX8SZ_}flDP&QhLO+M)*4^J92M2|)Wrq$?ZwR9T!0vZur}Oks9KAeX5DdEts`k3k
zi;sU1G2nfBw-R3W7^5%NKYPEjb7@h@L$
zph}eIi83%e-t;k=%5aQuRK!SA2)dreFU82VV~1u*(nk4H
zIry{ze!a42olXumR5CyvXHeU#V`+BN>0I!=WOq0xoS%Z2rvowR90*O=S?ZBQ@9hve
z4Q9yVswji6bVNi1nZQhl*}(h0b}%d+<2sj|lDP;>Q;(?CuGI^~VM!Dp%q*dRzz37L
zGZ|`1dnD=dtRY1|rKUzHTVXRpVq&_(p<=-Jbc>Zr{ot5l6N4EZuk#z^OR5Y$(n#Dk
zq?!a#S$~pQQZ;c7rDWVP07uxH!iv9&S33vR*Y3;5_yp>pSlCc
zvqta`+!WTMRu{rPk$3=D!$kqYIavRU7$c7vp1QIS4g!Gug%4*IiUyM%^SuDU2eRzH
zUvL^1K&z%GV^wWIY|4}aH*Mp$Y^U^1q%O!Mh;!}Vf0l;*tm6+Ghc73H(q}bOJu_N=
zc9q^Qhno)H
z_Zau8R1H`==tfBv(&Ee2fy$PUp#arLfUt#`Sz=4Jr2vJ5K@jY2&BH%2GY150)4zcS;NmIHS6qAJY1R^<
zDLApNC_tSG;imU)-+rgI^*McPe4KYWAb_(zS-17z6u$91ukX)(4-#y!fzQyORrF+f
zEli+__0`HJbMpAm-poai;`a%uMyW+DG_g+F^=qn#MH@GTn9pRh%*l01nreA9bKoKq
zpc^?u?wVM)Q~0Vc*rI5;peIDy9-D>EvP^^V@ts92pN&|n^+=vLeXq$T=t$S)8I*32
z1kL7M%*-LdO(D$o)`MEzUtLGqa=ojxU}PkO_?ItVB&M=V>P{1dHq2#2bgI8j1}|L&
z;$TR&Uzj~ioxZAX5)|K}s
z$i1GKH9z_AL6wAQz;*af3cE9J>?7Yl_AYgYhCt`>+g*M~j=g7x)00Fi+cX(VXJ!mP
z_1Wj1FsfS=@$O!m-(leE=f{<7`J&1w;~gTgNO01J82M#`?=J>kfJuHd+7qntjprZU
z0-djRKcG2_jUzdpk
z%I-}Clu!fzHa<0V&`10DMfsBLQBhIEzy(LdIp8fwA4xSeHJidkmd(YnuZOZ%pCyhc
zmaSXWOAh|7M&JQA_Zf)UVz?m&s0Od$Ch6RBt6u%?VKy`(HpUENMa?6tww_-8JEeU9
zuulO}Vj)xfppo<2_fH3gu1{g}z+SWE1q=Uo=d*8o$o2Z+c2ygWa&Oz_GQD4uu}0mCfOc=M#b4Vuly5e0RA@(syg2X61AuJ`Be?Q
z-%UYt6O--iM6Wn|k#*N;z(7T0X+FBYn#A`@xsrdyuS)#TxclVIGgLQWJ9zGL%RY_&
zHKy1nKd6f*Fg9Xw#)5j%%Cn6_?)Eo5kJ%=8523q{XwsT8gna!re%5ZZLl=gQ03bNl
z!&e^f4|6;rC@82<8U$R$;W8vh^5E{>y9reR=i1@RZ+|{@>XazEF;o{s`A6UYo;-cJ
z3{oT{bbw>8LR=$NAzrlZ;!Ey}e~r3_x-ucO{>I>g-6wrRLzA#xdgi_X7-?W!Kdxux
zWM=Mt$cKf2a-;#NF3WA+p3o)(FzvXyxlIAF#9>{3H%
z8DMox$lXx8#2J?Ro4{a7
zm>kHdxJC3O0Uz42NY<)6tGHCP#Vra(CpQaHy62xsR%3GpO){RY%v?q`W
zA-WT|y<`ALI;#{6m?Zqn&x|$_NNn0fOs_Dg>4d|VNn#m**Hgl04a2?CZ<_)kt`PMw
zFrKYzLxGbGsXS@12kaX3p1E@Tfk73a5brXrf49
zgPpK6*H}!XI^fcG)zyl%Z;Z{LLW>7nZ%~U-QvV*DlrG)b
ztdqoXe)vabjzj{6*KXtYRsE8$i3**MGSHTrA%)j-La`V7B!ClRKL<^6Y~0rp3I!^bzBkGh-P-a%~NC~Zk$J2((4l-)+X%V`30<*7Wb8RSS>
zBybmf$24^RE7%sEmbRkb)&%Ex0BJuaaSb0qzk6%c|)k=
zM$#v=2g&KE5#vEiA1Wd38ZCt`*#yx#jOt;=C^S-h5-o{T7)T;fgt>;rRtr`tc$#DL
zNAk`RDApxvBc1$Lh@I87PC9U1jRs1d`1rwv>S9ix!tn6~x&10cWHE5?9F8AXR65PBnEKU*P##$gd3ef@Xb=QeKKNQ(sH;%|NS+ISaO
z7VoOH5S-asH!?8fNbK0LgUG?0aj;yY$mI9yA8F3m-^kKfw&i?z{*z>(H
z4N�xl;E?OVp9&{aCK)M#6%_^BpJn@L$&sojiZ)`1g7HFZ*m_Ujx2$IT*<8c=iMR
zRMiw`@0^PA^7=i-)zjb4!+ig4+D8N-!!}l5#_6fo2$0C4TvC$DKc#dVF<|wNKgtqm
z!7nsa7s=KSP?Aqj@Ei>}|2(gTmXjhGl+wF@DP>4XgwKaL8_g2Msw>x&F&cQ04ij*5
zrGD@&CHR+Rt|2?<7}qwyhSbzwLnWM)!Nl~6T-?G<2x!lu$nSY77E2Hw{`bf-KoT#_
zU%q{X0YW5AQt$urX_kZ+&c*(MTJGX%sz-XWR_6=zC!BZGg%a1MthGYErKu_3qId_fg8kr0i3Y={Q7n|#}R&h
z1)6Gxk-U?W?&5A+xUthp7Y@OB$NRdRDN}FhAr!tpMoVUARs{7qC81AUnF~GWt3Oz4T5d{
z-)_AkKl#77yX*f;Kf$suv+9xXS!fp;5Q0#{eMmPvixlfp_Vz6|RTsV@XY_}CfVpEe
zBV#%I0C2UFWSDv3#qXQ0NRp{|z9S{w;+nV@o2WACJuHF%!ZDytB}pe&8SL^CNWifW
z2GEC7tYn9_tysb_Wf2U=UmorxU>{Ka|r!r#M0
zLU7mu8ePD@)xtJ9iC!Duk2=e%Gm`0NEvUVG6i}MJP^!AVN*q3{8ty(9$Jvhpy#ZUB
zNv#vkVlE1K6w-)fdOlprjdY+bJDBdY+mlH+wpeC&MG|x@+U0ovs<(CXY|mkD`T6@t
z14e9%SCE!2KH;}^{rZEhCzpRbAd1vvHoXcIQZ3ZOxE*5%iY_5AaQkput_t`ckLz|F
z)bfLbx9f15(ZyAKjr9!#lz@I>LL(VKT)WU_41n7g_KZ*zbRS}N?%X*zIa9lpgM(in
zh3fWogN{G7ut7#f<}3|LpRRu#AXjqwYA0}7S%&3X4{E0Wh8q(GfSp|s?3!+O+STzhz9adeS3TS65C7ywzwotWqgyY-~((gsHeS{c)j2+gz_z3q=6m1Bn6TV~L8?!k3k&*E5_F9BuWf!l8_j`xIxbM4MHf3dH
z+mSX=JW8S_>QZ_1M97{wW2IG6#NYI$v{bx7MjS>r@|?Pl_vVFkiVRoTiS~-Wm-!#8pd|GjvU(X_skdO-YbmQW}e5kj+7WfQ(qdK$^ra@qMig``-z_t0~
zsUCO@{&%~#!h3)#(cfU
zh$SBZMIpXom8mAd#pRo2kDW3Ss`~6KDri4+o{(;28DVdR`tJZgiMF8x(vyTO$&*^u
z)O4Qo_054moq}~|034VGVHb?6IS3a>3O=o8W~5jWtvamFKd;sV*s?W&;>&sufZI#}Qk`LFnuMQ5Tnde8p(r^c!MP|v&}d-X<@%VQukA^A+~@RI
zXn=8y*?6RYfk}pM;Z!%5)P?fB`}UO>?wC(~>G~dB7ILVAd^lr5MVwq*5}?(5FFn$-
zNQ}7lgtsvQG;af%^`3l=e;S3-mZF^OkdsFRA}JkBPOfO#HEln|AV#i!?uLkR@6SJ#
zpwe(s&@oaii|r|wYkxAD82xvpUWiOmfc$7-bLk-F~=3!ksAZzQ(zVC9j{3gIez9TgsZ
za^WnoNWv_c2~@1>HdQ}R6LI6j$&&;m6Js4M60@uX+tabQid|&%sz!9FOxapgA@%6S
zB$a1Nj`dUQ_z$t?_LAmJ(%;zi=C$X0RCfzlILk#WA~pqKO$~N^G}%sO7ca<@hDk^^
zvR2=F!R_gDl1X&-QIpkoyk|039;2(G?S^ln;OQl9ixHlg>SX}Vz{~$Xjkx|xNcP{q
zV4QdGM+WZjzc8J0IN>PkI?rl@z%MBH%7i@0ZNg@~f;$~KfKVOakqukC(M%Z7oEeNN
z1uZ_I9|6^sUIN)2dq~m**|&G^+~M`%tHh-Htq=H#VGGu-wknpJa3-
zfgGQ{0>aL)eH|){mqm=+;vkpkWnfPUrkLt`f8z!t3eF3it9%ch;JZeBXJlr$MtfbvnEE5uZMlto&4tnzs~8@0rjD*iyivpSymnmt
zN@~}~6$~F0hFlm-@$ym=U$Px;vi`!rV8u8sd2I?OB%D`1w}Ex&s3QZz7RGsw=Z{cq
z9O$L~^^wj7TxExX=e(@-zph%1t6Ioauh<#a+p|e9FfiS4W4_?_ohDm8RN;JyaHGys
zV7oX}zHgEtfombP`4J-INai{QrE_T*AsmL!mD{YpJU@;xb2rT}G=RUnAo@ns@#Eta
z3|6A#6ZCdmpaj@_zvd|U1Z?I7%~Rj6`M3d(HAo&S^Cg>vRSlkMKgOFyo=VT1k(mw3
zlIFGXItE1@`cfKXhJAy@zKLo7%a-BqiWn3N)4HDA~EgFHGa_pbiZW4;PgXzMSDsBDBha
z{}kRAO~G_)0*eDJ5>zh{pokw3>UKBU)BnX2g&=B+vam5|*jS-@T1`wSq@uD;NyISunc-q?9TA`fu9W^d}0WNhaYcdld7@LOm
z8UgAi#*VJD0p1}6{ad~^qB`ghqdRZjwCQt>TU$L94a+K4R;ipkuml3kDh$F@?uS3Zk_BaOO
z-FBbe36vvTT&GY-*|a2M@s=57@bK~RH9iYe!8?psjob5XDsKqNJc`XTWgfbqxZ|M<
zaOt*9XQZd2`WPD{+CAq&XI_X+M^{%Dd<`ml2B{ab`#&d_pK6b8{}_RPc~vD{uX
zyT4X1;BLNruTFsO!CN7NRQK`-LQ0M2IbQi_*2$UXk#4YLd(LDeoZtN9#YXy_RF6E{
z&SlTpQun&~n7d8Vx~l&3G1xXvjyj7zarC($b%*lYO<;Xx0M*ZdR|Itx3t(^+-0Rhj
zf?T}I!>A+p*hA=3Q=+55t^#c3}8rW3Gcy=%0>2w{t8YFeXC1Xo?`U
zt2ix$lbn~)Xqr;!A5#JDHWB_PK3HdNG4JA6pC8t^2&AZ$=zOi#9w*k6Ce1*_ne8~I
z^j@{ciSb%GAe+bEkgZWe3vNERz)J@2xCIuP9iIXlNFgSiQ2#U!j-5NT|L|Ylhx{_w
zHIPvJZO7a_G}sRUcFWVKg|F)UKvD>s)c!`BFrKEmD=@z#Poy;dU*YapvC-UK4oWTj
z+@Txv(T>a+hgcaIR%*y!Yr3|t9qw;KM1yD|O5tKAM(V4$RkvlJn+Uyf`lip4}Mt=qU|Em6?+Lf)BQA<
z+~`}$&sfctxn1##B-##IV2G>&1v(a71CZVcXdrc9LD)y{@+Ni^x|qTg0-@~zjYc~R
zQz*MA^*5LJ@C=cJn~$ZC|xU7{kSE52%UJsj$b~ymg~UD1J&*7
z*iIr&Gvt`De3m6VEyLqW|F>zgja4fnBcolDefc_?jp>ynurVF(^X9-tWuf)U@0yn<
z`(^{W=lkK$R71xMY0*)jNg9%fat>b%YL*04cBF-bxOS>2
z%c+BRMfOc~mt*kX#FEk##|mAA$Ab0Hpyv`aAzwMRTKP#z0s74eYmbKUCs;w`SPzJB
zwP^9Qg%vgq$qf?2ros)&;X8!aZzhuiq?W$>Y!oElgB0*2Xm=s*66kWa?RJg*p!B2O
zZ&|*-sPdBtJ9My;`;$dN%tVNNFOaK$&qOC!whAjcmqU$
zB`PtzkzRsiDUl@(SxX=EjzTF|79|9hDeV88jO8$Ljp=z(1mUX*osI-6eChR1Frz$E
z>arVdQ7hT*AhOH_Dy_ilE5e&{hEoZ?9QDXCqBd*JP`b3K{qB
z+jkZEGiiw^KSX*vP`>#2`gT`QmS>jt7mOIoJ^^yvoqPA}H1~n!#*44Pmi_$CYp@p*
z$k7}2&j}$ae5K|X+(*>j$mIpY@<99KAbN2>fH_-Pj=l#~BETk}Od7BhLjyxg+&2>8RzA7wd4hpQ~OG+zpg){B@YL25KtHp?Q{WakP
zCKe*p-8nY3QUhEX%S}4|0M)9EE(&z;)U_h0la)+eDeE`}UA1jC8lP}4amwW{a@BECk#8a&
zPX2S^2S!H)1X7k<qv~b1msXQ6}s-z`Xz^4WhG2gs0~GP0*^VMK$h125QLYX-w5fgRjxCM>%Y
za!3|BmVYG{AT&LbtxsI;Fu3%QMvZ@Oh0eN`6n$yK%a^<7hCn+lpIk-2I+WWy>|cT{
zK`m7L^LMHsO_9?G>TIedHEMm9gOTd57KM_I5bK>kd2VadANB#%)WMKYLEMGQHvkxs*`y*s8o}iwsr!9s_n~
zPjg5h;*la|Zf-Dz*z*QTc3P@}bEA)rhQ}o0XgfgQ&sFcfe_=a<2K4&&F*(x{Cw?au
z?ijC1BOk$=-=#Zl5)R-TmX(n^#-+Uf?Y`;{n`?v0GTL3D!<0gQ3HIg+PPbrUP?bed
zSypP~b#Yi=??3g~Cxqw(z~`ILveG;fRngROl;{Gb_Ujn9HHTc9W*&dzwHUEBm60QJ
z!jR*mZOQzjQtofx_%y+Tog4nrV%YAhyxC=sIwGIIpb^8ds5eE7v94ICOS9D;Tk{UE
z3$CDdrd@uTc+D#AYA8+#jC&_5BBAmFgW;cU%u2JqXGULCarjg=AK(0!D#`Q;u?98*
zn>i==l{V~>Ns2xvnFxWZL#td!q@4fzr4Xo#I)xh%1`{@Kta;zi&dhFuAPMf4R
zEF1O8VNQb^!dBA=&I~wGKA^s9|3_qnlv^Zc?
zUcz7pVR4za<(z|{X7%rFd~y)U*bkKux<~sldZ(M=N3Bl~Yas|(VjcfGzBuJ(q!U8D
z%$J9Br=!A[Y3#8z*c=0gOqL9}ZSC~$M#j(=uVb>sme&N+8KlsW7$qC-x7geQx}
zY2d}#?=3!3gJa0x@NM`zSMM)H{v@7P(vGrh#{np!ZP~FS4ggpaP8Qo4&hHsn^|tOv
z!E}qM9aY#yK*r$^1=nc?;Kk+@gL))jnmy>{l`ZNv#=|lHyqlUg)6kO(Npvlhxsx6-^
zL5@Vo0%3vU=uF`P{O6uYf_6tT1vyNCw7CO^1exXto<_{lLK`GG+Trh?0dl60<0vpy
zF;GoU5E&MA9v21O3?%PNH5uIJ`Q?}A?LNz>PW6}ta}tdl?ua84UiKe+zGlsu)7gZdm^TkOSY5K8UT!GM5;k6UTq*b!?1R#Z2PkL3Hm~&s_e#gdVh>
z7{p_sO5`MT3T6~8Z1m<2ps5L7ehs2BlXk(zZZZZ2zhh}9OfOw}QnWCwE&u57yZ3Q4
z!PA6Kfn$)|o6w-dQ_fcS({i{}I{$_#3F}Jz^y!;kf`s!T_BNOzV^PD5TwNAK9&h4P
zzao$0G2mszQ4{1yIcP|gy2HyE@)*aP9pt1fIFNw;iIWgCYZOXvZk*_(JG}fWs#w=a
zM9@MRPqH^TKN6{{KBHPHfuIK*$yyIhoh>DI`Jo@j-eJ+=iUVjP7T+x}9K!Sz3#StC
zlP79se27upg}PM{2+u}Fd-*GVa%m?MLcs*V1xb!T`EWL4%n4C{Tx~nfsPO8+Xi|
zsj!A}a+YI9Kk+d!a4W^>njADikG3-ETb50mo?N@ekY(1aOpZK2z`LV6Lo9;JHg#K6
zKY0<~b-kCdKS#h+^pNnJIBx4P7R(@$N&_0KC&?ke0I&%4-wYc=0|R4$JHQL&roCC7
zgO}Vh5q2E~POQ@I1@J>;IZYT5J)U?kQFhg%-6ToBSPy5Al^Hg|6mCVlNH9v|?9~4L
z3|`b-|CxWGDAC~+Agcy8IDu6$7+HSIi{)QF!Pgiug{GAjH1~0ue0+Z0>83+}?I|oR
zBBGHqS|js7K}{hvClI$c{E7Yk)5~k_PBfTAyii#hwtX^Q`O$+oFRoR`a4qS?RsvhQ
zPyP`n+zy|Gmy1y=&9ZL4BO~ZeK1jhiIkuY7@LcAWZ7sneA(QB?y@FT+qoBl!SJ
zOXU?6w%^=ww1jfFJsP|?$+5|B)wiGQYieqO*mOY{Zu5Qr=bi&4yD%gtMad~&WS?$F
zy=O;Js{dDHW$hb*y8k(@vcOa9KXR3#pkI$BCH#x6EJ@p&vwRrP0<(Zl&I5uARj@L-
zD6rVc%E{SkX;jf4z`rnQcjmiqK_~eLaVJkt`*%Bwex04=A$Cq!(&JIQ#MqIBBOoB0
zadYSXeFv?5h@5F*;i5LDL83a90X1{OvO*(=AOG95zyZ^OZSoBUakRsMS~^1m=pSur
z7z)V}B86U{M0l^x26QiwtWszHAKVK2;JQG6$-{1E-Dzw!h_0OUVt&*&YOqGrB0(5~
zwWjl6bv#7{K$$CsEujF%!jzYm#{AYm%m{CE1McR@=iBJoXR%R}%V*8&M$QKeQ!VPE
z3i-D={M)zi&Hp5^P%w7=AKVKj%Lh1sIqS>xF8c+iPQy~?G_6<#^77Axx>JU^|FkWf
zG^K3E0kLoZE*~8rO`sUJDPWC#$j>WzHwRm9Z7jb{x`l{pjo3&Z531j|_w*GUjzI`C
ztSvuFH*!`rVG_VwynH{A+79}2>h0e9MBUcH;ZNEvmc>lkE!jB?Z|wfBcuB$=9qj=8
z5~$ax1=?F?;g>;T2~d9*}Ky!fOIevHlPz-zxK^48I7S=LxvMVz&n&$@ci|F
zbULpTLXPgHw4iO)pb0wzp-V6GGy?eJw;+k@5jSx1Ga6DV*PMOq>o-_~jf+D^!ewv{
zsC~7tQo>=4u-!an
z;ZrA8+W$yh-f49qIpz81vIReN3JHwFV~(FB2YKVbpfv1iFwjR~FRL9Z;wIxeHmjVh
z@1)W(R<_eu!*4cAz|W~;#pCqP`MbEoFz-y(%bg<(N`l{goBzKj@qfp8@CR9pH|Y};
z)cr?!RWsxUvkHLJB2OhE*wJhKeL*NQNb$T9Sm4lsQxQDhXMJh)PH(qL!gS
znHpqDDYPPyp%zK|^~65=ch1@S?6a@yjMugMYpqu6`~3{>_w#<9`?>G?ImB57F-|I8
ze&-2+)oL!-|9y~N4Ku2wqbs)jw+gR+-lOcam_q-i!*`w}O%af<`cEorExGtV@*HgU+%V#k8`o)P)>FcJb~oQc%
zHsNpyRs!hmLT_B!WTG+^Po=w`@F80Ih2|NZ%2QOSuMjt9(T#|uOq10qprin1YnIeP
zT&H`JPoh*+hS7>Jj8>Xk11Sut{y51jQ@U+r->s(#MVHg!81@xoIMCxah(1qJTn#5O
zvL`)i!HlB^$?@1+!jMyjVPpHU{Zq_l$(E;(c8G7=nebdWfusw{waqI1A`edSxp(fJ
zSh}~9lmcagktaEY@MT~OqfwP+szl&Z<#W;p`d{rLbr0&3@?~%OrKhgT6PSVF
ziL(OWVS0H5y*@Qu>V#AFT~1>2^Iy?dgVR-u;!OS)qtGQ)r+c$WbvIH|ivbu=(aruD
zxc(#BQ-;y;7K2i!&a@_#9(wh~Sj0&x^f6;>P$g5-+?kY&N>$~0R%WLCQ_C4A?FNEp
zj#Z)^K~oSKE>1DxDrqLU679GB_vz;}#}fXg>Jb7|xFe0-!@PO3Ox_tj8
z&@zKWjgkK0tc&it8>LBo!IP{@{K@#s(EAh^>ZE68(ZOrKYv?u2=+FQ?|07ZL&HE;+
zXivjb)i4vip`D%G0euQ2X<-Y)wzrm6n$rw)^=R96iA|$jNJt3naqnBZb)Qb#v44MQ
zY&1`K@Q7KY(p&YsZa)7nd0#-6p=EJP4jD-EA*$SA&K%qg4*f{(+Lw{t{pE2oww7D^
zucUMK!Ao4TMokM2rA>8=`$gIixMN2_?6@DD?h}7M_5dmGT%}RVUbTO)@X^C7W^-!h
zrhF?oy!pJZd-AuTv0KPzqg-fXJ56~V1Gq(Bshd`fkX&zl+f+Si)We~uInu8cyL3NVMQ^#8mi9by``u?+UfRD_#@H;N27``!ADFH4vo^Aq
ztc++ypIiQ1$%Bpg7rJQMWK?z@xQRe6un@ykY|HKz4*cGPnM(2&HHzC`
zLF-PLoQfj^d7qZYWgy9Hh>^n5s6qDrr7FLs?EIiwq=2oqk(|q9MXZJs?kfWH6n$Vx
z2&+UHMn~k)z0T(~M0!NL!x8WD`|Jyz<#VL7$FdW+pm9{~m2bGYbj&!V2ik#GA@dL1
zgLN-mYjS;yI(6!ZClj4xnYJNXz4&CAGePs~eS)uuyJf}%Y!Unm50S=8)@K=BB}TeW
zC)CEcUhV&VhUo7H+6}wp4i!74sRxvTh*MX-+;n^>=Fs2adCe&M#PD+oMJ6Y?-1{ls
zcB>1u{-39T{+q3Fm~8U%@2i9U|Nex8n-`@r7-sgb9~X-ow`kp9x}09x37FR&2V}Sr
z9S(keqr*^+AC8C{u8pih+`RkA!%B*Y%LMiDYtk2?BQz6~|NlsOz5nJi*AEHATyQrs-_`%xpeI(7WwB@^PW{7)x^orkWnN60P?lszo>AE)WD
z0M5=AaQ#o%6MpVT$mKW_&6OZjqCzzo(vtQ);auD73-}&3!674Z
zD*veYI!5{n!d!voU^t|bmX&W!RV^itreY2bn0W?}!10(r0uq3u%
zkjAZ${>AS5zN40F+&kJWFMFb8#_XSq@Q6qoXv8fEx;4SwJw^=3ToMYiYoC9wEBxt^
zBj6774s_GST4~C|ubqrMQp+zz8f~%uj*$N?ks(-=_ey|;f*46cG)j=#u8;1U$Y1>t
zdO2KYJ`h3=?9*h7k=F*u0zwy4>LoiCPf$~lWkVh
z28Qhr?Rz}Zy&|aR&;~h0Wf#r8SH$L3FAd5)pOTzxt^~(NUN?zzY|auYO^#b>YxJVFQR?MU0qz
zX@C{R-Tn0NbrjgvnOA3vI!F0cl?ON|hB*mSJk-5voC_^%O*2BAZu*0b57S?UV}BQr
zza8@KkDUF+6xWe1Eg
zf&QqkIJZ*@fP?GrwQ?_-=Lc54H#3voRp{%_BnAJ#
zj2j)$=>&461(=0?0ZH%l$9()>0G(_gltPIKM^SVlY#ykwLa}4*t+-byar;$Ku^>+A
zN@o4vryFb(QN9Pf#~w7N`*!2QRDmy6{2G}@Tf+m#3)q?#?A6O+aRh4c3SO`^L{~{7
znn$s-h4sZERUPq{~5BL_AF
z#DiIT@!2L-{>YdfCwdI^C(EVx8ujAa5Bgi7Hgd<>ht;Otf4K7feo#4)GY2pIMbX+1
zlq3TOi^Z_wd)ZR8iI(2j_}g2z2da}c^r>6WOKxmcKpOUnu_Au5CCeR6=5|Ph+8Ixcp8cWL4Y-$SZC-hyr
zDGnYsv#{XhR)+}(4O3Zh5|ZK&Da2&yO3}}ZUHRFazQ?-ipJxvj$RN3Oh<_fn+NUP|HreN*9u9o=>3@NQhVMxJ+5(_smN;}6#`(T~ym}jrB
zq(L8NJA8G2U`N1jA;WQl$HcWjgrsONVga;AtyK=i|ankREjh+K^TOT
zBT6OAr6xpM>hW6qBYh?i#HkQI!aqb~H`*C{v#ojoKFZoC5+2P!2D3~CB}UT=fTtfg
z??au|e%(hZT+d6Jw~7pj*7WW{fLFa$)xK^Z2k
zJ`CiJ=6ER9BjP9+M0k;&8BU9_NOYlPOsCVeSqv*hZh+_B4vmyP(<;tBA$NQA^ztPQ
zCnh^7ulKpC@Ef7EApb0t4Yu1CXRhdYhUyVO@DN~xg|dt$s`a9werw9d^;TR+MdB!A
z4z@IeD5+hLLtW;kvw<&!M=NH_S&Jj7;yED%KoE^FowB7+Dzyd=-uJGF!tF%Assp{F
z!4nYZXmdMi-Kpbu6+81@tx))%J=^)t*9XhLNDq^Uz*gwNu#jw#
z6v}7>YF7E(#i%)UT#2}x@l~IId;`Ia*
zyy$pGzol+J)f;x`dFjG?sDh|&IKsw+o3R0N*kC6)w9}~afIX=AgCP&SHYkV-tWG)z
z<1=9Tx?q+%=6dP$k|@@W_FqfIpmoA$d!B4SeC7o)gQS=-aJ=IFpFxuqy3*w-aV&??
zCu>2R
zM=*jBz{?aXD=Vj4Mv8-51MgqyiDVVCs1OaOy9bBU2@m|}>!Ur_(ed7{ezR4G2?a`C
zu`Jcgz1i)-&X3QZeCk=${N&U#Bg9zq8l3R?l2i3_5m$787EOp03851+PeoVa33
zaa?vN4Ro3BDb!2x5_kW+e@FGWWk7`4T;}J`pGy#_XOHRP1H+-2StfRLM{u-BaiO|t
zGs*0&cWEN_+~bvUj+CH>_(C)ij~{s&yq_SAPj4RonqNGpPGE}>y{$GOTfnj~%W)%R
z(SFg(x1s%l0$Q*|u{d||gwZ17Qbk`WcTRU5Qt?Q$OlZ_%TD<_{5ux=IB@MSeP$DQ3
z%-75q$MjgYXCuT7TtF?TVUb+14IV_&Gc;eIh4yM!n+g+A_sdFVxh!OorJTLox~Q^*uE0V0Rr#Im|TfN5Y0Jf+WW$tl(;AF5gk#$^uRhI(X&c>
zF!+K2*q|^WCFPKYme4rD53DffdAk|cd(K86wut$ptZXsQ-_oel%$e;LWgiC>5}GRX
z3MWphfX)fb3g)0b$XjwDxCBf;2{J;*&Vv&w_1>{7K2;HjDzNzs60JHsj#=SwAB8S
zXaKkLJ~^zxuKcRMG$bLSa~~+oLP{eS1Sfp^AL1G(|4M~WJ2aROW}y`C9oA(pQhrGx
zNE$mW4XMeShpu4C34nQ-8%$rdSXHf4C;6~QE=E-Th`qW|16+pVE2&t~*7(c1RUz*1
zY*%`tMVM;g^|FPFX$7YAUrmk{k=Kh1Ht=DeC7t$LkLJ
z3_Em~(=+V%4Cj2ytvc+p@%Q}CY6#k8a+S=tkX=r%f@#+GSPKWKYK@Zf#l-{95P&>L
z3XrZQiMOe284@sYeq`lYV~Z@xR%q51%G#ryz$`|*z^X!`5wD#1M{S|+Nmzt4zuDh^
zM_^^npo)r@34K%*vyojY0tfEXOBh-Av{6xE^|57QZ3I_3n>!OHTUh|;+;fl$N<)Qp
zxm)Ap%*%F!0UL^BQ7ML2)V-o35+?
zf-OyeIIprg3dhwxEftXkfxlRd&Zsd`lRAWXdi~{m;TR3*|0muJ0aNNfXsL>q(g3D+
zzj=68>=Lt1)+AmYye&GysK(%%Nt))8wi>oz*eFaZbm(ZEe$DbJ)7sT1&g?
z=xp|Ia^CrubAxd~oA0YT=-gshUzenQ
zTGOOWLbt>_c3wMiXPlp(kov*6xU9OgPQAZDV5gp$(|~lfyQb!P@*UI?y=5RL8Ja1}
zH`D-|c9l+@GJpdZKkh(c#=x-5%ihZ`Rr?09aTddGAO&nXj4leER`m=G4LJ^jh`ZZO
zlkePV8JvD}VO>+xgupr5{jz&p1C-E%0c=xuJJ#^Qd%@sB
zp#ELDj5nKjsJBNB_)FWi^YBbu*rg*l^BJA=t(^zo$2`FP!z!3p`UKlS6fG4laG(Yy
ze#to%{pjbBrl!sfZ>Xu9aI%Tr{Iz~Wt96BibFx@<3~T>DzsE>ZpzdB{FK?s(Q=|I~%U&Wtg2);{Uf=JTdF`{>BBMpFik
zm-?9rn{Fwcm*2(Ih6%>EXMKC$u1fS+{jl(-DoKM&{3l4bn^CCiq$5;;swOK2ut=Y?
z`eJAOL6?C`JQ^Dt$nDVLb0NM%HB`g;4ywgE;<+9yL=FSvf}}k-x#9%>iopP
z_GRx>S@tNU6jPDxRj*juee_nvZ1Ng~YgINTVy#qEhQm906Lu&jCi6-AAUo
z7)Bz`WLeJZp)k-DI`(ltCplc6U~k`v`V4exBcW>MlKr;3bUU@!*orUgU|IWxd)0K_
z4|R{XpSpW#&aGS~UX65j*Qe3@j3&rjvX%^HU}2c!?Q9$LM|J7^PR5=)5HByGg==kR
zXFhD0f&t8J6F=Wt6i1#umE+ttcAWjqcgeD(h4m^T9Bg^CW6jO4K6Bf#Yu7ch8VhGw
zPC4M&m&D|XZW2i^M!EB$Dsw@17k08OW&yYLW0NU7#}!k4b>Y(MS!H%rw;rig;Xiia
z(ZjaS8xklcthY;CW#aqywJx4m5|N&t8LV^lh3*y(kHe_au@T__Z>fM&dRnI`Ib@X)
z183{ZM!v=c6c!h6%qr1f0U$9faF3tL?D5A25y~wT9d&|p`>d{dbVz%zO*eJxv;Xp?
z`-l1-py*#q-hVIA*LPmVp5ZH27_rO>X|&HL7q0N82DDtc(pbScjXRk|g
zVme|(T^x;;6>W~t&@eP??+1w~PP2#YrjeKsLjFG8agP1A1MXY=vwvryT2DW*piO)W
z+X-7SrU}t^Y(hn4OufJiInXp({m8anjZT4nM}#}
z^Gf~&q;aY|W`}9JF@5|sS#{Fn$$;Q5shI|WB=+Zt#6P>!OI`ixC|2w9s*-(^ch*et
z*>;DNI~x0HP#cC~yr(izL7=lNDmFITRauTIN=Aq%$U)8L%u$CDdoFyD&!uyTyu5z>
zc5uUW`yq!YxS1URUw0F{i7dLDHJ^@<4g_13s;GQkrEPa~GP^=;l5p@-ewkn_g{PUh
zxww`WP@1_GD@yYdyQIhbl@;F7&9DHh@E$Za#ObLS8DSLc9&a86QGC~hfM&wf*v2hd
z%p+A|_DZX`6>B$dR9oRs8y~S`Gke$L6MdTHKTVqPF}%
zNMBTm$!}&Ne16Gj%ioZz=uAEB2XSw^$s1GC=M~|+PIsS=T`XL^r+6FL6JGoJezw5$
znkjz+IaILq12G%gipO&ufOX|R4aGeHS2Mh=#p+TX41NA~`TH2xIZ=b><
z?j2tq}4H((nZHE)*qz}$Ngwx
z(BIJCsSFx$5{M^%{e}%!JP|szF^f~iwHkD4K{LOEl!NfX0#pErGDGFsqp}%kCA{Ac4c2y2Hk6tJ7axRb8vs5HA5W
zpN|vqkxJv5|7Nm!gCx!+r$@0`tiGcO2~WB^F=(Z(xLWW!WbAwqD9X9*%=v!L*t8fd
z?%s~;V9Ill^%wuyCf0o0l%Rh0E;aEaDbieqGAbiAO!`D19Ub75S>KO*=V5Nds!+my
zt9I?iEMBbl_`;$_a99r@CRLKPfEZ7(v+EEaA0IXSLIBQKXsEB!@{ji@X+?q&#tB^U
z2@4~9am#G?E%s+)ACi^udclg**`B){A%NUu4L5Gulozj-6f|%!NOS$Jnwkybl$|IH
zw!>|Gha2AMg^vdjTZ)&B_oI@`B4I6f-{BA!Krtt$KiCA$cpE_%Ahwq-VN4*`(hUBI
zN9lv|2#3k_nsNS(mkoY@;yvwwYxEihr#qffiTGx7F)VByWER*`zwlXUsj0K^){t*T
zXLJ@!k|KK~`DdNV@Io_!OXC|Zqc3k5yDII8$uNKLh?<8;j>F`mJp64PC&hYv0>E=!IOHS+cHh{zqTJ$u7
z&6mc?oKGh#m6O+kP_7%iWzCwpGOA3vON*=Fz%y=m45;!gRQqpk4D
z&4$mNtI^Wr6M`zZR{e&J;*Oz)j(qo<|H$=RilDP4oSN+o)`nk^DJqsIH9xzVBHT^n~u~*-|
zJ-c@GBg5LN7rXHBZjRZzWbsKp+BTy{H{8;$?@LlC`3o5uSN33Pwb`H~uqI2Y=+)`B
zv{zL<&aW%JLh#PkI8KwfQw#HU`ujh*LKZ(8CVu;Liu0w2sW#w`E^3SflwtJSt2jQhm?FlZ
zVm5Bv02Foo=#BP6Z7`5}tRFR)zGe7_$v(dmaDL*T_V?`4=49bHu5J$Jm#FoLZWRVY
zz(M$k_P@IB*GrDmY|cT~OwITKNd)_95%t$r$7#nXo0(1Y*v7^tF0?-^JNX|#`t_3c
z(1gf-UR<32e1aTbqgmb8tBzO3KWW8x@9lWncY010B9R73(1jW|Y2xlZ!PQllovMhM
zc^L(Q`-QRqkU2`GweY6N9Y&EQZ#St>1%YQ0z$OTG`*}opULVic$Sjm%|lfp2OfW3
z?=A-qR&HaxybP&_6!K)22DD~_*Ph@N2S<-{w*XGqLPvNnl}>*|-S)b{DLsmh+4XPg
z(2UhC#Q|{j&d^Ylci!*H%MbCC5%UVI4fx2ECRs`~e#he{Pi_mlvIV%EYbBY^9RsiT
z8u+qM#y+U@AwyNwpWB$SY{L1-+a_E!
zC-X7dIHsdVXCipQ;Q(gwcu$&yj7mzgJ(bA(H7TmOHtr{C>X|jvP&HhN%5)7Yc!0QN%=Ew?|@B66ZGkmZD`T?
zCd9^vmzMWMyHP;fKw>1T<{suyii^wMyc$%|<89_dO?Y9hWK{=V?Jc^4
z^Qmx&74B6|h|)tCK|(hEPIW@gXLWV!4-!$bU7fFMxQ;s)e$959x~_fPFjknvNl+`<
zC-kZH78jaURcmT$5)OmWyh)x2Tf+4B-*pr~5-5_^0zI8&qJj6qiDqV-Qq(+yf`V}E
z^o*Kh#2X|XmBCjOEGk{P$oR}?BUhgMGbdzda0duFNa*o?*6DFIg?|+OzM)
zTrVDO>NczH$Lu1Un*{1@07cS6*fP`SldYgnBs1?SE9WzTPG|c0tu%>4P$xbp7FT$B
zNbe|fVwewu6PfFBr{C`$tb<`#Cj&P>D!-kiM#%6jfrYSeVopq{uNsw@d0X~USI`HQ
zc$F{5&h6Ij)oV6>n22!RQ@-_w&Ivj^!rUCF@~4%gqsEMmH*`h&AzxwS?AlA}VSw1Ztk2%x7d*HjFz}#h3*Xe4T!Onh?TA>oLxN{aby2
z_Jh#K$xG(Hoz|{H2lI&&Rd}x#zSY~}pE^NcfFEB*Ol_XToK6hF)j+-l)=zp-{~2wi
z?t=#H-n_X9@NW})uSd;;E3bDnaOG*skMYX
z_wF{a_pQMoKw36rW?G9W2Gw<`!)`}RS(if^ZkB}KkVE3k8`S-K_Kcl6RaKB663zy0
z7EK6RRH}3Fqfl^|iVAGtaAK(-g8ok$w>f$40S7}J1K)4P#J0uWdz<%T1e$`JXe0;(
zgZNxG4QKn7U@i}(P8Oq|_x#j|{K$*Lnd&eul|pKJRXe-O7fT9fIk
zF%@D6-}w0Li36%IfpQP5ItTK&-Nj~M_7!@M3UTwGmu#$TD)Fv%*R43Q;N%S;Y9wN+
zsGZMM$9{P{AtCaZ6HJh1ZWEg6q#uUa)C|{C=;te<+8F^Tk%nvZ@4t(60`XR1|M2}V
z)?1g_^)_?&#QAIN7Q&M9D9Kz8wVbiUo3lzhELxD7nGq)_@JsV91=AO_*UhbLSVMjW
zm6ar!-i21`fEs1Sq2g|??A
zO`Epq+G4Lpnhwo&JMIh#NrRt)6xqRl>(&IXE+e?K*0rCdrfs(#VKdqM)R?(HQ5-C@
zsH=eAcAK=7^e1(KfjQq8TDw_W0==ZBr*}6p+N(R`;*Nd$s@5Kxy~*GIYv0<3w$b{Y
zwt~^5Keuju9n5Pu9gURo2&XAvK||ZhmwoPL6Wc_Aj5zz$teSAO4FuQP^Yr~lg50n9
zWd)>H?t-`Yr%LwB|Hscvi_R?p%lNtO0xUvHpPy4)$@gC-?A|>2`|rx_wC$Mi@_+je
atGl>0RZGqr9Fr^@9kWrErqLs2ulXnYhy_#t


From 9ec04bef5b95106fb74a1993dbc2e3b55a42a0e7 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Wed, 29 Oct 2025 13:36:35 -0700
Subject: [PATCH 070/104] more stable eval dataset

---
 .../data/eval_datasets/eval_dataset.json          | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/src/vuln_analysis/data/eval_datasets/eval_dataset.json b/src/vuln_analysis/data/eval_datasets/eval_dataset.json
index fe73b8cd6..7b834cd96 100644
--- a/src/vuln_analysis/data/eval_datasets/eval_dataset.json
+++ b/src/vuln_analysis/data/eval_datasets/eval_dataset.json
@@ -46,9 +46,9 @@
                     "label": "code_not_reachable"
                 },
                 {
-                    "vuln_id": "CVE-2023-36632",
-                    "status": "AFFECTED",
-                    "label": "vulnerable"
+                    "vuln_id": "CVE-2023-5388",
+                    "status": "NOT AFFECTED",
+                    "label": "false_positive"
                 }
             ]
         },
@@ -92,14 +92,9 @@
             },
             "ground_truth": [
                 {
-                    "vuln_id": "GHSA-3f63-hfp8-52jq",
+                    "vuln_id": "CVE-2023-5388",
                     "status": "NOT AFFECTED",
-                    "label": "code_not_reachable"
-                },
-                {
-                    "vuln_id": "CVE-2023-36632",
-                    "status": "AFFECTED",
-                    "label": "vulnerable"
+                    "label": "false_positive"
                 }
             ]
         }

From 7e268f7762fbd235f8cf7dfa0b21b96833a10767 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Wed, 29 Oct 2025 13:39:02 -0700
Subject: [PATCH 071/104] Update README.md

Co-authored-by: hsin-c <109615347+hsin-c@users.noreply.github.com>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e446c133e..fb2e54ac0 100644
--- a/README.md
+++ b/README.md
@@ -823,7 +823,7 @@ nat eval --config_file=configs/config-eval.yml --reps=3
 ```
 
 #### Visualizing Score Distribution Across Multiple Runs
-NeMo Agent Toolkit enables flexibility in defining evaluation output format, as well as flexibility in consuming outputs (eg. uploading to observability platforms like Phoenix, or running a custom visualization script). In this example, we use a custom script in `eval/visualizations/box_and_whisker_plot.py` to plot the distribution of accuracy outputs after running multiple repetitions. The script reads from the evaluation results written at your configured output directory. It can be run like this:
+NeMo Agent Toolkit enables flexibility in defining the evaluation output format, as well as flexibility in consuming outputs (e.g., uploading to observability platforms like Phoenix, or running a custom visualization script). In this example, we use a custom script in `eval/visualizations/box_and_whisker_plot.py` to plot the distribution of accuracy outputs after running multiple repetitions. The script reads from the evaluation results written at your configured output directory. It can be run like this:
 
 ```
 python3 src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py \

From db495a45a8706c1cb3d2d7846a18313e88e21ebd Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Wed, 29 Oct 2025 17:12:39 -0700
Subject: [PATCH 072/104] remove redundant output field

---
 src/vuln_analysis/eval/evaluators/accuracy.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/vuln_analysis/eval/evaluators/accuracy.py b/src/vuln_analysis/eval/evaluators/accuracy.py
index 348f20eac..b54d902d1 100644
--- a/src/vuln_analysis/eval/evaluators/accuracy.py
+++ b/src/vuln_analysis/eval/evaluators/accuracy.py
@@ -220,8 +220,6 @@ async def wrapped_process_rep(rep_id: str, items: list[EvalInputItem]) -> tuple[
             desc = series.describe()  # count, mean, std, min, 25%, 50%, 75%, max
             # Round values
             final_output = {k: (int(v) if k == "count" else round(float(v), 2)) for k, v in desc.items()}
-            # Add the raw scores list
-            final_output["scores_per_rep"] = scores_sequential
         else:
             final_output = {
                 "count": 0,
@@ -232,7 +230,6 @@ async def wrapped_process_rep(rep_id: str, items: list[EvalInputItem]) -> tuple[
                 "50%": 0.0,
                 "75%": 0.0,
                 "max": 0.0,
-                "scores_per_rep": [],
             }
 
         eval_output_items = [

From 9c04647772ddf48f2d5197ad8b67f77c58ae66fe Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Fri, 31 Oct 2025 05:28:24 -0700
Subject: [PATCH 073/104] readme updates

---
 README.md | 136 ++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 91 insertions(+), 45 deletions(-)

diff --git a/README.md b/README.md
index fb2e54ac0..3d40bc8d3 100644
--- a/README.md
+++ b/README.md
@@ -763,35 +763,25 @@ The workflow can then be updated to use the new function:
 Additional output options will be added in the future.
 
 ## Evaluation
-In this example evaluation workflow, we evaluate the pipeline's ability to identify vulnerability **status** and **label**.
+Evaluation is an essential part of developing agentic workflows. In this section, we use NeMo Agent Toolkit's Evaluation feature to implemenent an example evaluation pipeline that demonstrates best practices for performing evaluation on the vulnerability analysis workflow. There are two custom evaluators included in this example, which provide an extensible template for developers to add their own custom evaluators that target their desired metrics.
 
-Each CVE has one ground truth status and one corresponding label.
-Valid statuses include: `AFFECTED` or `NOT AFFECTED`.
-Valid labels include: `vulnerable` if the CVE status is `AFFECTED`, or one of 10 VEX statuses if the status is `NOT AFFECTED` (see **Justification Assignment** under the [Key Components section](#key-components)).
-
-To run evaluaton, run `nat eval` using the `config-eval.yml` configuration.
+To run evaluation, run `nat eval` using the `config-eval.yml` configuration.
 
 ```
 nat eval --config_file=configs/config-eval.yml
 ```
 
-This will write the files `status_accuracy_output.json` and `label_accuracy_output.json` to the configured output directory (`.tmp/evaluators` in this example), containing accuracy scores for status and label respectively. Scores range from 0 to 1, with 0 indicating no CVEs were categorized correctly, and 1 indicating all CVEs were categorized correctly.
+This will run the two custom evaluators, which measure accuracy and consistency. Evaluation results can be viewed in `.tmp/evaluators`.
 
-Under the hood, the source code of the evaluator being run can be found at `eval/evaluators/accuracy.py`; this custom evaluator is registered in the NeMo Agent Toolkit as an evaluator of type `accuracy`.  In `config-eval.yml`, under the `evaluators` section, we have defined two evaluations to be run: `status_accuracy` and `label_accuracy`. However, the underlying code for these evaluations is the same. In other words, they use the same evaluator, which accepts either `status` or `label` as a configurable field input.
-
-In certain instances, a provided ground-truth dataset may contain duplicate CVEs, with conflicting answers that originate from different sources. For example, a CVE may be listed twice in the dataset as both "AFFECTED" and "NOT AFFECTED" with different justifications. To handle such cases, we provide three policies for handling duplicates:
-- `drop_all`: Any CVE with conflicting ground truth will be omitted and not contribute towards evaluation results.
-- `keep_first`: Use the first encountered instance of a CVE as the truth, disregard all other instances.
-- `keep_all`: Any answer listed in the ground truth for a CVE is considered valid (eg. both "vulnerable" and "not vulnerable" would be scored as correct if the ground truth listed both).
-You can configure the policy in the `duplicates_policy` field.
+Visit the [Example Custom Evaluators](#example-custom-evaluators) section to learn more details about these evaluators.
 
-You can write similar evaluators to calculate any desired custom metrics; read more at the NeMo Agent Toolkit documentation for [custom evaluators](https://docs.nvidia.com/nemo/agent-toolkit/1.2/reference/evaluate.html#adding-custom-evaluators).
-
-**Note: Depending on the size of your test set, and how many evaluation [repetitions](#running-multiple-evaluations) are being run, you may have to lower  maximum concurrency to avoid `429: Too Many Requests` errors. See [Troubleshooting](#429-errors---too-many-requests) for details.**
+To extend this evaluation example, developers should also write their own custom evaluators to calculate desired metrics. Read more at the NeMo Agent Toolkit documentation for [custom evaluators](https://docs.nvidia.com/nemo/agent-toolkit/1.2/reference/evaluate.html#adding-custom-evaluators). Additionally, developers should provide their own test sets, which contain the ground truth data that the pipeline's output is compared against. See [Inputting Evaluation Test Sets](#inputting-evaluation-test-sets) for details.
 
 
 ### Inputting Evaluation Test Sets
-Ground truth datasets should be put in the `data/eval_datasets/` directory. Each JSON file represents an experiment, which can be run across multiple containers. The structure is as follows:
+Evaluation test sets containing ground truths should be put in the `data/eval_datasets/` directory. This file path is configurable via the `eval.general.dataset.filepath` field. 
+
+Each test set is a JSON file which represents an evaluation experiment. Experiments can be run across multiple containers. An example test set `eval_dataset.json` is provided, which includes two containers (morpheus_23.11 and morpheus_24.03) which have a few CVEs each. The structure of the test set is as follows:
 
 ```
 dataset_id
@@ -806,60 +796,116 @@ containers:
      ...
 ```
 
-In `config-eval.yml`, configure the `file_path` field to specify the dataset to use. You can also configure the evaluation output directory using the `output_dir` variable.
+Developers can input their own test sets by following this file structure.
 
-An example evaluation experiment has been set up to run `eval_dataset` which runs test sets of CVEs on the morpheus_23.11-runtime and morpheus_24.03-runtime images.
+A custom parsing script (`eval/parse_eval_input.py`)is used to transform test sets into a format ingestible by the workflow. The path to this script is configured in `eval.general.dataset.function`. Advanced developers can choose to modify the test set input structure by creating their own parsing script, and then updating the configuration field with their own script.
+
+You can additionally configure the output directory of evaluation results via the `eval.general.output_dir` field.
+
+**Note: Depending on the size of a given test set, and how many evaluation [repetitions](#running-multiple-evaluations) are run, developers may have to set a maximum concurrency limit in order to avoid `429: Too Many Requests` errors. See [Troubleshooting](#429-errors---too-many-requests) for details.**
 
-```yml
-file_path: data/eval_datasets/eval_dataset.json
-```
 
-### Running Multiple Evaluations
+### Running Multiple Evaluation Repetitions
 
-NeMo Agent Toolkit provides a `--reps` flag, which can be used to conveniently run an evaluation item multiple times. Note that **caching must be disabled first** (see [Disable Caching](#disable-caching) below) in order for evaluation repetitions to yield differing results.
+The NeMo Agent Toolkit provides a `--reps` flag, which can be used to conveniently run an evaluation experiment multiple times. This is useful for capturing metrics such as workflow consistency. Note that **caching must be disabled first** (see [Disable Caching](#disable-caching) below) in order to force evaluation repetitions to re-run the workflow (as opposed to relying on cached results).
+
+Below is an example of how to run an evaluation experiment with 3 repetitions:
 
 ```
 nat eval --config_file=configs/config-eval.yml --reps=3
 ```
 
-#### Visualizing Score Distribution Across Multiple Runs
-NeMo Agent Toolkit enables flexibility in defining the evaluation output format, as well as flexibility in consuming outputs (e.g., uploading to observability platforms like Phoenix, or running a custom visualization script). In this example, we use a custom script in `eval/visualizations/box_and_whisker_plot.py` to plot the distribution of accuracy outputs after running multiple repetitions. The script reads from the evaluation results written at your configured output directory. It can be run like this:
+
+#### Disable Caching
+LLM and embedder caching must be disabled for multiple evaluation runs. Two options for disabling caching are provided below.
+
+##### Option 1: Disable Caching Globally
+Globally disable all LLM and embedding caches by setting the `NVIDIA_API_BASE` variable to directly access the hosted model endpoint instead of going through NGINX.
 
 ```
-python3 src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py \
-  .tmp/evaluators/your-experiment-output-1.json \
-  .tmp/evaluators/your-experiment-output-2.json \
-  --title "Status Accuracy: Experiment1 vs Experiment2" \
-  --save src/vuln_analysis/eval/visualizations/my-plot.png \
-  --labels my-experiment1 my-experiment2
+export NVIDIA_API_BASE=https://integrate.api.nvidia.com/v1
 ```
 
-You can specify any number of output files created by the `accuracy` evaluator. You can also specify chart title, save location, and X axis labels.
+**WARNING: This approach will cause the embeddings of the input container's source code and documentation input to be regenerated every run, which is both time-consuming and compute-expensive.**
 
+Re-enable caching by setting the `NVIDIA_API_BASE` variable back to NGINX.
+```
+export NVIDIA_API_BASE=http://nginx-cache/nim_llm/v1
+```
 
-#### Disable Caching
-LLM and embedder caching must be disabled for multiple evaluation runs, to prevent getting the same results every time.
 
-##### Option 1 - Global Override
-Globally override all LLM and embedder caching by setting the `NVIDIA_API_BASE` variable to directly hit the model endpoint instead of going through nginx caching.
+##### Option 2: Disable Caching on Specific Sections
+This option enables developers to disable caching for specific parts of the workflow. It is useful for disabling caching on all model calls except the embedding model, in order to avoid the time-consuming embedding process. This approach is also useful for in-depth evaluation experiments; for example, a developer may want to disable caching for just the `justify_llm` in order to test consistency of the justification part of the workflow without re-running everything before that point. 
+
+In `config-eval.yml`, assign the `base_url` field of the target section to `"https://integrate.api.nvidia.com/v1"`. We recommend creating multiple config files to keep track of different evaluation experiments. To re-enable caching, simply assign the `base_url` back to `${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}`.
+
+For quick testing, developers can also change configuration in the CLI using the `--override` flag, which uses dot notation. This change will not persist across future runs. Below is an example that disables caching on the checklist_llm.
 
 ```
-NVIDIA_API_BASE=https://integrate.api.nvidia.com/v1 nat eval --config_file=configs/config-eval.yml --reps=3
+nat eval --config_file=configs/config-eval.yml --reps=3 --override llms.checklist_llm.base_url https://integrate.api.nvidia.com/v1
 ```
 
-##### Option 2 - Disable Caching on Specific Calls
-Users may want to disable caching for only specific parts of the pipeline. For example, more in-depth experiments may require disabling caching on specific LLMs. Or, you may want to disable all LLM call caching but keep embedder caching so the embedder does not have to regenerate embeddings for the static image source every evaluation run.
+### Example Custom Evaluators
+In this section, the implementation of the two custom evaluators for accuracy and consistency are discussed in detail.
+
+#### Simple Accuracy Evaluator
+For the vulnerability analysis workflow, the most simple evaluation we can begin with is evaluating the accuracy of the `status` and `label` fields which the workflow outputs per CVE.
+
+Each CVE has a ground truth `status`, as well as a `label` which corresponds to the status.
+Valid statuses include: `AFFECTED` or `NOT AFFECTED`.
+Valid labels include: `vulnerable` if the CVE status is `AFFECTED`, or one of 10 VEX statuses if the status is `NOT AFFECTED` (see **Justification Assignment** under the [Key Components section](#key-components)).
+
+For each CVE, if the workflow output matches the ground truth, a score of 1 is assigned. Otherwise, a score of 0 is assigned. The evaluator then takes the mean of all CVEs' scores to produce an average accuracy.
+
+
+##### Configuration Options
+The accuracy evaluator is implemented in `eval/evaluators/accuracy.py` and registered in NeMo Agent Toolkit with the unique identifer (`_type`) of "`accuracy`".
+
+The evaluator takes in a target field (`status` or `label`), which can be configured in `config-eval.yml` under `evaluators.[evaluator name].field`. This design follows a common NeMo Agent Toolkit design pattern to promote code reussability. In `config-eval.yml`, a `status_accuracy_evaluator` and a `label_accuracy_evaluator` are set up as two separate evaluators, but under the hood, they share the same `_type` and therefore the same source code. 
+
+The evaluator also takes in a `duplicates_policy` field. This is to account for instances where, if an input set is an aggregation of ground truths from different sources, the set may contain duplicate CVEs with conflicting answers. For example, a CVE may be listed twice in the dataset as both `AFFECTED` and `NOT AFFECTED`, with different label justifications. We provide three policies for handling duplicates:
+- `drop_all`: Any CVE with conflicting ground truth will be omitted and not contribute towards evaluation results.
+- `keep_first`: Use the first encountered instance of a CVE as the truth, disregard all other instances.
+- `keep_all`: Any answer listed in the ground truth for a CVE is considered valid (eg. if the ground truth has two listings of a CVE, with one being `AFFECTED` and the other being `NOT AFFECTED`, either answer given by the pipeline will be considered correct).
 
-You can accomplish specific caching overrides in `config-eval.yml` by assigning the `base_url` variable of the target component with `"https://integrate.api.nvidia.com/v1"`. Note that you can create multiple config files to keep track of different evaluation experiments.
 
+##### Output Format
+The final scores are written to the files `status_accuracy_output.json` and `label_accuracy_output.json`. The output will include an `average_score` overview field, and a an `eval_output_items` field which gives a more detailed breakdown.
 
-For quick testing, you can also change configuration at runtime using the `--override` flag, which uses dot notation. Below is an example that disables caching on the checklist_llm.
+###### average_score Field
+The `average_score` field contains a dict with metrics corresponding to `pandas.Dataframe.describe` output. The goal of this field is to capture the consistency of runs, when a user uses the NeMo Agent Toolkit `--reps` option to perform multiple run of the experiment. If only one run is performed, `count` and `mean` would be the only meaningful metrics; for example, if the run yielded accuracy of 0.8, the `average_score` field would look something like this:
+```json
+"average_score": {
+    "count": 1,
+    "mean": 0.8,
+    "std": NaN,
+    "min": 0.8,
+    "25%": 0.8,
+    "50%": 0.8,
+    "75%": 0.8,
+    "max": 0.8
+  }
+```
+Meanwhile, if there were multiple runs, the `average_score` field would capture the distribution of runs. A script is provided in `eval/visualizations/box_and_whisker_plot.py` to visualize this distribution as a box and whisker plot. It is especially helpful for comparing the plots of multiple experiments side-by-side, or comparing `status` accuracy against `label` accuracy. It can be run like so:
 
 ```
-nat eval --config_file=configs/config-eval.yml --reps=3 --override llms.checklist_llm.base_url https://integrate.api.nvidia.com/v1
+python3 src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py \
+  .tmp/evaluators/accuracy-output-1.json \
+  .tmp/evaluators/accuracy-output-2.json \
+  --title "Accuracy: Experiment 1 vs. Experiment 2" \
+  --save src/vuln_analysis/eval/visualizations/my-plot.png \
+  --labels my-experiment1 my-experiment2
 ```
+Users can list any number of output files, as well as configure chart title, save location, and X-axis labels.
+
+###### eval_output_items Field
+The `eval_output_items` field gives a more detailed output of the experiment. It lists the runs, giving each run a unique `id`, a `score` (the accuracy score for that run), and a `reasoning` field.
+
+This last field breaks the run down by container, and lists the specific CVEs that were run in each container (`question`), the ground truth answers `[target field]_answer`, and the answers generated by the workflow (`generated_[target field_answer`).
+
+
 
-#### Evaluation Across Multiple Runs
+#### Consistency Evaluator
 In addition to gathering metrics from within an evaluation run, metrics can also be calculated from across runs. We provide an example of such a metric, which measures consistency of results across repeated runs. It can be configured to measure consistency of the `status` or `label` fields.
 
 Consistency is defined as: ∑p² per CVE, where p is the probability of each unique status or label. For example, if the labels for a CVE across 3 runs are `{vulnerable, code_not_reachable, code_not_reachable}`, then the probabilities become `{1/3, 2/3}`, so the consistency score is `(1/3)² + (2/3)² = 0.56`. A score of 1.0 means perfect consistency (all runs yielded exact same values). The final output is a per-CVE consistency score and an average consistency score.

From 695f817c8fc56869cd83f7e875714702ad6a2ef7 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Fri, 31 Oct 2025 06:26:53 -0700
Subject: [PATCH 074/104] fix profiler dataset path

---
 src/vuln_analysis/configs/config-tracing.yml | 2 +-
 src/vuln_analysis/configs/config.yml         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/vuln_analysis/configs/config-tracing.yml b/src/vuln_analysis/configs/config-tracing.yml
index 7fe74acbd..ade78f678 100644
--- a/src/vuln_analysis/configs/config-tracing.yml
+++ b/src/vuln_analysis/configs/config-tracing.yml
@@ -188,7 +188,7 @@ eval:
     output_dir: ./.tmp/profiler
     dataset:
         _type: json
-        file_path: data/eval_datasets/profiler_dataset.json
+        file_path: data/profiler_datasets/profiler_dataset.json
 
     profiler:
         token_uniqueness_forecast: true
diff --git a/src/vuln_analysis/configs/config.yml b/src/vuln_analysis/configs/config.yml
index a9967c4d6..692bdbd7e 100644
--- a/src/vuln_analysis/configs/config.yml
+++ b/src/vuln_analysis/configs/config.yml
@@ -178,7 +178,7 @@ eval:
     output_dir: ./.tmp/profiler
     dataset:
         _type: json
-        file_path: data/eval_datasets/profiler_dataset.json
+        file_path: data/profiler_datasets/profiler_dataset.json
 
     profiler:
         token_uniqueness_forecast: true

From e41f63c9fa0ddb754827d207fc0f5e0a50563540 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Fri, 31 Oct 2025 07:05:32 -0700
Subject: [PATCH 075/104] remove max concurrency config from summarize,
 justify, cve_checklist

---
 src/vuln_analysis/functions/cve_checklist.py | 19 +++++--------------
 src/vuln_analysis/functions/cve_justify.py   | 10 +---------
 src/vuln_analysis/functions/cve_summarize.py | 17 +++--------------
 3 files changed, 9 insertions(+), 37 deletions(-)

diff --git a/src/vuln_analysis/functions/cve_checklist.py b/src/vuln_analysis/functions/cve_checklist.py
index 9d2cb4146..fba78ce62 100644
--- a/src/vuln_analysis/functions/cve_checklist.py
+++ b/src/vuln_analysis/functions/cve_checklist.py
@@ -32,8 +32,6 @@ class CVEChecklistToolConfig(FunctionBaseConfig, name="cve_checklist"):
     Defines a function that generates tailored, context-sensitive task checklist for impact analysis.
     """
     llm_name: str = Field(description="The LLM model to use")
-    max_concurrency: int | None = Field(
-        ge=1, default=None, description="The maximum number of concurrent checklist generation invocations.")
     prompt: str | None = Field(
         default=None,
         description=
@@ -49,20 +47,13 @@ async def cve_checklist(config: CVEChecklistToolConfig, builder: Builder):
     from vuln_analysis.utils.checklist_prompt_generator import generate_checklist
 
     llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN)
-    semaphore = asyncio.Semaphore(config.max_concurrency) if config.max_concurrency else None
 
     async def generate_checklist_for_cve(cve_intel):
-        if semaphore:
-            async with semaphore:
-                checklist = await generate_checklist(prompt=config.prompt,
-                                                     llm=llm,
-                                                     input_dict=cve_intel,
-                                                     enable_llm_list_parsing=False)
-        else:
-            checklist = await generate_checklist(prompt=config.prompt,
-                                                 llm=llm,
-                                                 input_dict=cve_intel,
-                                                 enable_llm_list_parsing=False)
+
+        checklist = await generate_checklist(prompt=config.prompt,
+                                             llm=llm,
+                                             input_dict=cve_intel,
+                                             enable_llm_list_parsing=False)
 
         checklist = await _parse_list([checklist])
 
diff --git a/src/vuln_analysis/functions/cve_justify.py b/src/vuln_analysis/functions/cve_justify.py
index 3d3de2cf2..ed08267f3 100644
--- a/src/vuln_analysis/functions/cve_justify.py
+++ b/src/vuln_analysis/functions/cve_justify.py
@@ -31,9 +31,6 @@ class CVEJustifyToolConfig(FunctionBaseConfig, name="cve_justify"):
     Defines a function that assigns justification label and reason to each CVE based on summary.
     """
     llm_name: str = Field(description="The LLM model to use")
-    max_concurrency: int | None = Field(ge=1,
-                                        default=None,
-                                        description="The maximum number of concurrent justification invocations.")
 
 
 @register_function(config_type=CVEJustifyToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN])
@@ -50,14 +47,9 @@ async def cve_justify(config: CVEJustifyToolConfig, builder: Builder):
 
     prompt = PromptTemplate(input_variables=["summary"], template=jp.JUSTIFICATION_PROMPT)
     chain = prompt | llm
-    semaphore = asyncio.Semaphore(config.max_concurrency) if config.max_concurrency else None
 
     async def justify_cve(summary):
-        if semaphore:
-            async with semaphore:
-                justification_text = await chain.ainvoke({"summary": summary})
-        else:
-            justification_text = await chain.ainvoke({"summary": summary})
+        justification_text = await chain.ainvoke({"summary": summary})
         return justification_text.content
 
     async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState:
diff --git a/src/vuln_analysis/functions/cve_summarize.py b/src/vuln_analysis/functions/cve_summarize.py
index 852634724..2aea14103 100644
--- a/src/vuln_analysis/functions/cve_summarize.py
+++ b/src/vuln_analysis/functions/cve_summarize.py
@@ -33,9 +33,6 @@ class CVESummarizeToolConfig(FunctionBaseConfig, name="cve_summarize"):
     Defines a function that generates concise, human-readable summarization paragraph from agent results.
     """
     llm_name: str = Field(description="The LLM model to use")
-    max_concurrency: int | None = Field(ge=1,
-                                        default=None,
-                                        description="The maximum number of concurrent summarization invocations.")
 
 
 @register_function(config_type=CVESummarizeToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN])
@@ -49,19 +46,11 @@ async def cve_summarize(config: CVESummarizeToolConfig, builder: Builder):
     llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN)
     prompt = PromptTemplate(input_variables=["response"], template=SUMMARY_PROMPT)
     chain = prompt | llm
-    semaphore = asyncio.Semaphore(config.max_concurrency) if config.max_concurrency else None
 
     async def summarize_cve(results):
-        if semaphore:
-            async with semaphore:
-                response = '\n'.join([
-                    get_checklist_item_string(idx + 1, checklist_item) for idx, checklist_item in enumerate(results[1])
-                ])
-                final_summary = await chain.ainvoke({"response": response})
-        else:
-            response = '\n'.join(
-                [get_checklist_item_string(idx + 1, checklist_item) for idx, checklist_item in enumerate(results[1])])
-            final_summary = await chain.ainvoke({"response": response})
+        response = '\n'.join(
+            [get_checklist_item_string(idx + 1, checklist_item) for idx, checklist_item in enumerate(results[1])])
+        final_summary = await chain.ainvoke({"response": response})
 
         return final_summary.content
 

From a90785a5054b737c384f0228751fd130da946770 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Fri, 31 Oct 2025 07:13:11 -0700
Subject: [PATCH 076/104] update config file and readme to match removed
 max_concurrency fields

---
 README.md                                 | 11 +++--------
 src/vuln_analysis/configs/config-eval.yml |  2 +-
 2 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 6e0f47c7a..2ac5f67df 100644
--- a/README.md
+++ b/README.md
@@ -1094,19 +1094,14 @@ Error requesting [1/10]: (Retry 0.1 sec) https://services.nvd.nist.gov/rest/json
 
 ##### 429 Errors - Too Many Requests
 
-429 errors can occur when your requests exceed the rate limit for the model. These errors most commonly occur during the `cve_agent_executor` part of the workflow. Try setting the `cve_agent_executor.max_concurrency` in the [config.yml](src/vuln_analysis/configs/config.yml) to a low value such as 5 to reduce the rate of requests.
+429 errors can occur when your requests exceed the rate limit for the hosted model endpoint.
 ```
 Exception: [429] Too Many Requests
 ```
+These errors most commonly occur during the `cve_agent_executor` part of the workflow. Try setting the `cve_agent_executor.max_concurrency` in the [config.yml](src/vuln_analysis/configs/config.yml) to a low value such as 5 to reduce the rate of requests.
 
-When running a large input dataset, or running evaluation with multiple repetitions, other parts of the workflow can also run into 429 errors. Aside from `cve_agent_executor.max_concurrency`, the following pipeline pieces also have `max_concurrency` limiters that you can set in the config file:
+Additionally, when running evaluation, `nat eval` parallelizes evaluation input items and evaluation repetitions. You can set the `eval.general.max_concurrency` to a low value to reduce parallelization and therefore reduce the rate of requests. 
 
-  - `eval.general.max_concurrency`: Controls how many complete evaluation runs can execute simultaneously. Each evaluation run executes the entire workflow from start to finish on a single source code input (so if your test set contains 10 source code repositories, you will have 10 evaluation runs, and this setting determines how many of those can run in parallel).
-  - `functions.cve_checklist.max_concurrency`: Controls maximum concurrency of the checklist creation part of the workflow, within a single evaluation run.
-  - `functions.cve_summarize.max_concurrency`: Controls maximum concurrency of the summarization part of the workflow, within a single evaluation run.
-  - `functions.cve_justify.max_concurrency`: Controls maximum concurrency of the justification part of the workflow, within a single evaluation run.
-
-To identify which workflow stage is triggering the 429 error, search the error output for these keywords: `agent_executor`, `checklist`, `summarize`, or `justify`. Once you find which stage is failing, adjust the corresponding `max_concurrency` setting for that specific part of the workflow.
 
 #### Authentication errors
 
diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml
index 9b830e551..4a7333bed 100644
--- a/src/vuln_analysis/configs/config-eval.yml
+++ b/src/vuln_analysis/configs/config-eval.yml
@@ -60,7 +60,7 @@ functions:
       - Container Image Developer Guide QA System
       # - Lexical Search Container Image Code QA System  # Uncomment to enable lexical search
       - Internet Search
-    max_concurrency: 5
+    max_concurrency: null
     max_iterations: 10
     prompt_examples: false
     replace_exceptions: true

From 54e1c33a87a2822b38361990cb9965304c862e8f Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Fri, 31 Oct 2025 08:26:48 -0700
Subject: [PATCH 077/104] readme updates

---
 README.md                                 | 40 ++++++++++++++++++-----
 src/vuln_analysis/configs/config-eval.yml |  3 +-
 2 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 2ac5f67df..f28456e9a 100644
--- a/README.md
+++ b/README.md
@@ -67,6 +67,10 @@ limitations under the License.
 - [Evaluation](#evaluation)
   - [Inputting Evaluation Test Sets](#inputting-evaluation-test-sets)
   - [Running Multiple Evaluations](#running-multiple-evaluations)
+    - [Disable Caching](#disable-caching)
+  - [Example Custom Evaluators](#example-custom-evaluators)
+    - [Simple Accuracy Evaluator](#simple-accuracy-evaluator)
+    - [Consistency Evaluator](#consistency-evaluator)
 - [Troubleshooting](#troubleshooting)
   - [Git LFS issues](#git-lfs-issues)
   - [Container build issues](#container-build-issues)
@@ -885,7 +889,7 @@ A custom parsing script (`eval/parse_eval_input.py`)is used to transform test se
 
 You can additionally configure the output directory of evaluation results via the `eval.general.output_dir` field.
 
-**Note: Depending on the size of a given test set, and how many evaluation [repetitions](#running-multiple-evaluations) are run, developers may have to set a maximum concurrency limit in order to avoid `429: Too Many Requests` errors. See [Troubleshooting](#429-errors---too-many-requests) for details.**
+**Note: For large input test sets, developers may encounter `429: Too Many Requests` errors if using hosted model endpoints. See [Troubleshooting](#429-errors---too-many-requests) for how to mitigate this.**
 
 
 ### Running Multiple Evaluation Repetitions
@@ -898,6 +902,8 @@ Below is an example of how to run an evaluation experiment with 3 repetitions:
 nat eval --config_file=configs/config-eval.yml --reps=3
 ```
 
+**Note: Running multiple repetitions may take some extra time. Additionally, developers may encounter `429: Too Many Requests` errors if using hosted model endpoints, due to the concurrency introduced when running multiple repetition. See [Troubleshooting](#429-errors---too-many-requests) for how to mitigate this.**
+
 
 #### Disable Caching
 LLM and embedder caching must be disabled for multiple evaluation runs. Two options for disabling caching are provided below.
@@ -989,11 +995,27 @@ This last field breaks the run down by container, and lists the specific CVEs th
 
 
 #### Consistency Evaluator
-In addition to gathering metrics from within an evaluation run, metrics can also be calculated from across runs. We provide an example of such a metric, which measures consistency of results across repeated runs. It can be configured to measure consistency of the `status` or `label` fields.
+This evaluator measures the consistency of results across repeated runs. It can be configured to measure consistency of the `status` or `label` fields.
+
+Consistency is defined as: `∑p²` per CVE, where *p* is the probability of each unique status or label. For example, if the status for a CVE across 3 runs are `(AFFECTED, NOT_AFFECTED, NOT_AFFECTED)`, then the probabilities become `p(AFFECTEd, NOT_AFFECTED) = {1/3, 2/3}`. Therefore, the consistency score is `(1/3)² + (2/3)² = 0.56`. A score of 1.0 means perfect consistency (all runs yielded the exact same values). The evaluator outputs a consistency score for each CVE.
 
-Consistency is defined as: ∑p² per CVE, where p is the probability of each unique status or label. For example, if the labels for a CVE across 3 runs are `{vulnerable, code_not_reachable, code_not_reachable}`, then the probabilities become `{1/3, 2/3}`, so the consistency score is `(1/3)² + (2/3)² = 0.56`. A score of 1.0 means perfect consistency (all runs yielded exact same values). The final output is a per-CVE consistency score and an average consistency score.
+**Note: This evaluator should only be used if evaluation is being run with multiple repetitions. An error will occur if this criteria is not met.** To enable this evaluator, uncomment the last section in `config-eval.yml`.
 
-Uncomment the last section in `config-eval.yml` to enable this evaluator. You can view the source code at `eval/evaluators/consistency.py`.
+
+##### Configuration Options
+The accuracy evaluator is implemented in `eval/evaluators/consistency.py` and registered in NeMo Agent Toolkit with the unique identifer (`_type`) of "`consistency`".
+
+The evaluator takes in a configurable target field (`status` or `label`), following a similar design pattern as the [Simple Accuracy Evaluator](#configuration-options)
+
+##### Output Format
+The final scores are written to the files `status_consistency_output.json` and `label_consistency_output.json`.
+
+The output will include an `average_score` field which gives the average consistency score across all containers, and an `eval_output_items` field which lists the input containers.
+
+For each container in the `eval_output_items` field, there is an unique `id` and a consistency `score`, which is the mean of the consistency scores of each CVE within that container. There is also a `reasoning` field that displays the following details:
+  - `status_consistencies`: consistency score for a specific CVE
+  - `total_cves`: the total number of CVEs in the container
+  - `num_reps`: number of evaluation repetitions that were run
 
 ## Troubleshooting
 
@@ -1094,13 +1116,13 @@ Error requesting [1/10]: (Retry 0.1 sec) https://services.nvd.nist.gov/rest/json
 
 ##### 429 Errors - Too Many Requests
 
-429 errors can occur when your requests exceed the rate limit for the hosted model endpoint.
-```
-Exception: [429] Too Many Requests
-```
+`Exception: [429] Too Many Requests` can occur when using hosted model endpoints, if your requests exceed the rate limit.
+
 These errors most commonly occur during the `cve_agent_executor` part of the workflow. Try setting the `cve_agent_executor.max_concurrency` in the [config.yml](src/vuln_analysis/configs/config.yml) to a low value such as 5 to reduce the rate of requests.
 
-Additionally, when running evaluation, `nat eval` parallelizes evaluation input items and evaluation repetitions. You can set the `eval.general.max_concurrency` to a low value to reduce parallelization and therefore reduce the rate of requests. 
+With large input sets, you may additionally run into `429` errors from the concurrency of other parts of the workflow, such as `cve_checklist`, `cve_summarize`, and `cve_justify`. In this case, we do not have a tunable `max_concurrency` limiter built in; instead, developers should consider switching to [self-hosting](#using-self-hosted-nims).
+
+Additionally, when running evaluation, `nat eval` parallelizes evaluation input items and evaluation repetitions. Therefore, evaluation pipelines are especially prone to triggering 429 errors. You can set the `eval.general.max_concurrency` to a low value to reduce parallelization and therefore reduce the rate of requests. However, note that if your evaluation test set is large, or you are running many repetitions, the rate-limited hosted endpoint will be very slow; in this case, we recommend switching to [self-hosting](#using-self-hosted-nims). 
 
 
 #### Authentication errors
diff --git a/src/vuln_analysis/configs/config-eval.yml b/src/vuln_analysis/configs/config-eval.yml
index 4a7333bed..f43edae2c 100644
--- a/src/vuln_analysis/configs/config-eval.yml
+++ b/src/vuln_analysis/configs/config-eval.yml
@@ -189,7 +189,8 @@ eval:
       field: label
       duplicates_policy: drop_all
 
-    # Uncomment to enable response consistency evaluators
+    # Uncomment to enable consistency evaluator
+    # NOTE: Only use this evaluator if evaluation is being run with multiple repetitions
     # status_consistency:
     #   _type: consistency
     #   field: status

From 8a17dbb5d423576dc7b53b65ecb92acf35969e7b Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 11:13:00 -0800
Subject: [PATCH 078/104] remove max concurrency from readme

---
 README.md | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index f28456e9a..10ff3347a 100644
--- a/README.md
+++ b/README.md
@@ -66,11 +66,19 @@ limitations under the License.
     - [Writing a custom TTC strategy](#writing-a-custom-ttc-strategy)
 - [Evaluation](#evaluation)
   - [Inputting Evaluation Test Sets](#inputting-evaluation-test-sets)
-  - [Running Multiple Evaluations](#running-multiple-evaluations)
+  - [Running Multiple Evaluation Repetitions](#running-multiple-evaluation-repetitions)
     - [Disable Caching](#disable-caching)
+      - [Option 1: Disable Caching Globally](#option-1-disable-caching-globally)
+      - [Option 2: Disable Caching on Specific Sections](#option-2-disable-caching-on-specific-sections)
   - [Example Custom Evaluators](#example-custom-evaluators)
     - [Simple Accuracy Evaluator](#simple-accuracy-evaluator)
+      - [Configuration Options](#configuration-options)
+      - [Output Format](#output-format)
+        - [average\_score Field](#average_score-field)
+        - [eval\_output\_items Field](#eval_output_items-field)
     - [Consistency Evaluator](#consistency-evaluator)
+      - [Configuration Options](#configuration-options-1)
+      - [Output Format](#output-format-1)
 - [Troubleshooting](#troubleshooting)
   - [Git LFS issues](#git-lfs-issues)
   - [Container build issues](#container-build-issues)
@@ -86,6 +94,7 @@ limitations under the License.
       - [429 Errors - Too Many Requests](#429-errors---too-many-requests)
     - [Authentication errors](#authentication-errors)
   - [Running out of credits](#running-out-of-credits)
+  - [Mac Workarounds](#mac-workarounds)
 - [Testing and validation](#testing-and-validation)
 - [Roadmap](#roadmap)
 - [Cite](#cite)
@@ -546,7 +555,7 @@ The configuration defines how the workflow operates, including functions, LLMs,
     - Preprocessing functions:
       - `cve_generate_vdbs`: Generates vector database from code repositories and documentation.
         - `agent_name`: Name of the agent executor (`cve_agent_executor`). Used to determine which tools are enabled in the agent to conditionally generate vector databases or indexes.
-        - `embedder_name`: Name of embedder (`nim-embedder`) configured in `embedders` section.
+        - `embedder_name`: Name of embedder (`nim-embedder`) configured in `embedders` section. 
         - `base_vdb_dir`: The directory used for storing vector database files. Default is `.cache/am_cache/vdb`.
         - `base_git_dir`: The directory for storing pulled git repositories used for code analysis. Default is `.cache/am_cache/git`.
         - `base_code_index_dir`: The directory used for storing code index files. Default is `./cache/am_cache/code_index`.
@@ -560,7 +569,6 @@ The configuration defines how the workflow operates, including functions, LLMs,
         - `skip`: If true, skips this check and runs all CVEs through the agent. Useful when the input CVEs to the workflow have been validated by a dedicated vulnerability scanner. Default is false.
     - Core LLM engine functions:
         - `cve_checklist`: Generates tailored, context-sensitive task checklist for impact analysis.
-          - `max_concurrency`: Controls the maximum number of concurrent requests to the LLM. Default is `null`, which doesn't limit concurrency.
         - `Container Image Code QA System`: Retriever tool used by `cve_agent_executor` to query source code vector database.
         - `Container Image Developer Guide QA System`: Retriever tool used by `cve_agent_executor` to query documentation vector database.
         - `Lexical Search Container Image Code QA System`: Lexical search tool used by `cve_agent_executor` to search source code. This tool is an alternative to `Container Image Code QA System` and can be useful for very large code bases that take a long time to embed as a vector database. Disabled by default, enable by uncommenting the tool in `cve_agent_executor`.
@@ -577,16 +585,14 @@ The configuration defines how the workflow operates, including functions, LLMs,
           - `verbose`: Set to true for verbose output. Default is `false`.
         - `cve_summarize`: Generates concise, human-readable summarization paragraph from agent results.
           - `llm_name`: Name of LLM (`summarize_llm`) configured in `llms` section.
-          - `max_concurrency`: Controls the maximum number of concurrent requests to the LLM. Default is `null`, which doesn't limit concurrency.
         - `cve_justify`: Assigns justification label and reason to each CVE based on summary.
-          - `llm_name`: Name of LLM (`justify_llm`) configured in `llms` section.
-          - `max_concurrency`: Controls the maximum number of concurrent requests to the LLM. Default is `null`, which doesn't limit concurrency.
+          - `llm_name`: Name of LLM (`justify_llm`) configured in `llms` section.concurrency.
     - Postprocessing/Output functions
       - `cve_file_output`: Outputs workflow results to a file.
         - `file_path`: Defines the path to the file where the output will be saved.
         - `markdown_dir`: Defines the path to the directory where the output will be saved in individual navigable markdown files per CVE-ID.
         - `overwrite`: Indicates whether the output file should be overwritten when the workflow starts if it already exists. Will throw an error if set to `False` and the file already exists. Note that the overwrite behavior only occurs on workflow initialization. For pipelines started in HTTP mode, each new request will append the existing file until the workflow is restarted.
-3. **LLMs** (`llms`): The `llms` section contains the LLMs used by the workflow. Functions can reference LLMs in this section to use. The supported LLM API types in NeMo Agent toolkit are `nim` and `openai` The models in this workflow use `nim`.
+1. **LLMs** (`llms`): The `llms` section contains the LLMs used by the workflow. Functions can reference LLMs in this section to use. The supported LLM API types in NeMo Agent toolkit are `nim` and `openai` The models in this workflow use `nim`.
     - Configured models in this workflow: `checklist_llm`, `code_vdb_retriever_llm`, `doc_vdb_retriever_llm`, `cve_agent_executor_llm`, `summarize_llm`, `justify_llm`
     - Each `nim` model is configured with the following attributes defined in the NeMo Agent toolkit's [NimModelConfig](https://docs.nvidia.com/nemo/agent-toolkit/latest/api/nat/llm/nim_llm/index.html#nat.llm.nim_llm.NIMModelConfig). Use [OpenAIModelConfig](https://docs.nvidia.com/nemo/agent-toolkit/latest/api/nat/llm/openai_llm/index.html#nat.llm.openai_llm.OpenAIModelConfig) for `openai` models.
       - `base_url`: Optional attribute to override `https://integrate.api.nvidia.com/v1`
@@ -599,7 +605,7 @@ The configuration defines how the workflow operates, including functions, LLMs,
       - `num_retries`: Number of times to retry a method call that fails with a retryable error.
       - `retry_on_status_codes`: List of HTTP status codes that should trigger a retry.
       - `retry_on_errors`: List of error messages that should trigger a retry. We recommend including both the error message and error code to accommodate libraries that don’t expose a HTTP status code attribute in raised exceptions.
-4. **Embedding models** (`embedders`): The `embedders` section contains the embedding models used by the workflow. Functions can reference embedding models in this section to use. The supported embedding model API types in NeMo Agent toolkit are `nim` and `openai`.
+2. **Embedding models** (`embedders`): The `embedders` section contains the embedding models used by the workflow. Functions can reference embedding models in this section to use. The supported embedding model API types in NeMo Agent toolkit are `nim` and `openai`.
     - The models uses `nim` model, `nvidia/nv-embedqa-e5-v5`.
     - Each `nim` embedding model is configured with the following attributes defined in the NeMo Agent Toolkit\'s [NimEmbedderModelConfig](https://docs.nvidia.com/nemo/agent-toolkit/latest/api/nat/embedder/nim_embedder/index.html). Use [OpenAIEmbedderModelConfig](https://docs.nvidia.com/nemo/agent-toolkit/latest/api/nat/embedder/openai_embedder/index.html#nat.embedder.openai_embedder.OpenAIEmbedderModelConfig) for `openai` embedding models.
       - `base_url`: Optional attribute to override `https://integrate.api.nvidia.com/v1`
@@ -611,14 +617,14 @@ The configuration defines how the workflow operates, including functions, LLMs,
         - `num_retries`: Number of times to retry a method call that fails with a retryable error.
         - `retry_on_status_codes`: List of HTTP status codes that should trigger a retry.
         - `retry_on_errors`: List of error messages that should trigger a retry. We recommend including both the error message and error code to accommodate libraries that don’t expose a HTTP status code attribute in raised exceptions.
-5. **Workflow** (`workflow`): The `workflow` section ties the previous sections together by defining the tools and LLM models to use in the workflow.
+3. **Workflow** (`workflow`): The `workflow` section ties the previous sections together by defining the tools and LLM models to use in the workflow.
     - `_type`: This is set to `cve_agent` indicating to NeMo Agent toolkit to use the function defined in [register.py](src/vuln_analysis/register.py) for the workflow.
     - `missing_source_action`: Action when source analysis is unavailable in the agent due to: missing source_info, VDB generation failures, or inaccessible repositories. Options are:
       - `error`: Fail pipeline with validation error
       - `skip_agent`: Collect intel and check dependencies, but skip agent
       - `continue_with_warning`: Run full pipeline with warning log (degraded analysis quality). Default is "continue_with_warning" for backwards compatibility.
     - The remaining configuration items correspond to attributes in [CVEWorkflowConfig](src/vuln_analysis/register.py) to specify the registered tools to use in the workflow.
-6. **Evaluations and Profiling** (`eval`): The `eval` section contains the evaluation settings for the workflow. Refer to [Evaluating NVIDIA Agent Intelligence Toolkit Workflows](https://docs.nvidia.com/nemo/agent-toolkit/latest/workflows/evaluate.html) for more information about NeMo Agent toolkit built-in evaluators as well as the plugin system to add custom evaluators. The CVE workflow uses the `eval` section to configure a profiler that uses the NeMo Agent toolkit evaluation system to collect usage statistics and store them to the local file system. You can find more information about NeMo Agent toolkit profiling and performance monitoring [here](https://docs.nvidia.com/nemo/agent-toolkit/latest/workflows/profiler.html).
+4. **Evaluations and Profiling** (`eval`): The `eval` section contains the evaluation settings for the workflow. Refer to [Evaluating NVIDIA Agent Intelligence Toolkit Workflows](https://docs.nvidia.com/nemo/agent-toolkit/latest/workflows/evaluate.html) for more information about NeMo Agent toolkit built-in evaluators as well as the plugin system to add custom evaluators. The CVE workflow uses the `eval` section to configure a profiler that uses the NeMo Agent toolkit evaluation system to collect usage statistics and store them to the local file system. You can find more information about NeMo Agent toolkit profiling and performance monitoring [here](https://docs.nvidia.com/nemo/agent-toolkit/latest/workflows/profiler.html).
     - `general.output_dir`: Defines the path to the directory where profiling results will be saved.
     - `general.dataset`: Defines file path and format of dataset used to run profiling.
     - `evaluators`: Custom evaluators defined in `eval/evaluators`. See [Evaluation](#evaluation) for more details on each custom evaluator and their configurations.

From fa20933636160cf67d48b8bcca4b90cca4803e5b Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 11:14:33 -0800
Subject: [PATCH 079/104] Update README.md

Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
---
 README.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 10ff3347a..27cb8ae00 100644
--- a/README.md
+++ b/README.md
@@ -1003,13 +1003,14 @@ This last field breaks the run down by container, and lists the specific CVEs th
 #### Consistency Evaluator
 This evaluator measures the consistency of results across repeated runs. It can be configured to measure consistency of the `status` or `label` fields.
 
-Consistency is defined as: `∑p²` per CVE, where *p* is the probability of each unique status or label. For example, if the status for a CVE across 3 runs are `(AFFECTED, NOT_AFFECTED, NOT_AFFECTED)`, then the probabilities become `p(AFFECTEd, NOT_AFFECTED) = {1/3, 2/3}`. Therefore, the consistency score is `(1/3)² + (2/3)² = 0.56`. A score of 1.0 means perfect consistency (all runs yielded the exact same values). The evaluator outputs a consistency score for each CVE.
+Consistency is defined as: `∑p²` per CVE, where *p* is the probability of each unique status or label. For example, if the status for a CVE across 3 runs are `(AFFECTED, NOT_AFFECTED, NOT_AFFECTED)`, then the probabilities become `p(AFFECTED) = 1/3`, `p(NOT_AFFECTED) = 2/3`. Therefore, the consistency score is `(1/3)² + (2/3)² = 0.56`. A score of 1.0 means perfect consistency (all runs yielded the exact same values). The evaluator outputs a consistency score for each CVE.
 
-**Note: This evaluator should only be used if evaluation is being run with multiple repetitions. An error will occur if this criteria is not met.** To enable this evaluator, uncomment the last section in `config-eval.yml`.
+> [!NOTE]
+> This evaluator should only be used if evaluation is being run with multiple repetitions. An error will occur if this criteria is not met. To enable this evaluator, uncomment the last section in `config-eval.yml`.
 
 
 ##### Configuration Options
-The accuracy evaluator is implemented in `eval/evaluators/consistency.py` and registered in NeMo Agent Toolkit with the unique identifer (`_type`) of "`consistency`".
+The accuracy evaluator is implemented in `eval/evaluators/consistency.py` and registered in NeMo Agent Toolkit with the unique identifer (`_type`) of `consistency`.
 
 The evaluator takes in a configurable target field (`status` or `label`), following a similar design pattern as the [Simple Accuracy Evaluator](#configuration-options)
 

From e41d49eddb704d3f33123c6fcbf73c4b709b3037 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 11:14:50 -0800
Subject: [PATCH 080/104] Update README.md

Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 27cb8ae00..902d7fac3 100644
--- a/README.md
+++ b/README.md
@@ -996,7 +996,7 @@ Users can list any number of output files, as well as configure chart title, sav
 ###### eval_output_items Field
 The `eval_output_items` field gives a more detailed output of the experiment. It lists the runs, giving each run a unique `id`, a `score` (the accuracy score for that run), and a `reasoning` field.
 
-This last field breaks the run down by container, and lists the specific CVEs that were run in each container (`question`), the ground truth answers `[target field]_answer`, and the answers generated by the workflow (`generated_[target field_answer`).
+This last field breaks the run down by container, and lists the specific CVEs that were run in each container (`question`), the ground truth answers `[target field]_answer`, and the answers generated by the workflow (`generated_[target field]_answer`).
 
 
 

From 8ff40d3a39a7b922f35443caf4e12556289b10d9 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 11:15:05 -0800
Subject: [PATCH 081/104] Update README.md

Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 902d7fac3..d58e301a2 100644
--- a/README.md
+++ b/README.md
@@ -973,7 +973,7 @@ The `average_score` field contains a dict with metrics corresponding to `pandas.
 "average_score": {
     "count": 1,
     "mean": 0.8,
-    "std": NaN,
+    "std": null,
     "min": 0.8,
     "25%": 0.8,
     "50%": 0.8,

From e08aac9b5e65a86ab7d7d34cc97881df9c17d581 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 11:15:30 -0800
Subject: [PATCH 082/104] Update README.md

Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d58e301a2..0bfd408a1 100644
--- a/README.md
+++ b/README.md
@@ -968,7 +968,7 @@ The evaluator also takes in a `duplicates_policy` field. This is to account for
 The final scores are written to the files `status_accuracy_output.json` and `label_accuracy_output.json`. The output will include an `average_score` overview field, and a an `eval_output_items` field which gives a more detailed breakdown.
 
 ###### average_score Field
-The `average_score` field contains a dict with metrics corresponding to `pandas.Dataframe.describe` output. The goal of this field is to capture the consistency of runs, when a user uses the NeMo Agent Toolkit `--reps` option to perform multiple run of the experiment. If only one run is performed, `count` and `mean` would be the only meaningful metrics; for example, if the run yielded accuracy of 0.8, the `average_score` field would look something like this:
+The `average_score` field contains a dict with metrics corresponding to `pandas.Dataframe.describe` output. The goal of this field is to aggregate the accuracy score across all runs, e.g., when using the `--reps` option. If only one run is performed, `count` and `mean` would be the only meaningful metrics; for example, if the run yielded accuracy of 0.8, the `average_score` field would look like this:
 ```json
 "average_score": {
     "count": 1,

From 5aa552e922fcae1f4c3f74ee7c9e40f9bc0114e1 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 11:15:56 -0800
Subject: [PATCH 083/104] Update README.md

Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
---
 README.md | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 0bfd408a1..6296194bf 100644
--- a/README.md
+++ b/README.md
@@ -1123,14 +1123,13 @@ Error requesting [1/10]: (Retry 0.1 sec) https://services.nvd.nist.gov/rest/json
 
 ##### 429 Errors - Too Many Requests
 
-`Exception: [429] Too Many Requests` can occur when using hosted model endpoints, if your requests exceed the rate limit.
+`Exception: [429] Too Many Requests` can occur when using NVIDIA-hosted model endpoints, if your requests exceed the rate limit.
 
-These errors most commonly occur during the `cve_agent_executor` part of the workflow. Try setting the `cve_agent_executor.max_concurrency` in the [config.yml](src/vuln_analysis/configs/config.yml) to a low value such as 5 to reduce the rate of requests.
+These errors most commonly occur during the `cve_agent_executor` part of the workflow, which has high default concurrency. Try setting the `cve_agent_executor.max_concurrency` in the [config.yml](src/vuln_analysis/configs/config.yml) to a low value such as 5 to reduce the rate of requests.
 
-With large input sets, you may additionally run into `429` errors from the concurrency of other parts of the workflow, such as `cve_checklist`, `cve_summarize`, and `cve_justify`. In this case, we do not have a tunable `max_concurrency` limiter built in; instead, developers should consider switching to [self-hosting](#using-self-hosted-nims).
-
-Additionally, when running evaluation, `nat eval` parallelizes evaluation input items and evaluation repetitions. Therefore, evaluation pipelines are especially prone to triggering 429 errors. You can set the `eval.general.max_concurrency` to a low value to reduce parallelization and therefore reduce the rate of requests. However, note that if your evaluation test set is large, or you are running many repetitions, the rate-limited hosted endpoint will be very slow; in this case, we recommend switching to [self-hosting](#using-self-hosted-nims). 
+With a large number of input CVEs, you may also run into `429` errors from the concurrency of other parts of the workflow. In this case, developers should consider switching to [self-hosting](#using-self-hosted-nims) to accommodate larger inputs for production use cases.
 
+Additionally, when running evaluation, `nat eval` parallelizes evaluation input items and evaluation repetitions. Therefore, evaluation pipelines are especially prone to triggering 429 errors. You can set the `eval.general.max_concurrency` to a low value such as 1 to reduce parallelization and the rate of requests. However, note that this will reduce throughput. If your evaluation test set is large, or you are running many repetitions, we also recommend switching to [self-hosting](#using-self-hosted-nims) for faster evaluation. 
 
 #### Authentication errors
 

From a51d5ed6e6771e3a841f8e0aede3987c621dadd7 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 11:16:42 -0800
Subject: [PATCH 084/104] Update
 src/vuln_analysis/data/eval_datasets/eval_dataset.json

Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
---
 src/vuln_analysis/data/eval_datasets/eval_dataset.json | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/vuln_analysis/data/eval_datasets/eval_dataset.json b/src/vuln_analysis/data/eval_datasets/eval_dataset.json
index 7b834cd96..c488405b5 100644
--- a/src/vuln_analysis/data/eval_datasets/eval_dataset.json
+++ b/src/vuln_analysis/data/eval_datasets/eval_dataset.json
@@ -1,5 +1,6 @@
-{   "dataset_id": "common-test-set",
-    "dataset_description": "Experiment using two containers",
+{
+    "dataset_id": "example-eval-dataset",
+    "dataset_description": "Example eval dataset with two containers",
     "containers": {
         "morpheus-23.11": {
             "container_image": {

From b0051a77e949e6b61a712abe077b965a15c940af Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 11:17:12 -0800
Subject: [PATCH 085/104] Update src/vuln_analysis/eval/parse_eval_input.py

Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
---
 src/vuln_analysis/eval/parse_eval_input.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/vuln_analysis/eval/parse_eval_input.py b/src/vuln_analysis/eval/parse_eval_input.py
index cc461b46f..59891f5ad 100644
--- a/src/vuln_analysis/eval/parse_eval_input.py
+++ b/src/vuln_analysis/eval/parse_eval_input.py
@@ -32,7 +32,7 @@ def parse_input(file_path: Path) -> EvalInput:
     Usage (outside of the full NAT pipeline):
         python3 parse_eval_input.py [test_set_file].json --preview -o [your output file].json
         Example:
-        python3 parse_eval_input.py ../data/eval_datasets/common-test-set.json --preview -o my_eval_string.json
+        python3 parse_eval_input.py ../data/eval_datasets/eval_dataset.json --preview -o my_eval_string.json
     Run this inside the vuln_analysis container, from the eval directory.
 
     Example output:

From f3b82f5b5efaaec0eeb15d2553be1d2e128d36e1 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 11:17:37 -0800
Subject: [PATCH 086/104] Update README.md

Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 6296194bf..4f07037c8 100644
--- a/README.md
+++ b/README.md
@@ -856,7 +856,7 @@ You can also implement custom TTC strategies other than majority voting by follo
 
 
 ## Evaluation
-Evaluation is an essential part of developing agentic workflows. In this section, we use NeMo Agent Toolkit's Evaluation feature to implemenent an example evaluation pipeline that demonstrates best practices for performing evaluation on the vulnerability analysis workflow. There are two custom evaluators included in this example, which provide an extensible template for developers to add their own custom evaluators that target their desired metrics.
+Evaluation is an essential part of developing agentic workflows. In this section, we use NeMo Agent Toolkit's Evaluation feature to implement an example evaluation pipeline that demonstrates best practices for performing evaluation on the vulnerability analysis workflow. There are two custom evaluators included in this example, which provide an extensible template for developers to add their own custom evaluators that target their desired metrics.
 
 To run evaluation, run `nat eval` using the `config-eval.yml` configuration.
 

From e560b6c8165aac626532d8ed836fd1e4b564ac86 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 11:18:19 -0800
Subject: [PATCH 087/104] Update README.md

Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 4f07037c8..e0e6e9f3d 100644
--- a/README.md
+++ b/README.md
@@ -868,7 +868,7 @@ This will run the two custom evaluators, which measure accuracy and consistency.
 
 Visit the [Example Custom Evaluators](#example-custom-evaluators) section to learn more details about these evaluators.
 
-To extend this evaluation example, developers should also write their own custom evaluators to calculate desired metrics. Read more at the NeMo Agent Toolkit documentation for [custom evaluators](https://docs.nvidia.com/nemo/agent-toolkit/1.2/reference/evaluate.html#adding-custom-evaluators). Additionally, developers should provide their own test sets, which contain the ground truth data that the pipeline's output is compared against. See [Inputting Evaluation Test Sets](#inputting-evaluation-test-sets) for details.
+To extend this evaluation example, developers should also write their own custom evaluators to calculate desired metrics. Read more at the NeMo Agent Toolkit documentation for [custom evaluators](https://docs.nvidia.com/nemo/agent-toolkit/1.2/reference/evaluate.html#adding-custom-evaluators). Additionally, developers should provide their own evaluation datasets, which contain the ground truth data that the pipeline's output is compared against. See [Inputting Evaluation Datasets](#inputting-evaluation-datasets) for details.
 
 
 ### Inputting Evaluation Test Sets

From ff48ab61ded55e3f596b71c0ccdf8da1cac2b24d Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 11:18:41 -0800
Subject: [PATCH 088/104] Update README.md

Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index e0e6e9f3d..0fa3bb063 100644
--- a/README.md
+++ b/README.md
@@ -870,6 +870,8 @@ Visit the [Example Custom Evaluators](#example-custom-evaluators) section to lea
 
 To extend this evaluation example, developers should also write their own custom evaluators to calculate desired metrics. Read more at the NeMo Agent Toolkit documentation for [custom evaluators](https://docs.nvidia.com/nemo/agent-toolkit/1.2/reference/evaluate.html#adding-custom-evaluators). Additionally, developers should provide their own evaluation datasets, which contain the ground truth data that the pipeline's output is compared against. See [Inputting Evaluation Datasets](#inputting-evaluation-datasets) for details.
 
+> [!NOTE]
+> The custom evaluators and evaluation dataset format shown below are example implementations and not part of the stable API. Teams should use these as a starting point and adapt them for their specific evaluation needs. These examples may be updated in future releases.
 
 ### Inputting Evaluation Test Sets
 Evaluation test sets containing ground truths should be put in the `data/eval_datasets/` directory. This file path is configurable via the `eval.general.dataset.filepath` field. 

From 622ea094bfebadd466fb69cca6bff9e352a67438 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 11:20:00 -0800
Subject: [PATCH 089/104] Update README.md

Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
---
 README.md | 38 ++++++++++++++++++++++++++------------
 1 file changed, 26 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index 0fa3bb063..33707f1a2 100644
--- a/README.md
+++ b/README.md
@@ -878,18 +878,32 @@ Evaluation test sets containing ground truths should be put in the `data/eval_da
 
 Each test set is a JSON file which represents an evaluation experiment. Experiments can be run across multiple containers. An example test set `eval_dataset.json` is provided, which includes two containers (morpheus_23.11 and morpheus_24.03) which have a few CVEs each. The structure of the test set is as follows:
 
-```
-dataset_id
-dataset_description
-containers:
-     containerA
-          metadata
-          ground-truth: [list of CVEs]
-     containerB
-          metadata
-          ground-truth: [list of CVEs]
-     ...
-```
+```json
+{
+  "dataset_id": "...",
+  "dataset_description": "...",
+  "containers": {
+    "containerA": {
+      "container_image": {
+        "name": "...",
+        "tag": "...",
+        "source_info": [...],
+        "sbom_info": {...}
+      },
+      "ground_truth": [
+        {
+          "vuln_id": "CVE-XXXX-XXXX",
+          "status": "NOT AFFECTED",
+          "label": "..."
+        }
+      ]
+    },
+    "containerB": {
+      "container_image": {...},
+      "ground_truth": [...]
+    }
+  }
+}
 
 Developers can input their own test sets by following this file structure.
 

From 516f187713d0d813c50a5b76d8970b2dbaf68d89 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 11:22:02 -0800
Subject: [PATCH 090/104] Update README.md

Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 33707f1a2..c1b592af0 100644
--- a/README.md
+++ b/README.md
@@ -907,7 +907,7 @@ Each test set is a JSON file which represents an evaluation experiment. Experime
 
 Developers can input their own test sets by following this file structure.
 
-A custom parsing script (`eval/parse_eval_input.py`)is used to transform test sets into a format ingestible by the workflow. The path to this script is configured in `eval.general.dataset.function`. Advanced developers can choose to modify the test set input structure by creating their own parsing script, and then updating the configuration field with their own script.
+A custom parsing script (`eval/parse_eval_input.py`) is used to transform test sets into a format ingestible by the workflow. The path to this script is configured in `eval.general.dataset.function`. Advanced developers can choose to modify the test set input structure by creating their own parsing script, and then updating the configuration field with their own script. See the NAT documentation on [Custom Dataset Format](https://docs.nvidia.com/nemo/agent-toolkit/latest/reference/evaluate.html#custom-dataset-format) to learn more about customizing datasets for your evaluation needs.
 
 You can additionally configure the output directory of evaluation results via the `eval.general.output_dir` field.
 

From 21f1cbcedcadf5653c835db582b74d7784cb836e Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 11:22:29 -0800
Subject: [PATCH 091/104] Update README.md

Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index c1b592af0..90b2804da 100644
--- a/README.md
+++ b/README.md
@@ -911,7 +911,8 @@ A custom parsing script (`eval/parse_eval_input.py`) is used to transform test s
 
 You can additionally configure the output directory of evaluation results via the `eval.general.output_dir` field.
 
-**Note: For large input test sets, developers may encounter `429: Too Many Requests` errors if using hosted model endpoints. See [Troubleshooting](#429-errors---too-many-requests) for how to mitigate this.**
+> [!NOTE]
+> For large input test sets, developers may encounter `429: Too Many Requests` errors if using NVIDIA-hosted model endpoints. See [Troubleshooting](#429-errors---too-many-requests) for how to mitigate this.
 
 
 ### Running Multiple Evaluation Repetitions

From c7d20beff372cee32eb8585f848ca439e958b6bd Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 11:27:18 -0800
Subject: [PATCH 092/104] Update README.md

Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 90b2804da..ad4627eb6 100644
--- a/README.md
+++ b/README.md
@@ -873,10 +873,10 @@ To extend this evaluation example, developers should also write their own custom
 > [!NOTE]
 > The custom evaluators and evaluation dataset format shown below are example implementations and not part of the stable API. Teams should use these as a starting point and adapt them for their specific evaluation needs. These examples may be updated in future releases.
 
-### Inputting Evaluation Test Sets
-Evaluation test sets containing ground truths should be put in the `data/eval_datasets/` directory. This file path is configurable via the `eval.general.dataset.filepath` field. 
+### Inputting Evaluation Datasets
+By default, evaluation datasets containing ground truth labels should be put in the `data/eval_datasets/` directory. This file path is configurable via the `eval.general.dataset.filepath` field.
 
-Each test set is a JSON file which represents an evaluation experiment. Experiments can be run across multiple containers. An example test set `eval_dataset.json` is provided, which includes two containers (morpheus_23.11 and morpheus_24.03) which have a few CVEs each. The structure of the test set is as follows:
+Each dataset is a JSON file which represents an evaluation test set. The structure of the dataset is as follows:
 
 ```json
 {

From 0644ab7c9b24d2d1671b5ed29d66e5674d2f90ae Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 11:27:37 -0800
Subject: [PATCH 093/104] Update README.md

Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ad4627eb6..5a0635e63 100644
--- a/README.md
+++ b/README.md
@@ -905,7 +905,7 @@ Each dataset is a JSON file which represents an evaluation test set. The structu
   }
 }
 
-Developers can input their own test sets by following this file structure.
+Evaluation datasets can contain ground truth data across multiple containers. Each entry in the `containers` field consists of input info for a container, along with a list of CVEs and their corresponding ground truth labels. An example test set `eval_dataset.json` is provided, which includes two containers (`morpheus:23.11-runtime` and `morpheus:24.03-runtime`) which have a few CVEs each. Developers can input their own test sets by following this file structure.
 
 A custom parsing script (`eval/parse_eval_input.py`) is used to transform test sets into a format ingestible by the workflow. The path to this script is configured in `eval.general.dataset.function`. Advanced developers can choose to modify the test set input structure by creating their own parsing script, and then updating the configuration field with their own script. See the NAT documentation on [Custom Dataset Format](https://docs.nvidia.com/nemo/agent-toolkit/latest/reference/evaluate.html#custom-dataset-format) to learn more about customizing datasets for your evaluation needs.
 

From 5347c96cc5f312777a2fe9c0f853f32463f8e8c9 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 11:28:20 -0800
Subject: [PATCH 094/104] Update README.md

Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5a0635e63..78cf30ff4 100644
--- a/README.md
+++ b/README.md
@@ -925,7 +925,8 @@ Below is an example of how to run an evaluation experiment with 3 repetitions:
 nat eval --config_file=configs/config-eval.yml --reps=3
 ```
 
-**Note: Running multiple repetitions may take some extra time. Additionally, developers may encounter `429: Too Many Requests` errors if using hosted model endpoints, due to the concurrency introduced when running multiple repetition. See [Troubleshooting](#429-errors---too-many-requests) for how to mitigate this.**
+> [!NOTE]
+> Running multiple repetitions may take some extra time. Additionally, developers may encounter `429: Too Many Requests` errors if using hosted model endpoints, due to the concurrency introduced when running multiple repetitions. See [Troubleshooting](#429-errors---too-many-requests) for how to mitigate this.
 
 
 #### Disable Caching

From e795af7609d46037af10b677a97b0235ebcbd8bb Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 11:29:14 -0800
Subject: [PATCH 095/104] Update README.md

Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
---
 README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 78cf30ff4..50254b9b0 100644
--- a/README.md
+++ b/README.md
@@ -947,12 +947,12 @@ export NVIDIA_API_BASE=http://nginx-cache/nim_llm/v1
 ```
 
 
-##### Option 2: Disable Caching on Specific Sections
-This option enables developers to disable caching for specific parts of the workflow. It is useful for disabling caching on all model calls except the embedding model, in order to avoid the time-consuming embedding process. This approach is also useful for in-depth evaluation experiments; for example, a developer may want to disable caching for just the `justify_llm` in order to test consistency of the justification part of the workflow without re-running everything before that point. 
+##### Option 2: Disable Caching on Specific Components
+This option enables developers to disable caching for specific parts of the workflow. This approach is useful for in-depth evaluation experiments; for example, a developer may want to disable caching for just the `justify_llm` in order to test consistency of the justification part of the workflow without re-running everything before that point.
 
-In `config-eval.yml`, assign the `base_url` field of the target section to `"https://integrate.api.nvidia.com/v1"`. We recommend creating multiple config files to keep track of different evaluation experiments. To re-enable caching, simply assign the `base_url` back to `${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}`.
+In `config-eval.yml`, assign the `base_url` field of the target components to `"https://integrate.api.nvidia.com/v1"`. We recommend creating multiple config files to keep track of different evaluation experiments. To re-enable caching, simply assign the `base_url` back to `${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}`.
 
-For quick testing, developers can also change configuration in the CLI using the `--override` flag, which uses dot notation. This change will not persist across future runs. Below is an example that disables caching on the checklist_llm.
+For quick testing, developers can also change configuration in the CLI using the `--override` flag, which uses dot notation. This change will not persist across future runs. Below is an example that disables caching on the `checklist_llm`.
 
 ```
 nat eval --config_file=configs/config-eval.yml --reps=3 --override llms.checklist_llm.base_url https://integrate.api.nvidia.com/v1

From d7772651273cf779bfcf91e57624e96168c69ce4 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 11:29:42 -0800
Subject: [PATCH 096/104] Update README.md

Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
---
 README.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/README.md b/README.md
index 50254b9b0..aa7febaba 100644
--- a/README.md
+++ b/README.md
@@ -939,8 +939,6 @@ Globally disable all LLM and embedding caches by setting the `NVIDIA_API_BASE` v
 export NVIDIA_API_BASE=https://integrate.api.nvidia.com/v1
 ```
 
-**WARNING: This approach will cause the embeddings of the input container's source code and documentation input to be regenerated every run, which is both time-consuming and compute-expensive.**
-
 Re-enable caching by setting the `NVIDIA_API_BASE` variable back to NGINX.
 ```
 export NVIDIA_API_BASE=http://nginx-cache/nim_llm/v1

From 110aada666b10a11a8844db3643101ab0328ddc9 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 11:30:21 -0800
Subject: [PATCH 097/104] Update README.md

Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index aa7febaba..a50bbb40c 100644
--- a/README.md
+++ b/README.md
@@ -970,9 +970,9 @@ For each CVE, if the workflow output matches the ground truth, a score of 1 is a
 
 
 ##### Configuration Options
-The accuracy evaluator is implemented in `eval/evaluators/accuracy.py` and registered in NeMo Agent Toolkit with the unique identifer (`_type`) of "`accuracy`".
+The accuracy evaluator is implemented in `eval/evaluators/accuracy.py` and registered in NeMo Agent Toolkit with the unique identifer (`_type`) of `accuracy`.
 
-The evaluator takes in a target field (`status` or `label`), which can be configured in `config-eval.yml` under `evaluators.[evaluator name].field`. This design follows a common NeMo Agent Toolkit design pattern to promote code reussability. In `config-eval.yml`, a `status_accuracy_evaluator` and a `label_accuracy_evaluator` are set up as two separate evaluators, but under the hood, they share the same `_type` and therefore the same source code. 
+The evaluator takes in a target field (`status` or `label`), which can be configured in `config-eval.yml` under `evaluators.[evaluator name].field`. This design follows a common NeMo Agent Toolkit design pattern to promote code reusability. In `config-eval.yml`, a `status_accuracy_evaluator` and a `label_accuracy_evaluator` are set up as two separate evaluators, but under the hood, they share the same `_type` and therefore the same source code.
 
 The evaluator also takes in a `duplicates_policy` field. This is to account for instances where, if an input set is an aggregation of ground truths from different sources, the set may contain duplicate CVEs with conflicting answers. For example, a CVE may be listed twice in the dataset as both `AFFECTED` and `NOT AFFECTED`, with different label justifications. We provide three policies for handling duplicates:
 - `drop_all`: Any CVE with conflicting ground truth will be omitted and not contribute towards evaluation results.

From 4233d428f2f9ec31b341a5bc5f7b0f49bc56ceaf Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 12:11:04 -0800
Subject: [PATCH 098/104] Update README.md

Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a50bbb40c..92d855814 100644
--- a/README.md
+++ b/README.md
@@ -966,7 +966,7 @@ Each CVE has a ground truth `status`, as well as a `label` which corresponds to
 Valid statuses include: `AFFECTED` or `NOT AFFECTED`.
 Valid labels include: `vulnerable` if the CVE status is `AFFECTED`, or one of 10 VEX statuses if the status is `NOT AFFECTED` (see **Justification Assignment** under the [Key Components section](#key-components)).
 
-For each CVE, if the workflow output matches the ground truth, a score of 1 is assigned. Otherwise, a score of 0 is assigned. The evaluator then takes the mean of all CVEs' scores to produce an average accuracy.
+For each CVE, if the workflow output matches the ground truth, a score of 1 is assigned. Otherwise, a score of 0 is assigned. The evaluator then takes the mean of all CVEs' scores to produce an accuracy score.
 
 
 ##### Configuration Options

From 88d60f85b8f44a21cf9033d80f68c4da684e339e Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 12:22:35 -0800
Subject: [PATCH 099/104] minor readme updates

---
 README.md | 21 ++++-----------------
 1 file changed, 4 insertions(+), 17 deletions(-)

diff --git a/README.md b/README.md
index 92d855814..155d98471 100644
--- a/README.md
+++ b/README.md
@@ -65,20 +65,11 @@ limitations under the License.
     - [Customizing the TTC configuration](#customizing-the-ttc-configuration)
     - [Writing a custom TTC strategy](#writing-a-custom-ttc-strategy)
 - [Evaluation](#evaluation)
-  - [Inputting Evaluation Test Sets](#inputting-evaluation-test-sets)
-  - [Running Multiple Evaluation Repetitions](#running-multiple-evaluation-repetitions)
-    - [Disable Caching](#disable-caching)
-      - [Option 1: Disable Caching Globally](#option-1-disable-caching-globally)
-      - [Option 2: Disable Caching on Specific Sections](#option-2-disable-caching-on-specific-sections)
-  - [Example Custom Evaluators](#example-custom-evaluators)
-    - [Simple Accuracy Evaluator](#simple-accuracy-evaluator)
-      - [Configuration Options](#configuration-options)
-      - [Output Format](#output-format)
-        - [average\_score Field](#average_score-field)
+  - [Inputting Evaluation Datasets](#inputting-evaluation-datasets)
         - [eval\_output\_items Field](#eval_output_items-field)
     - [Consistency Evaluator](#consistency-evaluator)
-      - [Configuration Options](#configuration-options-1)
-      - [Output Format](#output-format-1)
+      - [Configuration Options](#configuration-options)
+      - [Output Format](#output-format)
 - [Troubleshooting](#troubleshooting)
   - [Git LFS issues](#git-lfs-issues)
   - [Container build issues](#container-build-issues)
@@ -94,7 +85,6 @@ limitations under the License.
       - [429 Errors - Too Many Requests](#429-errors---too-many-requests)
     - [Authentication errors](#authentication-errors)
   - [Running out of credits](#running-out-of-credits)
-  - [Mac Workarounds](#mac-workarounds)
 - [Testing and validation](#testing-and-validation)
 - [Roadmap](#roadmap)
 - [Cite](#cite)
@@ -624,7 +614,7 @@ The configuration defines how the workflow operates, including functions, LLMs,
       - `skip_agent`: Collect intel and check dependencies, but skip agent
       - `continue_with_warning`: Run full pipeline with warning log (degraded analysis quality). Default is "continue_with_warning" for backwards compatibility.
     - The remaining configuration items correspond to attributes in [CVEWorkflowConfig](src/vuln_analysis/register.py) to specify the registered tools to use in the workflow.
-4. **Evaluations and Profiling** (`eval`): The `eval` section contains the evaluation settings for the workflow. Refer to [Evaluating NVIDIA Agent Intelligence Toolkit Workflows](https://docs.nvidia.com/nemo/agent-toolkit/latest/workflows/evaluate.html) for more information about NeMo Agent toolkit built-in evaluators as well as the plugin system to add custom evaluators. The CVE workflow uses the `eval` section to configure a profiler that uses the NeMo Agent toolkit evaluation system to collect usage statistics and store them to the local file system. You can find more information about NeMo Agent toolkit profiling and performance monitoring [here](https://docs.nvidia.com/nemo/agent-toolkit/latest/workflows/profiler.html).
+4. **Evaluations and Profiling** (`eval`): The `eval` section contains the evaluation settings for the workflow. Refer to [Evaluating NVIDIA Agent Intelligence Toolkit Workflows](https://docs.nvidia.com/nemo/agent-toolkit/latest/workflows/evaluate.html) for more information about NeMo Agent toolkit built-in evaluators as well as the plugin system to add custom evaluators. An example evaluation pipeline has been provided in the [Evaluation](#evaluation) section of this README. Additionally, the CVE workflow uses the `eval` section to configure a profiler that uses the NeMo Agent Toolkit evaluation system to collect usage statistics and store them to the local file system. You can find more information about NeMo Agent toolkit profiling and performance monitoring [here](https://docs.nvidia.com/nemo/agent-toolkit/latest/workflows/profiler.html).
     - `general.output_dir`: Defines the path to the directory where profiling results will be saved.
     - `general.dataset`: Defines file path and format of dataset used to run profiling.
     - `evaluators`: Custom evaluators defined in `eval/evaluators`. See [Evaluation](#evaluation) for more details on each custom evaluator and their configurations.
@@ -1162,9 +1152,6 @@ Note that exporting the required environment variables in a container shell will
 
 If you run out of credits for the NVIDIA API Catalog, you will need to obtain more credits to continue using the API. Please contact your NVIDIA representative to get more credits added.
 
-### Mac Workarounds
-Development on macOS is supported on a best-effort basis and may require workarounds. One current known issue is that running the project docker container causes a `ModuleNotFoundError: No module named 'numpy.distutils'` related to faiss. To work around this, in the `Dockerfile` change the `ARG PYTHON_VERSION=3.12` line to `ARG PYTHON_VERSION=3.11`. The full issue is tracked [here](https://github.com/NVIDIA-AI-Blueprints/vulnerability-analysis/issues/115).
-
 ## Testing and validation
 Test-driven development is essential for building reliable LLM-based agentic systems, especially when deploying or scaling them in production environments.
 

From 5f84e4667c4b115016023fbdeee014cd5f1c3ff5 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 12:25:44 -0800
Subject: [PATCH 100/104] update eval dataset container names to match example
 input message datasets

---
 src/vuln_analysis/data/eval_datasets/eval_dataset.json | 4 ++--
 src/vuln_analysis/eval/parse_eval_input.py             | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/vuln_analysis/data/eval_datasets/eval_dataset.json b/src/vuln_analysis/data/eval_datasets/eval_dataset.json
index c488405b5..a0107930d 100644
--- a/src/vuln_analysis/data/eval_datasets/eval_dataset.json
+++ b/src/vuln_analysis/data/eval_datasets/eval_dataset.json
@@ -2,7 +2,7 @@
     "dataset_id": "example-eval-dataset",
     "dataset_description": "Example eval dataset with two containers",
     "containers": {
-        "morpheus-23.11": {
+        "morpheus:23.11-runtime": {
             "container_image": {
                 "name": "nvcr.io/nvidia/morpheus/morpheus",
                 "tag": "23.11-runtime",
@@ -53,7 +53,7 @@
                 }
             ]
         },
-        "morpheus-24.03": {
+        "morpheus:24.03-runtime": {
             "container_image": {
                 "name": "nvcr.io/nvidia/morpheus/morpheus",
                 "tag": "v24.03.02-runtime",
diff --git a/src/vuln_analysis/eval/parse_eval_input.py b/src/vuln_analysis/eval/parse_eval_input.py
index 59891f5ad..baf6478b8 100644
--- a/src/vuln_analysis/eval/parse_eval_input.py
+++ b/src/vuln_analysis/eval/parse_eval_input.py
@@ -38,12 +38,12 @@ def parse_input(file_path: Path) -> EvalInput:
     Example output:
         [
             {
-                "id": "morpheus-23.11",
+                "id": "morpheus:23.11-runtime",
                 "question": "{\"image\":{\"name\":\"nvcr.io/nvidia/morpheus/morpheus\",\"tag\":\"23.11-runtime\",\"source_info\":[{\"type\":\"code\",\"git_repo\":\"https://github.com/nv-morpheus/Morpheus.git\",\"ref\":\"branch-23.11\",\"include\":[\"**/*.cpp\",\"**/*.cu\",\"**/*.cuh\",\"**/*.h\",\"**/*.hpp\",\"**/*.ipynb\",\"**/*.py\",\"**/*Dockerfile\"],\"exclude\":[\"tests/**/*\"]},{\"type\":\"doc\",\"git_repo\":\"https://github.com/nv-morpheus/Morpheus.git\",\"ref\":\"branch-23.11\",\"include\":[\"**/*.md\",\"docs/**/*.rst\"]}],\"sbom_info\":{\"_type\":\"file\",\"file_path\":\"data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom\"}},\"scan\":{\"vulns\":[{\"vuln_id\":\"GHSA-3f63-hfp8-52jq\"},{\"vuln_id\":\"CVE-2023-36632\"}]}}",
                 "answer": "{\"GHSA-3f63-hfp8-52jq\":\"NOT AFFECTED\",\"GHSA-3f63-hfp8-52jq_label\":\"code_not_reachable\",\"CVE-2023-36632\":\"AFFECTED\",\"CVE-2023-36632_label\":\"vulnerable\"}"
             },
             {
-                "id": "morpheus-24.03",
+                "id": "morpheus:24.03-runtime",
                 "question": "{\"image\":{\"name\":\"nvcr.io/nvidia/morpheus/morpheus\",\"tag\":\"v24.03.02-runtime\",\"source_info\":[{\"type\":\"code\",\"git_repo\":\"https://github.com/nv-morpheus/Morpheus.git\",\"ref\":\"v24.03.02\",\"include\":[\"**/*.cpp\",\"**/*.cu\",\"**/*.cuh\",\"**/*.h\",\"**/*.hpp\",\"**/*.ipynb\",\"**/*.py\",\"**/*Dockerfile\"],\"exclude\":[\"tests/**/*\"]},{\"type\":\"doc\",\"git_repo\":\"https://github.com/nv-morpheus/Morpheus.git\",\"ref\":\"v24.03.02\",\"include\":[\"**/*.md\",\"docs/**/*.rst\"]}],\"sbom_info\":{\"_type\":\"file\",\"file_path\":\"data/sboms/nvcr.io/nvidia/morpheus/morpheus:v24.03.02-runtime.sbom\"}},\"scan\":{\"vulns\":[{\"vuln_id\":\"GHSA-3f63-hfp8-52jq\"},{\"vuln_id\":\"CVE-2023-36632\"}]}}",
                 "answer": "{\"GHSA-3f63-hfp8-52jq\":\"NOT AFFECTED\",\"GHSA-3f63-hfp8-52jq_label\":\"code_not_reachable\",\"CVE-2023-36632\":\"AFFECTED\",\"CVE-2023-36632_label\":\"vulnerable\"}"
             }

From 6c02d5d34be8dab754e72e860204fb1213d8e1ca Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 12:31:15 -0800
Subject: [PATCH 101/104] example output for accuracy evaluator with multiple
 runs

---
 README.md | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 155d98471..11fa4dbe3 100644
--- a/README.md
+++ b/README.md
@@ -66,12 +66,6 @@ limitations under the License.
     - [Writing a custom TTC strategy](#writing-a-custom-ttc-strategy)
 - [Evaluation](#evaluation)
   - [Inputting Evaluation Datasets](#inputting-evaluation-datasets)
-        - [eval\_output\_items Field](#eval_output_items-field)
-    - [Consistency Evaluator](#consistency-evaluator)
-      - [Configuration Options](#configuration-options)
-      - [Output Format](#output-format)
-- [Troubleshooting](#troubleshooting)
-  - [Git LFS issues](#git-lfs-issues)
   - [Container build issues](#container-build-issues)
     - [Device error](#device-error)
     - [Deploy.Resources.Reservations.devices error](#deployresourcesreservationsdevices-error)
@@ -987,7 +981,19 @@ The `average_score` field contains a dict with metrics corresponding to `pandas.
     "max": 0.8
   }
 ```
-Meanwhile, if there were multiple runs, the `average_score` field would capture the distribution of runs. A script is provided in `eval/visualizations/box_and_whisker_plot.py` to visualize this distribution as a box and whisker plot. It is especially helpful for comparing the plots of multiple experiments side-by-side, or comparing `status` accuracy against `label` accuracy. It can be run like so:
+Meanwhile, if there were multiple runs, the `average_score` field would capture the distribution of runs. For example, with 5 runs, the output might look like this:
+```json
+"average_score": {
+    "count": 5,
+    "mean": 0.74,
+    "std": 0.07,
+    "min": 0.65,
+    "25%": 0.7,
+    "50%": 0.75,
+    "75%": 0.8,
+    "max": 0.8
+  }
+A script is provided in `eval/visualizations/box_and_whisker_plot.py` to visualize this distribution as a box and whisker plot. It is especially helpful for comparing the plots of multiple experiments side-by-side, or comparing `status` accuracy against `label` accuracy. It can be run like so:
 
 ```
 python3 src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py \

From ef1a856f6a43bda421bdedf91a0d2ca3820a1814 Mon Sep 17 00:00:00 2001
From: Katherine Huang 
Date: Mon, 3 Nov 2025 12:32:47 -0800
Subject: [PATCH 102/104] minor readme

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 11fa4dbe3..5be89ebd4 100644
--- a/README.md
+++ b/README.md
@@ -981,7 +981,7 @@ The `average_score` field contains a dict with metrics corresponding to `pandas.
     "max": 0.8
   }
 ```
-Meanwhile, if there were multiple runs, the `average_score` field would capture the distribution of runs. For example, with 5 runs, the output might look like this:
+Meanwhile, if there were multiple runs, the `average_score` field would capture the accuracy distribution across runs. For example, with 5 runs, the output might look like this:
 ```json
 "average_score": {
     "count": 5,

From 631eae47f4fe9341e481b2e29d994fb8cfe545df Mon Sep 17 00:00:00 2001
From: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
Date: Mon, 3 Nov 2025 20:55:26 +0000
Subject: [PATCH 103/104] README cleanup

---
 README.md | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 5be89ebd4..87fa845b5 100644
--- a/README.md
+++ b/README.md
@@ -66,6 +66,21 @@ limitations under the License.
     - [Writing a custom TTC strategy](#writing-a-custom-ttc-strategy)
 - [Evaluation](#evaluation)
   - [Inputting Evaluation Datasets](#inputting-evaluation-datasets)
+  - [Running Multiple Evaluation Repetitions](#running-multiple-evaluation-repetitions)
+    - [Disable Caching](#disable-caching)
+      - [Option 1: Disable Caching Globally](#option-1-disable-caching-globally)
+      - [Option 2: Disable Caching on Specific Components](#option-2-disable-caching-on-specific-components)
+  - [Example Custom Evaluators](#example-custom-evaluators)
+    - [Simple Accuracy Evaluator](#simple-accuracy-evaluator)
+      - [Configuration Options](#configuration-options)
+      - [Output Format](#output-format)
+        - [average\_score Field](#average_score-field)
+        - [eval\_output\_items Field](#eval_output_items-field)
+    - [Consistency Evaluator](#consistency-evaluator)
+      - [Configuration Options](#configuration-options-1)
+      - [Output Format](#output-format-1)
+- [Troubleshooting](#troubleshooting)
+  - [Git LFS issues](#git-lfs-issues)
   - [Container build issues](#container-build-issues)
     - [Device error](#device-error)
     - [Deploy.Resources.Reservations.devices error](#deployresourcesreservationsdevices-error)
@@ -539,7 +554,7 @@ The configuration defines how the workflow operates, including functions, LLMs,
     - Preprocessing functions:
       - `cve_generate_vdbs`: Generates vector database from code repositories and documentation.
         - `agent_name`: Name of the agent executor (`cve_agent_executor`). Used to determine which tools are enabled in the agent to conditionally generate vector databases or indexes.
-        - `embedder_name`: Name of embedder (`nim-embedder`) configured in `embedders` section. 
+        - `embedder_name`: Name of embedder (`nim-embedder`) configured in `embedders` section.
         - `base_vdb_dir`: The directory used for storing vector database files. Default is `.cache/am_cache/vdb`.
         - `base_git_dir`: The directory for storing pulled git repositories used for code analysis. Default is `.cache/am_cache/git`.
         - `base_code_index_dir`: The directory used for storing code index files. Default is `./cache/am_cache/code_index`.
@@ -888,6 +903,7 @@ Each dataset is a JSON file which represents an evaluation test set. The structu
     }
   }
 }
+```
 
 Evaluation datasets can contain ground truth data across multiple containers. Each entry in the `containers` field consists of input info for a container, along with a list of CVEs and their corresponding ground truth labels. An example test set `eval_dataset.json` is provided, which includes two containers (`morpheus:23.11-runtime` and `morpheus:24.03-runtime`) which have a few CVEs each. Developers can input their own test sets by following this file structure.
 
@@ -979,7 +995,7 @@ The `average_score` field contains a dict with metrics corresponding to `pandas.
     "50%": 0.8,
     "75%": 0.8,
     "max": 0.8
-  }
+}
 ```
 Meanwhile, if there were multiple runs, the `average_score` field would capture the accuracy distribution across runs. For example, with 5 runs, the output might look like this:
 ```json
@@ -992,10 +1008,11 @@ Meanwhile, if there were multiple runs, the `average_score` field would capture
     "50%": 0.75,
     "75%": 0.8,
     "max": 0.8
-  }
+}
+```
 A script is provided in `eval/visualizations/box_and_whisker_plot.py` to visualize this distribution as a box and whisker plot. It is especially helpful for comparing the plots of multiple experiments side-by-side, or comparing `status` accuracy against `label` accuracy. It can be run like so:
 
-```
+```bash
 python3 src/vuln_analysis/eval/visualizations/box_and_whisker_plot.py \
   .tmp/evaluators/accuracy-output-1.json \
   .tmp/evaluators/accuracy-output-2.json \
@@ -1141,7 +1158,7 @@ These errors most commonly occur during the `cve_agent_executor` part of the wor
 
 With a large number of input CVEs, you may also run into `429` errors from the concurrency of other parts of the workflow. In this case, developers should consider switching to [self-hosting](#using-self-hosted-nims) to accommodate larger inputs for production use cases.
 
-Additionally, when running evaluation, `nat eval` parallelizes evaluation input items and evaluation repetitions. Therefore, evaluation pipelines are especially prone to triggering 429 errors. You can set the `eval.general.max_concurrency` to a low value such as 1 to reduce parallelization and the rate of requests. However, note that this will reduce throughput. If your evaluation test set is large, or you are running many repetitions, we also recommend switching to [self-hosting](#using-self-hosted-nims) for faster evaluation. 
+Additionally, when running evaluation, `nat eval` parallelizes evaluation input items and evaluation repetitions. Therefore, evaluation pipelines are especially prone to triggering 429 errors. You can set the `eval.general.max_concurrency` to a low value such as 1 to reduce parallelization and the rate of requests. However, note that this will reduce throughput. If your evaluation test set is large, or you are running many repetitions, we also recommend switching to [self-hosting](#using-self-hosted-nims) for faster evaluation.
 
 #### Authentication errors
 
@@ -1178,7 +1195,7 @@ We recommend that teams looking to test or optimize their CVE analysis system cu
 - [x] Upgrade to [NAT v1.2](https://docs.nvidia.com/nemo/agent-toolkit/1.2/index.html)
 - [x] Configurable NIM error handling
 - [x] Configurable missing source info and VDB handling
-- [ ] [NAT Evaluation](https://docs.nvidia.com/nemo/agent-toolkit/1.2/workflows/evaluate.html) integration
+- [x] [NAT Evaluation](https://docs.nvidia.com/nemo/agent-toolkit/latest/workflows/evaluate.html) integration
 - [ ] [NAT W&B Weave](https://docs.nvidia.com/nemo/agent-toolkit/1.2/workflows/observe/observe-workflow-with-weave.html) integration
 - [x] [NAT Test Time Compute](https://docs.nvidia.com/nemo/agent-toolkit/latest/reference/test-time-compute.html) integration
 - [ ] Upgrade to NAT v1.3

From 54b0cc34922968da49daedafbc5c19ca241ba588 Mon Sep 17 00:00:00 2001
From: Ashley Song <165685692+ashsong-nv@users.noreply.github.com>
Date: Mon, 3 Nov 2025 21:02:18 +0000
Subject: [PATCH 104/104] Fix JSON syntax highlighting in README

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 87fa845b5..139070c7c 100644
--- a/README.md
+++ b/README.md
@@ -877,7 +877,7 @@ By default, evaluation datasets containing ground truth labels should be put in
 
 Each dataset is a JSON file which represents an evaluation test set. The structure of the dataset is as follows:
 
-```json
+```
 {
   "dataset_id": "...",
   "dataset_description": "...",