Evaluation

src/vuln_analysis/configs/config-eval.yml

@@ -0,0 +1,199 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+general:
+  use_uvloop: true
+
+functions:
+  cve_generate_vdbs:
+    _type: cve_generate_vdbs
+    agent_name: cve_agent_executor  # Used to determine which tools are enabled
+    embedder_name: nim_embedder
+    base_git_dir: .cache/am_cache/git
+    base_vdb_dir: .cache/am_cache/vdb
+    base_code_index_dir: .cache/am_cache/code_index
+  cve_fetch_intel:
+    _type: cve_fetch_intel
+  cve_process_sbom:
+    _type: cve_process_sbom
+  cve_check_vuln_deps :
+    _type: cve_check_vuln_deps
+    skip: false
+  cve_checklist:
+    _type: cve_checklist
+    llm_name: checklist_llm
+  Container Image Code QA System:
+    _type: local_vdb_retriever
+    embedder_name: nim_embedder
+    llm_name: code_vdb_retriever_llm
+    vdb_type: code
+    return_source_documents: false
+  Container Image Developer Guide QA System:
+    _type: local_vdb_retriever
+    embedder_name: nim_embedder
+    llm_name: doc_vdb_retriever_llm
+    vdb_type: doc
+    return_source_documents: false
+  Lexical Search Container Image Code QA System:
+    _type: lexical_code_search
+    top_k: 5
+  Internet Search:
+     _type: serp_wrapper
+     max_retries: 5
+  cve_agent_executor:
+    _type: cve_agent_executor
+    llm_name: cve_agent_executor_llm
+    tool_names:
+      - Container Image Code QA System
+      - Container Image Developer Guide QA System
+      # - Lexical Search Container Image Code QA System  # Uncomment to enable lexical search
+      - Internet Search
+    max_concurrency: null
+    max_iterations: 10
+    prompt_examples: false
+    replace_exceptions: true
+    replace_exceptions_value: "I do not have a definitive answer for this checklist item."
+    return_intermediate_steps: false
+    verbose: false
+  cve_summarize:
+    _type: cve_summarize
+    llm_name: summarize_llm
+  cve_justify:
+    _type: cve_justify
+    llm_name: justify_llm
+  cve_file_output:
+    _type: cve_file_output
+    file_path: .tmp/output.json
+    markdown_dir: .tmp/vulnerability_markdown_reports
+    overwrite: true
+
+llms:
+  checklist_llm:
+    _type: nim
+    base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}
+    model_name: ${CHECKLIST_MODEL_NAME:-meta/llama-3.1-70b-instruct}
+    temperature: 0.0
+    max_tokens: 2000
+    top_p: 0.01
+    do_auto_retry: True
+    max_retries: 5
+    retry_on_status_codes: [429, 500, 502, 503, 504]
+    retry_on_errors: ["Too Many Requests", "Internal Server Error", "Bad Gateway", "Service Unavailable", "Gateway Time-out", "Gateway Timeout"]
+  code_vdb_retriever_llm:
+    _type: nim
+    base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}
+    model_name: ${CODE_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct}
+    temperature: 0.0
+    max_tokens: 2000
+    top_p: 0.01
+    do_auto_retry: True
+    max_retries: 5
+    retry_on_status_codes: [429, 500, 502, 503, 504]
+    retry_on_errors: ["Too Many Requests", "Internal Server Error", "Bad Gateway", "Service Unavailable", "Gateway Time-out", "Gateway Timeout"]
+  doc_vdb_retriever_llm:
+    _type: nim
+    base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}
+    model_name: ${DOC_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct}
+    temperature: 0.0
+    max_tokens: 2000
+    top_p: 0.01
+    do_auto_retry: True
+    max_retries: 5
+    retry_on_status_codes: [429, 500, 502, 503, 504]
+    retry_on_errors: ["Too Many Requests", "Internal Server Error", "Bad Gateway", "Service Unavailable", "Gateway Time-out", "Gateway Timeout"]
+  cve_agent_executor_llm:
+    _type: nim
+    base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}
+    model_name: ${CVE_AGENT_EXECUTOR_MODEL_NAME:-meta/llama-3.1-70b-instruct}
+    temperature: 0.0
+    max_tokens: 2000
+    top_p: 0.01
+    do_auto_retry: True
+    max_retries: 5
+    retry_on_status_codes: [429, 500, 502, 503, 504]
+    retry_on_errors: ["Too Many Requests", "Internal Server Error", "Bad Gateway", "Service Unavailable", "Gateway Time-out", "Gateway Timeout"]
+  summarize_llm:
+    _type: nim
+    base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}
+    model_name: ${SUMMARIZE_MODEL_NAME:-meta/llama-3.1-70b-instruct}
+    temperature: 0.0
+    max_tokens: 1024
+    top_p: 0.01
+    do_auto_retry: True
+    max_retries: 5
+    retry_on_status_codes: [429, 500, 502, 503, 504]
+    retry_on_errors: ["Too Many Requests", "Internal Server Error", "Bad Gateway", "Service Unavailable", "Gateway Time-out", "Gateway Timeout"]
+  justify_llm:
+    _type: nim
+    base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}
+    model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct}
+    temperature: 0.0
+    max_tokens: 1024
+    top_p: 0.01
+    do_auto_retry: True
+    max_retries: 5
+    retry_on_status_codes: [429, 500, 502, 503, 504]
+    retry_on_errors: ["Too Many Requests", "Internal Server Error", "Bad Gateway", "Service Unavailable", "Gateway Time-out", "Gateway Timeout"]
+
+embedders:
+  nim_embedder:
+    _type: nim
+    base_url: ${NIM_EMBED_BASE_URL:-https://integrate.api.nvidia.com/v1}
+    model_name: ${EMBEDDER_MODEL_NAME:-nvidia/nv-embedqa-e5-v5}
+    truncate: END
+    max_batch_size: 128
+    do_auto_retry: True
+    max_retries: 5
+    retry_on_status_codes: [429, 500, 502, 503, 504]
+    retry_on_errors: ["Too Many Requests", "Internal Server Error", "Bad Gateway", "Service Unavailable", "Gateway Time-out", "Gateway Timeout"]
+
+workflow:
+  _type: cve_agent
+  cve_generate_vdbs_name: cve_generate_vdbs
+  cve_fetch_intel_name: cve_fetch_intel
+  cve_process_sbom_name: cve_process_sbom
+  cve_check_vuln_deps_name: cve_check_vuln_deps
+  cve_checklist_name: cve_checklist
+  cve_agent_executor_name: cve_agent_executor
+  cve_summarize_name: cve_summarize
+  cve_justify_name: cve_justify
+  cve_output_config_name: cve_file_output
+
+eval:
+  general:
+    output_dir: ./.tmp/evaluators
+    dataset:
+        _type: custom
+        file_path: data/eval_datasets/eval_dataset.json
+        function: vuln_analysis.eval.parse_eval_input.parse_input
+
+  evaluators:
+    status_accuracy:
+      _type: accuracy
+      field: status
+      duplicates_policy: drop_all  # Options: drop_all, keep_first, keep_all
+    label_accuracy:
+      _type: accuracy
+      field: label
+      duplicates_policy: drop_all
+
+    # Uncomment to enable consistency evaluator
+    # NOTE: Only use this evaluator if evaluation is being run with multiple repetitions
+    # status_consistency:
+    #   _type: consistency
+    #   field: status
+    # label_consistency:
+    #   _type: consistency
+    #   field: label

src/vuln_analysis/configs/config-tracing.yml

@@ -185,10 +185,10 @@ workflow:
 
 eval:
   general:
-    output_dir: ./.tmp/eval/cve_agent
+    output_dir: ./.tmp/profiler
     dataset:
         _type: json
-        file_path: data/eval_datasets/eval_dataset.json
+        file_path: data/profiler_datasets/profiler_dataset.json
 
     profiler:
         token_uniqueness_forecast: true

src/vuln_analysis/configs/config.yml

@@ -175,10 +175,10 @@ workflow:
 
 eval:
   general:
-    output_dir: ./.tmp/eval/cve_agent
+    output_dir: ./.tmp/profiler
     dataset:
         _type: json
-        file_path: data/eval_datasets/eval_dataset.json
+        file_path: data/profiler_datasets/profiler_dataset.json
 
     profiler:
         token_uniqueness_forecast: true

src/vuln_analysis/data/eval_datasets/eval_dataset.json

@@ -1,7 +1,103 @@
-[
-    {
-        "id": 1,
-        "question": "{\"image\":{\"name\":\"nvcr.io\/nvidia\/morpheus\/morpheus\",\"tag\":\"23.11-runtime\",\"source_info\":[{\"type\":\"code\",\"git_repo\":\"https:\/\/github.com\/nv-morpheus\/Morpheus.git\",\"ref\":\"v23.11.01\",\"include\":[\"**\/*.cpp\",\"**\/*.cu\",\"**\/*.cuh\",\"**\/*.h\",\"**\/*.hpp\",\"**\/*.ipynb\",\"**\/*.py\",\"**\/*Dockerfile\"],\"exclude\":[\"tests\/**\/*\"]},{\"type\":\"doc\",\"git_repo\":\"https:\/\/github.com\/nv-morpheus\/Morpheus.git\",\"ref\":\"v23.11.01\",\"include\":[\"**\/*.md\",\"docs\/**\/*.rst\"]}],\"sbom_info\":{\"_type\":\"file\",\"file_path\":\"data\/sboms\/nvcr.io\/nvidia\/morpheus\/morpheus:v23.11.01-runtime.sbom\"}},\"scan\":{\"vulns\":[{\"vuln_id\":\"GHSA-3f63-hfp8-52jq\"},{\"vuln_id\":\"CVE-2023-50782\"}]}}",
-        "answer": "N/A"
+{
+    "dataset_id": "example-eval-dataset",
+    "dataset_description": "Example eval dataset with two containers",
+    "containers": {
+        "morpheus:23.11-runtime": {
+            "container_image": {
+                "name": "nvcr.io/nvidia/morpheus/morpheus",
+                "tag": "23.11-runtime",
+                "source_info": [
+                    {
+                        "type": "code",
+                        "git_repo": "https://github.com/nv-morpheus/Morpheus.git",
+                        "ref": "branch-23.11",
+                        "include": [
+                            "**/*.cpp",
+                            "**/*.cu",
+                            "**/*.cuh",
+                            "**/*.h",
+                            "**/*.hpp",
+                            "**/*.ipynb",
+                            "**/*.py",
+                            "**/*Dockerfile"
+                        ],
+                        "exclude": [
+                            "tests/**/*"
+                        ]
+                    },
+                    {
+                        "type": "doc",
+                        "git_repo": "https://github.com/nv-morpheus/Morpheus.git",
+                        "ref": "branch-23.11",
+                        "include": [
+                            "**/*.md",
+                            "docs/**/*.rst"
+                        ]
+                    }
+                ],
+                "sbom_info": {
+                    "_type": "file",
+                    "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom"
+                }
+            },
+            "ground_truth": [
+                {
+                    "vuln_id": "GHSA-3f63-hfp8-52jq",
+                    "status": "NOT AFFECTED",
+                    "label": "code_not_reachable"
+                },
+                {
+                    "vuln_id": "CVE-2023-5388",
+                    "status": "NOT AFFECTED",
+                    "label": "false_positive"
+                }
+            ]
+        },
+        "morpheus:24.03-runtime": {
+            "container_image": {
+                "name": "nvcr.io/nvidia/morpheus/morpheus",
+                "tag": "v24.03.02-runtime",
+                "source_info": [
+                    {
+                        "type": "code",
+                        "git_repo": "https://github.com/nv-morpheus/Morpheus.git",
+                        "ref": "v24.03.02",
+                        "include": [
+                            "**/*.cpp",
+                            "**/*.cu",
+                            "**/*.cuh",
+                            "**/*.h",
+                            "**/*.hpp",
+                            "**/*.ipynb",
+                            "**/*.py",
+                            "**/*Dockerfile"
+                        ],
+                        "exclude": [
+                            "tests/**/*"
+                        ]
+                    },
+                    {
+                        "type": "doc",
+                        "git_repo": "https://github.com/nv-morpheus/Morpheus.git",
+                        "ref": "v24.03.02",
+                        "include": [
+                            "**/*.md",
+                            "docs/**/*.rst"
+                        ]
+                    }
+                ],
+                "sbom_info": {
+                    "_type": "file",
+                    "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v24.03.02-runtime.sbom"
+                }
+            },
+            "ground_truth": [
+                {
+                    "vuln_id": "CVE-2023-5388",
+                    "status": "NOT AFFECTED",
+                    "label": "false_positive"
+                }
+            ]
+        }
     }
-]
+}

src/vuln_analysis/data/profiler_datasets/profiler_dataset.json

@@ -0,0 +1,7 @@
+[
+    {
+        "id": 1,
+        "question": "{\"image\":{\"name\":\"nvcr.io\/nvidia\/morpheus\/morpheus\",\"tag\":\"23.11-runtime\",\"source_info\":[{\"type\":\"code\",\"git_repo\":\"https:\/\/github.com\/nv-morpheus\/Morpheus.git\",\"ref\":\"v23.11.01\",\"include\":[\"**\/*.cpp\",\"**\/*.cu\",\"**\/*.cuh\",\"**\/*.h\",\"**\/*.hpp\",\"**\/*.ipynb\",\"**\/*.py\",\"**\/*Dockerfile\"],\"exclude\":[\"tests\/**\/*\"]},{\"type\":\"doc\",\"git_repo\":\"https:\/\/github.com\/nv-morpheus\/Morpheus.git\",\"ref\":\"v23.11.01\",\"include\":[\"**\/*.md\",\"docs\/**\/*.rst\"]}],\"sbom_info\":{\"_type\":\"file\",\"file_path\":\"data\/sboms\/nvcr.io\/nvidia\/morpheus\/morpheus:v23.11.01-runtime.sbom\"}},\"scan\":{\"vulns\":[{\"vuln_id\":\"GHSA-3f63-hfp8-52jq\"},{\"vuln_id\":\"CVE-2023-50782\"}]}}",
+        "answer": "N/A"
+    }
+]