diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..78ccfecd5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,23 @@ +# Folders to ignore +.vscode +**/__pycache__ +**/*.ipynb_checkpoints +**/*Dockerfile* +.tmp + +# Ignore any build folder +./build* +# Ignore the build cache +./.cache + +# Ignore specific files +docker/commands.sh +*.egg-info/ +kafka-docker +# Ignore env files +**/*.env + +#Ignore the deployment configuration +./kustomize/ +**/.venv +**/.git diff --git a/.gitignore b/.gitignore index aa5db811f..97c68c84f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ ###### Place new entries directly below this line! ###### +kustomize/components/oidc-ca/*.crt + # Ignore anything in the ./.tmp directory .tmp/ @@ -155,6 +157,11 @@ ENV/ env.bak/ venv.bak/ +# Env files +*.env +# but keep this one +!kustomize/base/image-registry-credentials.env + # Spyder project settings .spyderproject .spyproject @@ -193,5 +200,16 @@ tags # Persistent undo [._]*.un~ +**/exploit-iq-ips.json +**/user-feedback-ips.json +**/google-sheets-secrets.yaml +**/integration-tests-secrets.yaml +**/server-model-config.yaml +**/sec-decryption.key +**/registry-app-creds.yaml +**/exploit-iq-client-build-ips.yaml +**/exploit-iq-automation-token.yaml + + # End of https://www.gitignore.io/api/vim,c++,cmake,python,synology diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..555ccf7bc --- /dev/null +++ b/.gitmodules @@ -0,0 +1,8 @@ +[submodule ".tests-automation"] + path = .tests-automation + url = https://github.com/RHEcosystemAppEng/exploitiq-tests-automation.git + +[submodule "exploit-iq-models"] + path = exploit-iq-models + url = https://github.com/RHEcosystemAppEng/exploit-iq-models.git + diff --git a/.tekton/cache-cleaner.yaml b/.tekton/cache-cleaner.yaml new file mode 100644 index 000000000..2b456e9d4 --- /dev/null +++ b/.tekton/cache-cleaner.yaml @@ -0,0 +1,65 @@ +--- +# RBAC: The Tekton ServiceAccount (usually `pipeline`) needs `edit` (or a custom +# Role) in target application namespaces to create/delete Jobs and mount PVCs +# (e.g., exploit-iq-testings, exploit-iq-prod). +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: generic-cache-cleaner +spec: + description: >- + Purges application cache from a PVC by spawning a short-lived Job in the + target namespace, waiting for completion, and removing the Job. + params: + - name: pvc-name + type: string + description: Name of the PersistentVolumeClaim holding the cache. + - name: namespace + type: string + description: Target namespace where the PVC lives (e.g., exploit-iq-testings). + steps: + - name: create-cache-cleaner-job + image: image-registry.openshift-image-registry.svc:5000/openshift/cli:latest + script: | + #!/bin/sh + set -eu + + oc apply -f - < enabled=$(cat "$(results.enabled.path)")" + # ------------------------------------------------ + # 1. CLONE + # ------------------------------------------------ + - name: fetch-repository + taskRef: + name: git-clone + workspaces: + - name: output + workspace: source + - name: basic-auth + workspace: basic-auth + params: + - name: url + value: $(params.repo_url) + - name: revision + value: $(params.revision) + # Ensure submodules (like .tests-automation) are fetched + - name: submodules + value: "true" + + + + # ------------------------------------------------ + # 2. RUN TEST & TAGGING & RELEASING + # ------------------------------------------------ + - name: integration-test + timeout: 10h0m0s # Timeout for the task + runAfter: + - fetch-repository + workspaces: + - name: source + workspace: source + - name: basic-auth + workspace: basic-auth # Needed for pushing tags/releases + - name: exploit-iq-data + workspace: exploit-iq-data + params: + - name: CURRENT_REVISION + value: $(params.revision) + - name: PR_NUMBER + value: $(params.pr_number) + - name: TRIGGER_COMMENT + value: $(params.trigger_comment) + + taskSpec: + params: + - name: CURRENT_REVISION + type: string + - name: PR_NUMBER + type: string + - name: TRIGGER_COMMENT + type: string + workspaces: + - name: source + - name: basic-auth + - name: exploit-iq-data + volumes: + - name: google-creds-volume + secret: + secretName: google-sheets-secrets + - name: otel-config-volume + configMap: + name: otel-collector-config + - name: openshift-service-ca + configMap: + name: openshift-service-ca.crt + items: + - key: service-ca.crt + path: service-ca.crt # Mounts as a file named service-ca.crt + + # >>> THE SERVER (Sidecar) <<< + sidecars: + - name: server-application + # [CHANGE 3] Use the image passed in params (the one built by the other pipeline) + image: $(params.target-image) + imagePullPolicy: Always + + resources: + requests: + cpu: "1000m" # CPU request (1 core) + memory: "12Gi" # Memory request (8 gigabytes) + limits: + cpu: "2000m" # CPU limit (2 cores) + memory: "32Gi" # Memory limit (16 gigabytes) + + volumeMounts: + - name: google-creds-volume + mountPath: /etc/secrets/google + readOnly: true + # Inject Env Vars (API Keys, Models, RedHat Creds) + - name: $(workspaces.exploit-iq-data.volume) + mountPath: /exploit-iq-data + - name: openshift-service-ca + mountPath: /app/certs + readOnly: true + envFrom: + - secretRef: + name: integration-tests + - configMapRef: + name: server-model-config + - secretRef: + name: redhat-app-creds + + env: + - name: GOTOOLCHAIN + value: "auto" + - name: PYTHONUNBUFFERED + value: "1" + - name: GOOGLE_SERVICE_ACCOUNT_FILE + value: "/etc/secrets/google/credentials.json" + - name: GSHEETS_INPUT_SHEET_ID + valueFrom: + secretKeyRef: + name: google-sheets-secrets + key: input_sheet_id + # Pass the raw comment text into the container + - name: TRIGGER_COMMENT + value: "$(params.TRIGGER_COMMENT)" + - name: GOMODCACHE + value: "/exploit-iq-data/go/pkg/mod" + - name: UV_CACHE_DIR + value: "/tmp/uv-cache" + - name: SERPAPI_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/serpapi + - name: CVE_DETAILS_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/cve-details + - name: CWE_DETAILS_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/cwe-details + - name: FIRST_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/first + - name: GHSA_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/ghsa + - name: NVD_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/nvd + - name: RHSA_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/rhsa + - name: DATA_DIR + value: $(workspaces.exploit-iq-data.path) + - name: OTEL_SERVICE_NAME + value: cve_agent + - name: OTEL_TRACES_ENDPOINT + value: http://localhost:4318/v1/traces + - name: JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH + value: $(workspaces.source.path)/kustomize/base/settings.xml + + script: | + #!/bin/sh + set -e + # We use python to safely update config + echo "--- Generating Config without Tracing ---" + python3 -c "import sys, yaml; c=yaml.safe_load(open('configs/config-http-openai.yml')); c['workflow'].pop('cve_output_config_name', None);c['functions']['cve_agent_executor']['cve_web_search_enabled'] = False;yaml.dump(c, open('configs/config-no-tracing.yml', 'w'))" + FETCHER_SCRIPT="ci/scripts/config_fetcher.py" + + # [CHANGE] Logic: Check if the comment contains "config=sheet" + # We use grep because the comment might be "/test-heavy config=sheet" + if echo "$TRIGGER_COMMENT" | grep -q "config=sheet"; then + echo ">> Trigger detected 'config=sheet'. using Dynamic Config." + pip install --user google-auth + if [ -f "$FETCHER_SCRIPT" ]; then + eval $(python "$FETCHER_SCRIPT") + else + echo "Error: Config fetcher script missing." + exit 1 + fi + else + echo ">> No custom config requested. Using Defaults." + fi + + CACHE_DIR_TARGET=".cache/am_cache" + CACHE_PVC_SOURCE="/exploit-iq-data" + + echo "--- Preparing Cache Link ---" + echo "Source PVC path: ${CACHE_PVC_SOURCE}" + echo "Target link path: $(pwd)/${CACHE_DIR_TARGET}" + + mkdir -p "$(dirname ${CACHE_DIR_TARGET})" + + # 2. [CRITICAL] Remove existing folder/file if the image created it + # This ensures 'ln' creates the link named 'am_cache', instead of putting it INSIDE 'am_cache' + if [ -e "${CACHE_DIR_TARGET}" ]; then + echo "Removing existing item at ${CACHE_DIR_TARGET} to replace with PVC link..." + rm -rf "${CACHE_DIR_TARGET}" + fi + + ln -s "${CACHE_PVC_SOURCE}" "${CACHE_DIR_TARGET}" + + echo "Cache link created successfully." + ls -ld "${CACHE_DIR_TARGET}" + + # Remove cached pickle files if REMOVE_CACHE_FILE is set (via Google Sheet config) + # e.g. REMOVE_CACHE_FILE=https.github.com.postgres.postgres + if [ -n "${REMOVE_CACHE_FILE:-}" ]; then + echo "--- Removing cache files matching: ${CACHE_DIR_TARGET}/pickle/${REMOVE_CACHE_FILE}* ---" + rm -fv "${CACHE_DIR_TARGET}/pickle/${REMOVE_CACHE_FILE}"* || echo " No matching files found." + fi + + # Copy with verbose error if it fails + echo "--- Exporting Telemetry Config ---" + cp -v configs/config-no-tracing.yml $DATA_DIR/config-no-tracing.yml || echo "Failed to copy config" + + echo "--- Exporting Env Vars ---" + # Explicit env var names to never export (e.g. from integration-tests, server-model-config, redhat-app-creds / .env) + ENV_BLACKLIST="GHSA_API_KEY NVD_API_KEY NVIDIA_API_KEY SERPAPI_API_KEY" + # Regex (case-insensitive) for key names to exclude: .password., .token., .api-key., .api_key., .secret. + ENV_KEY_PATTERN='password|token|api.key|api_key|api-key|secret|credential|private-key|private_key|auth|pwd|pass_|_pass|connection_string|connection-string|pat|certificate' + + : > $DATA_DIR/env.txt + env | while IFS= read -r line; do + key="${line%%=*}" + case " $ENV_BLACKLIST " in *" $key "*) continue ;; esac + echo "$key" | grep -qiE "$ENV_KEY_PATTERN" && continue + echo "$line" >> $DATA_DIR/env.txt + done + + echo "--- Starting 'nat' Server ---" + exec /tini -- nat --log-level debug serve --config_file=configs/config-no-tracing.yml --host 0.0.0.0 --port 26466 + readinessProbe: + tcpSocket: + port: 26466 + initialDelaySeconds: 10 + periodSeconds: 5 + - name: redpanda + image: docker.redpanda.com/redpandadata/redpanda:v23.2.1 + imagePullPolicy: IfNotPresent + script: | + #!/bin/bash + set -e + exec rpk redpanda start --smp=1 --memory=1Gi --overprovisioned --kafka-addr=internal://0.0.0.0:29092,external://0.0.0.0:9092 --advertise-kafka-addr=internal://localhost:29092,external://localhost:9092 + + resources: + requests: + memory: "1Gi" + cpu: "300m" + readinessProbe: + tcpSocket: + port: 9092 + initialDelaySeconds: 5 + + - name: otel-collector + image: otel/opentelemetry-collector-contrib:0.148.0 + # Mount your otel-config.yaml here via a ConfigMap + volumeMounts: + - name: otel-config-volume + mountPath: /etc/otelcol-contrib/ + - name: trace-collector + image: registry.access.redhat.com/ubi9/python-312:9.6 + volumeMounts: + - name: openshift-service-ca + mountPath: /app/certs + readOnly: true + workingDir: $(workspaces.source.path) + script: | + #!/bin/bash + set -e + pip install --no-cache-dir kafka-python-ng pydantic requests + echo "--- STARTING TRACE COLLECTOR ---" + python3 ci/scripts/collect_and_dispatch_traces.py --enable-verify + echo "--- DEBUG: Current Directory ---" + pwd + # >>> THE CLIENT (Test Runner) <<< + steps: + + # ------------------------------------------------------- + # STEP A: Tag the MAIN Repository (RUNS FIRST) + # ------------------------------------------------------- + - name: git-tag-main-repo + image: alpine/git + workingDir: $(workspaces.source.path) + script: | + #!/bin/sh + set -e + + # 1. Trust workspace + git config --global --add safe.directory $(workspaces.source.path) + + SHORT_HASH=$(echo $(params.CURRENT_REVISION) | cut -c1-7) + TAG_NAME="pr-$(params.PR_NUMBER)-${SHORT_HASH}" + + echo "--- Configuring Git for Main Repo ---" + git config --global user.email "tekton-bot@redhat.com" + git config --global user.name "Tekton Pipeline" + + # ------------------------------------------------------- + # AUTH LOGIC: PAC ONLY + # ------------------------------------------------------- + CRED_FILE="/workspace/basic-auth/.git-credentials" + if [ -f "$CRED_FILE" ]; then + echo "✅ Found PAC credentials. Configuring..." + + # [FIX 1] Use head to safely get the first line + RAW_CRED=$(head -n 1 "$CRED_FILE") + + # Extract 'user:token' + AUTH_PART=$(echo "$RAW_CRED" | sed -n 's|https://\(.*\)@.*|\1|p') + + if [ -z "$AUTH_PART" ]; then + echo "❌ Error parsing credentials." + exit 1 + fi + + # [FIX 2] Robust URL Cleaning + # Remove 'https://' AND any existing 'user:pass@' from the cloned URL + CURRENT_URL=$(git remote get-url origin) + CLEAN_URL=$(echo "$CURRENT_URL" | sed 's|https://||' | sed 's|.*@||') + + echo "Injecting credentials into git remote..." + git remote set-url origin "https://${AUTH_PART}@${CLEAN_URL}" + else + echo "❌ CRITICAL ERROR: .git-credentials file missing." + exit 1 + fi + + echo "--- Pushing Tag '$TAG_NAME' to Main Repo ---" + + # [FIX 3] Updated Message (Since tests haven't run yet) + git tag -f -a "$TAG_NAME" -m "Automated Build Tag for PR #$(params.PR_NUMBER)" + + # Push + git push origin "refs/tags/$TAG_NAME" --force + # ------------------------------------------------------- + # STEP B: Run the Python Automation Tests + # ------------------------------------------------------- + - name: run-test-suite + image: quay.io/ecosystem-appeng/auto-cm-testing:latest + imagePullPolicy: Always + workingDir: $(workspaces.source.path) + + volumeMounts: + - name: google-creds-volume + mountPath: /etc/secrets/google + readOnly: true + - name: openshift-service-ca + mountPath: /app/certs + readOnly: true + + env: + - name: GOOGLE_SERVICE_ACCOUNT_FILE + value: "/etc/secrets/google/credentials.json" + - name: GSHEETS_MODE + value: "both" + - name: GSHEETS_TAG + value: "Ver1" + - name: GSHEETS_INPUT_SHEET_ID + valueFrom: + secretKeyRef: + name: google-sheets-secrets + key: input_sheet_id + - name: GSHEETS_OUTPUT_SHEET_ID + valueFrom: + secretKeyRef: + name: google-sheets-secrets + key: output_sheet_id + - name: DATA_DIR + value: $(workspaces.exploit-iq-data.path) + + script: | + #!/bin/bash + set -e + + # 1. Construct the Tag Name + SHORT_HASH=$(echo $(params.CURRENT_REVISION) | cut -c1-7) + TAG_NAME="pr-$(params.PR_NUMBER)-${SHORT_HASH}" + + echo "--- STARTING AUTOMATION TESTS ---" + echo "Generated Tag: $TAG_NAME" + echo "--- DEBUG: Current Directory ---" + pwd + # prepare the mlops telemetry data + TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) + #copy the token to src/config/token.txt + echo $TOKEN > /app/src/config/token.txt + + if [ -f "$DATA_DIR/config-no-tracing.yml" ]; then + echo "--- Config file found in $DATA_DIR/config-no-tracing.yml ---" + else + echo "--- Config file not found in $DATA_DIR/config-no-tracing.yml ---" + exit 1 + fi + + # 2. Run Python script with the tag + cd /app/ + python3 src/vulnerability_main_automation.py --gsheets-tag "$TAG_NAME" --telemetry-mode true --agent-config-file $DATA_DIR/config-no-tracing.yml --agent-env-file $DATA_DIR/env.txt --agent-commit $(params.CURRENT_REVISION) --telemetry-tag $TAG_NAME --telemetry-verify-enable true + + # 2. [FIX] Copy reports to the Shared Workspace so the next step can see them + echo "--- Copying Reports to Shared Workspace ---" + # Ensure destination exists + mkdir -p $(workspaces.source.path)/src/reports + # Copy the archive folder from the container image (/app) to the PVC (/workspace) + cp -r /app/src/reports/archive $(workspaces.source.path)/src/reports + echo "--- TESTS FINISHED SUCCESSFULLY ---" + + + + # ------------------------------------------------------- + # STEP C: Release & Upload Artifacts to AUTOMATION Repo + # ------------------------------------------------------- + - name: release-to-automation-repo + image: alpine/git + workingDir: $(workspaces.source.path) + env: + - name: GH_TOKEN + valueFrom: + secretKeyRef: + name: exploit-iq-automation-token + key: gh-token + script: | + #!/bin/sh + set -e + apk add --no-cache github-cli + + # Define the target repo explicitly + TARGET_REPO="RHEcosystemAppEng/exploitiq-tests-automation" + + + # [CHANGE 2] Priority Check: Use the Env Var if it exists + if [ -n "$GH_TOKEN" ]; then + echo "✅ Using provided GH_TOKEN from secret 'exploit-iq-automation-token'." + else + echo "⚠️ GH_TOKEN env var is missing. Falling back to basic-auth workspace (Might fail for cross-repo)..." + if [ -f "/workspace/basic-auth/.git-credentials" ]; then + # Extract token from URL format https://user:token@host + export GH_TOKEN=$(cat /workspace/basic-auth/.git-credentials | sed -n 's/.*:\(.*\)@.*/\1/p') + elif [ -f "/workspace/basic-auth/password" ]; then + export GH_TOKEN=$(cat /workspace/basic-auth/password) + fi + fi + + # 2. Identify Tag and File + SHORT_HASH=$(echo $(params.CURRENT_REVISION) | cut -c1-7) + TAG_NAME="pr-$(params.PR_NUMBER)-${SHORT_HASH}" + + #[FIX] Point to the Shared Workspace path where Step A copied the files + # We look in src/reports relative to the workspace root + REPORT_PATH="src/reports/archive" + + echo "--- Searching for reports in: $(pwd)/$REPORT_PATH ---" + REPORT_FILE=$(ls -t "$REPORT_PATH"/*.tar 2>/dev/null | head -n 1) + + if [ -z "$REPORT_FILE" ]; then + echo "❌ Error: No report file found in $REPORT_PATH" + echo "Listing directory contents for debugging:" + ls -R src/ || echo "src directory not found" + exit 1 + fi + + echo "--- Found Report: $REPORT_FILE ---" + + # 2. Blind Delete (Simpler) + # We attempt to delete. If it fails (doesn't exist), we just continue. + # We redirect output to /dev/null so logs stay clean. + gh release delete "$TAG_NAME" \ + --repo "$TARGET_REPO" \ + --cleanup-tag \ + --yes >/dev/null 2>&1 || echo "No existing release to delete." + + echo "--- Creating Release '$TAG_NAME' on Automation Repo ---" + + echo "--- Creating release in $TARGET_REPO ---" + gh release create "$TAG_NAME" \ + "$REPORT_FILE" \ + --repo "$TARGET_REPO" \ + --title "Report: $TAG_NAME" \ + --notes "Automated Test Report for Main Repo PR #$(params.PR_NUMBER)" \ + --target "main" + + # ------------------------------------------------------- + # STEP D: Cleanup Old Releases (Keep last 5) + # ------------------------------------------------------- + echo "--- Running Cleanup (Keeping last 5 releases) ---" + + KEEP_COUNT=5 + + gh release list --limit 50 | \ + tail -n +$((KEEP_COUNT + 1)) | \ + cut -f3 | \ + while read -r OLD_TAG; do + if [ -n "$OLD_TAG" ]; then + echo "Deleting old release: $OLD_TAG" + gh release delete "$OLD_TAG" --cleanup-tag --yes + fi + done + + echo "--- Release Process Complete ---" + + finally: + - name: post-integration-evaluation + when: + - input: $(tasks.integration-test.status) + operator: in + values: + - Succeeded + - Failed + - input: $(tasks.eval-feature-gate.results.enabled) + operator: in + values: + - "true" + taskRef: + name: post-integration-evaluation + + # ------------------------------------------------ + # WORKSPACE BINDINGS + # ------------------------------------------------ + workspaces: + - name: source + volumeClaimTemplate: + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 2Gi + - name: basic-auth + secret: + secretName: "{{ git_auth_secret }}" + - name: exploit-iq-data + persistentVolumeClaim: + claimName: exploit-iq-cach-pvc diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml new file mode 100644 index 000000000..e72258311 --- /dev/null +++ b/.tekton/on-pull-request.yaml @@ -0,0 +1,589 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: vulnerability-analysis-on-pr + annotations: + # Trigger on non-draft PRs to target branches when specific paths are changed. + pipelinesascode.tekton.dev/on-cel-expression: | + event == "pull_request" && + !body.pull_request.draft && + (target_branch == "main" || target_branch == "rh-aiq-main") && + ("src/**".pathChanged() || "metrics_lib/**".pathChanged() || "pyproject.toml".pathChanged() || "uv.lock".pathChanged() || "Dockerfile".pathChanged() || ".dockerignore".pathChanged()) + # Fetch the git-clone task from hub, we are able to reference later on it + # with taskRef and it will automatically be embedded into our pipeline. + pipelinesascode.tekton.dev/task: "[git-clone, ./.tekton/post-integration-evaluation.yaml]" + + # How many runs we want to keep. + pipelinesascode.tekton.dev/max-keep-runs: "5" +spec: + timeouts: + pipeline: 2h30m0s # Timeout for the entire PipelineRun + taskRunTemplate: + podTemplate: + imagePullSecrets: + - name: ecosystem-appeng-morpheus-quay + params: + # The variable with brackets are special to Pipelines as Code + # They will automatically be expanded with the events from Github. + - name: repo_url + value: "{{ repo_url }}" + - name: revision + value: "{{ revision }}" + - name: image-expires-after + value: 5d + - name: output-image + value: quay.io/ecosystem-appeng/agent-morpheus-rh:on-pr-{{revision}} + - name: path-context + value: . + - name: dockerfile + value: ./Dockerfile + - name: pr_number + value: "{{ pull_request_number }}" + pipelineSpec: + params: + - name: repo_url + - name: revision + - name: output-image + description: Fully Qualified Output Image + type: string + - name: path-context + default: . + description: Path to the source code of an application's component from where to build image. + type: string + - name: dockerfile + default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter path-context + type: string + - name: image-expires-after + default: "" + description: Image tag expiration time, time values could be something like 1h, 2d, 3w for hours, days, and weeks, respectively. + workspaces: + - name: source + - name: basic-auth + - name: exploit-iq-data + tasks: + # Reads evaluation-config POST_INTEGRATION_EVAL_ENABLED (default: enabled if key absent). + # Ops: kubectl edit configmap evaluation-config -> set POST_INTEGRATION_EVAL_ENABLED: "false" + - name: eval-feature-gate + taskSpec: + results: + - name: enabled + type: string + steps: + - name: read-eval-toggle + image: registry.redhat.io/ubi9/ubi-micro:latest + env: + - name: POST_INTEGRATION_EVAL_ENABLED + valueFrom: + configMapKeyRef: + name: evaluation-config + key: POST_INTEGRATION_EVAL_ENABLED + optional: true + script: | + #!/bin/sh + set -eu + if [ "${POST_INTEGRATION_EVAL_ENABLED:-true}" = "true" ]; then + echo -n true | tee "$(results.enabled.path)" + else + echo -n false | tee "$(results.enabled.path)" + fi + echo "POST_INTEGRATION_EVAL_ENABLED=${POST_INTEGRATION_EVAL_ENABLED:-true} -> $(cat "$(results.enabled.path)")" + - name: fetch-repository + taskRef: + resolver: cluster + params: + - name: kind + value: task + - name: name + value: git-clone + - name: namespace + value: openshift-pipelines + workspaces: + - name: output + workspace: source + - name: basic-auth + workspace: basic-auth + params: + - name: URL + value: $(params.repo_url) + - name: REVISION + value: $(params.revision) + - name: DEPTH + value: "0" + - name: buildah-pvc + runAfter: + - lint-and-test + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: BUILDER_IMAGE + value: >- + registry.redhat.io/rhel8/buildah@sha256:6d2dcb651ba680cf4ec74331f8349dec43d071d420625a1703370acc8d984e9e + - name: STORAGE_DRIVER + value: overlay + - name: FORMAT + value: docker + - name: BUILD_EXTRA_ARGS + value: >- + --target base + --layers + --label=quay.expires-after=$(params.image-expires-after) + --label=exploit_iq_vcs_branch={{source_branch}} + --label=exploit_iq_vcs_build_commit_id={{revision}} + --label=exploit_iq_vcs_repo={{repo_url}} + --build-arg=AGENT_GIT_COMMIT={{revision}} + --format=docker + taskRef: + resolver: cluster + params: + - name: kind + value: task + - name: name + value: buildah-pvc + - name: namespace + value: exploit-iq-tests + workspaces: + - name: source + workspace: source + - name: dockerconfig + workspace: dockerconfig-ws + - name: buildah-storage-dir + workspace: buildah-storage-dir + - name: buildah-temp-cache + workspace: buildah-temp-cache + - name: lint-and-test + runAfter: + - fetch-repository + workspaces: + - name: source + workspace: source + - name: unit-test-cache + workspace: unit-test-cache + taskSpec: + workspaces: + - name: source + - name: unit-test-cache + steps: + - name: setup-cache-link + image: registry.redhat.io/ubi9/ubi-micro:latest + workingDir: $(workspaces.source.path) + script: | + #!/bin/bash + set -eou pipefail + + CACHE_DIR_TARGET=".cache/am_cache" + CACHE_PVC_SOURCE="$(workspaces.unit-test-cache.path)" + + echo "--- Preparing Cache Link ---" + echo "Source PVC path: ${CACHE_PVC_SOURCE}" + echo "Target link path: $(pwd)/${CACHE_DIR_TARGET}" + + mkdir -p "$(dirname ${CACHE_DIR_TARGET})" + ln -s "${CACHE_PVC_SOURCE}" "${CACHE_DIR_TARGET}" + + echo "Cache link created successfully." + ls -ld "${CACHE_DIR_TARGET}" + - name: lint-test + env: + - name: TARGET_BRANCH_NAME + value: "{{target_branch}}" + - name: JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH + value: $(workspaces.source.path)/kustomize/base/settings.xml + - name: UV_PYTHON_INSTALL_DIR + value: "/exploit-iq-data/python" + volumeMounts: + - name: $(workspaces.unit-test-cache.volume) + mountPath: /exploit-iq-data + image: registry.access.redhat.com/ubi9/ubi-minimal:9.8 + workingDir: $(workspaces.source.path) + script: | + #!/bin/bash + set -eou pipefail + + print_banner() { + echo + echo "----------- ${1} -----------" + } + + # ubi-minimal has no dev tools — install the essentials + print_banner "Installing system packages" + microdnf install -y git tar gzip findutils make gcc gcc-c++ krb5-devel bsdtar + microdnf module enable -y nodejs:20 + microdnf install -y nodejs npm + + # Mark the workspace as safe before any tool (uv, git, etc.) touches it. + git config --global --add safe.directory /workspace/source + + # Install uv + print_banner "Installing uv" + curl -LsSf https://astral.sh/uv/install.sh | sh + source $HOME/.local/bin/env + # Download Python 3.12 to the shared PVC before creating the venv. + # UV_PYTHON_INSTALL_DIR alone does not trigger a download during uv venv. + uv python install 3.12 + + print_banner "CREATING AND ACTIVATING TEST ENV" + uv venv --python 3.12 .venv + source .venv/bin/activate + + print_banner "INSTALLING DEPENDENCIES" + # Add the current directory to git's safe directories before uv sync, + # because setuptools-scm needs git access for version introspection. + git config --global --add safe.directory /workspace/source + uv sync + + # Install Java + JAVA_ARCH="x64" + JDK_URL="https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.18%2B8/OpenJDK17U-jdk_${JAVA_ARCH}_linux_hotspot_17.0.18_8.tar.gz" + JDK_DIR="jdk-17.0.18+8" + + echo ">> Downloading $JDK_URL" + mkdir -p /tekton/home/jdk + curl -fsSL -o /tekton/home/jdk/jdk.tgz "$JDK_URL" + tar -C /tekton/home/jdk -xzf /tekton/home/jdk/jdk.tgz + rm -f /tekton/home/jdk/jdk.tgz + + export JAVA_HOME="/tekton/home/jdk/${JDK_DIR}" + export PATH="$JAVA_HOME/bin:$PATH" + + echo "Java version:" + java -version || true + + # Install Go + print_banner "Installing Go" + + GO_VERSION="1.24.1" + ARCH="amd64" + curl -s -L -o go.tar.gz https://go.dev/dl/go${GO_VERSION}.linux-${ARCH}.tar.gz + mkdir -p $HOME/go-sdk + tar -C $HOME/go-sdk -xzf go.tar.gz + export PATH=$HOME/go-sdk/go/bin:$PATH + export GOTOOLCHAIN=auto + echo "Go version:" + go version + + # Install Maven + print_banner "Installing Maven" + + MAVEN_VERSION="3.9.11" + ARCHIVE="apache-maven-${MAVEN_VERSION}-bin.tar.gz" + URL="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/${ARCHIVE}" + + curl -s -L -o "${ARCHIVE}" "${URL}" + mkdir -p "$HOME/maven-sdk" + tar -C "$HOME/maven-sdk" -xzf "${ARCHIVE}" + + export MAVEN_HOME="$HOME/maven-sdk/apache-maven-${MAVEN_VERSION}" + export M2_HOME="$MAVEN_HOME" + export PATH="$MAVEN_HOME/bin:$PATH" + + echo "Maven version:" + mvn -v + + print_banner "RUNNING LINTER" + echo "Fetching target branch..." + git fetch origin $TARGET_BRANCH_NAME + echo "Git branches:" + git branch -a + # TODO: This step is temporarily configured to pass even if linting errors are found. + # This is handled in the Makefile's lint-pr target and should be reverted after migration. + make lint-pr TARGET_BRANCH=$TARGET_BRANCH_NAME + + print_banner "RUNNING UNIT TESTS" + make test-unit PYTEST_OPTS="--log-cli-level=DEBUG" + + print_banner "LINT AND TEST COMPLETE" + - name: integration-test + timeout: 10h0m0s # Timeout for the task + runAfter: + - buildah-pvc + workspaces: + - name: source + workspace: source + - name: basic-auth + workspace: basic-auth # Needed for pushing tags/releases + - name: exploit-iq-data + workspace: exploit-iq-data + - name: dockerconfig + workspace: dockerconfig-ws + params: + - name: CURRENT_REVISION + value: $(params.revision) + - name: PR_NUMBER + value: $(params.pr_number) + + taskSpec: + params: + - name: CURRENT_REVISION + type: string + - name: PR_NUMBER + type: string + workspaces: + - name: source + - name: basic-auth + - name: exploit-iq-data + volumes: + - name: otel-config-volume + configMap: + name: otel-collector-config + - name: openshift-service-ca + configMap: + name: openshift-service-ca.crt + items: + - key: service-ca.crt + path: service-ca.crt # Mounts as a file named service-ca.crt + + # >>> THE SERVER (Sidecar) <<< + sidecars: + - name: server-application + # [CHANGE 3] Use the image passed in params (the one built by the other pipeline) + image: $(params.output-image) + imagePullPolicy: IfNotPresent + + volumeMounts: + # Inject Env Vars (API Keys, Models, RedHat Creds) + - name: $(workspaces.exploit-iq-data.volume) + mountPath: /exploit-iq-data + - name: openshift-service-ca + mountPath: /app/certs + readOnly: true + envFrom: + - secretRef: + name: integration-tests + - configMapRef: + name: server-model-config + - secretRef: + name: redhat-app-creds + + env: + - name: PYTHONUNBUFFERED + value: "1" + - name: GOPROXY + value: https://proxy.golang.org,direct + # Pass the raw comment text into the container + - name: GOMODCACHE + value: "/exploit-iq-data/go/pkg/mod" + - name: UV_CACHE_DIR + value: "/tmp/uv-cache" + - name: UV_PYTHON_INSTALL_DIR + value: "/exploit-iq-data/python" + - name: SERPAPI_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/serpapi + - name: CVE_DETAILS_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/cve-details + - name: CWE_DETAILS_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/cwe-details + - name: FIRST_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/first + - name: GHSA_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/ghsa + - name: NVD_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/nvd + - name: RHSA_BASE_URL + value: http://nginx-cache.exploit-iq-tests.svc.cluster.local:8080/rhsa + - name: DATA_DIR + value: $(workspaces.exploit-iq-data.path) + - name: OTEL_SERVICE_NAME + value: cve_agent + - name: OTEL_TRACES_ENDPOINT + value: http://localhost:4318/v1/traces + - name: RPM_USER_TYPE + value: external + + script: | + #!/bin/sh + set -e + # We use python to: + #1. safely update config + #2. removing the cve_output_config_name function from the agent flow ( no need for callback, result will be returned to IT process which calls the agent. + #3. Making sure that Internet search tool is enabled ( part of the tests are relying on that). + echo "--- Generating Config without Tracing ---" + python3 -c "import sys, yaml; c=yaml.safe_load(open('configs/config-http-openai.yml')); c['workflow'].pop('cve_output_config_name', None);c['functions']['cve_agent_executor']['cve_web_search_enabled'] = True;yaml.dump(c, open('configs/config-no-tracing.yml', 'w'))" + + CACHE_DIR_TARGET=".cache/am_cache" + CACHE_PVC_SOURCE="/exploit-iq-data" + + echo "--- Preparing Cache Link ---" + echo "Source PVC path: ${CACHE_PVC_SOURCE}" + echo "Target link path: $(pwd)/${CACHE_DIR_TARGET}" + + mkdir -p "$(dirname ${CACHE_DIR_TARGET})" + + # 2. [CRITICAL] Remove existing folder/file if the image created it + # This ensures 'ln' creates the link named 'am_cache', instead of putting it INSIDE 'am_cache' + if [ -e "${CACHE_DIR_TARGET}" ]; then + echo "Removing existing item at ${CACHE_DIR_TARGET} to replace with PVC link..." + rm -rf "${CACHE_DIR_TARGET}" + fi + + ln -s "${CACHE_PVC_SOURCE}" "${CACHE_DIR_TARGET}" + + echo "Cache link created successfully." + ls -ld "${CACHE_DIR_TARGET}" + + # Copy with verbose error if it fails + + echo "--- Exporting Telemetry Config ---" + cp -v configs/config-no-tracing.yml $DATA_DIR/config-no-tracing.yml || echo "Failed to copy config" + + # 4. Safer Env Dump (blacklist only: explicit names + regex on key) + echo "--- Exporting Env Vars ---" + # Explicit env var names to never export (e.g. from integration-tests, server-model-config, redhat-app-creds / .env) + ENV_BLACKLIST="GHSA_API_KEY NVD_API_KEY NVIDIA_API_KEY SERPAPI_API_KEY" + # Regex (case-insensitive) for key names to exclude: .password., .token., .api-key., .api_key., .secret. + ENV_KEY_PATTERN='password|token|api.key|api_key|api-key|secret|credential|private-key|private_key|auth|pwd|pass_|_pass|connection_string|connection-string|pat|certificate' + : > $DATA_DIR/env.txt + env | while IFS= read -r line; do + key="${line%%=*}" + case " $ENV_BLACKLIST " in *" $key "*) continue ;; esac + echo "$key" | grep -qiE "$ENV_KEY_PATTERN" && continue + echo "$line" >> $DATA_DIR/env.txt + done + + echo "--- Starting 'nat' Server ---" + exec /tini -- nat --log-level debug serve --config_file=configs/config-no-tracing.yml --host 0.0.0.0 --port 26466 + readinessProbe: + tcpSocket: + port: 26466 + initialDelaySeconds: 10 + periodSeconds: 5 + - name: redpanda + image: docker.redpanda.com/redpandadata/redpanda:v23.2.1 + imagePullPolicy: IfNotPresent + script: | + #!/bin/bash + set -e + exec rpk redpanda start --smp=1 --memory=1Gi --overprovisioned --kafka-addr=internal://0.0.0.0:29092,external://0.0.0.0:9092 --advertise-kafka-addr=internal://localhost:29092,external://localhost:9092 + + resources: + requests: + memory: "1Gi" + cpu: "500m" + readinessProbe: + tcpSocket: + port: 9092 + initialDelaySeconds: 5 + + - name: otel-collector + image: otel/opentelemetry-collector-contrib:latest + # Mount your otel-config.yaml here via a ConfigMap + volumeMounts: + - name: otel-config-volume + mountPath: /etc/otelcol-contrib/ + - name: trace-collector + image: registry.access.redhat.com/ubi9/python-312:9.6 + volumeMounts: + - name: openshift-service-ca + mountPath: /app/certs + readOnly: true + workingDir: $(workspaces.source.path) + script: | + #!/bin/bash + set -e + pip install --no-cache-dir kafka-python-ng pydantic requests + echo "--- STARTING TRACE COLLECTOR ---" + python3 ci/scripts/collect_and_dispatch_traces.py --enable-verify + echo "--- DEBUG: Current Directory ---" + pwd + # >>> THE CLIENT (Test Runner) <<< + steps: + # ------------------------------------------------------- + # STEP A: Run the Integration Tests + # ------------------------------------------------------- + - name: run-test-suite + image: quay.io/ecosystem-appeng/auto-cm-testing:latest + imagePullPolicy: Always + volumeMounts: + - name: openshift-service-ca + mountPath: /app/certs + readOnly: true + workingDir: $(workspaces.source.path) + env: + - name: DATA_DIR + value: $(workspaces.exploit-iq-data.path) + script: | + #!/bin/bash + set -e + + echo "--- STARTING INTEGRATION TESTS ---" + echo "--- DEBUG: Current Directory ---" + pwd + echo " -- check if certificate is mounted" + ls -l /app/certs/service-ca.crt + # 2. Prepares the IT input for test + cp -f ci/it/integration-tests-input.json /app/src/input/scan_it.json + cd /app/ + # prepare the mlops telemetry data + TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) + #copy the token to src/config/token.txt + echo $TOKEN > src/config/token.txt + SHORT_HASH=$(echo $(params.CURRENT_REVISION) | cut -c1-7) + TAG_NAME="pr-$(params.PR_NUMBER)-${SHORT_HASH}" + + # 3. Run IT Script + python3 src/main_integration_tests.py --telemetry-mode true --agent-config-file $DATA_DIR/config-no-tracing.yml --agent-env-file $DATA_DIR/env.txt --agent-commit $(params.CURRENT_REVISION) --telemetry-tag $TAG_NAME --telemetry-verify-enable true + + echo "--- INTEGRATION TESTS FINISHED SUCCESSFULLY ---" + + finally: + - name: post-integration-evaluation + when: + - input: $(tasks.integration-test.status) + operator: in + values: + - Succeeded + - Failed + - input: $(tasks.eval-feature-gate.results.enabled) + operator: in + values: + - "true" + taskRef: + name: post-integration-evaluation + + workspaces: + - name: source + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + - name: buildah-storage-dir + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + + + - name: buildah-temp-cache + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + + - name: unit-test-cache + persistentVolumeClaim: + claimName: unit-test-shared-cache + # This workspace will inject secret to help the git-clone task to be able to + # checkout the private repositories + - name: basic-auth + secret: + secretName: "{{ git_auth_secret }}" + - name: dockerconfig-ws + secret: + secretName: ecosystem-appeng-morpheus-quay + - name: exploit-iq-data + persistentVolumeClaim: + claimName: unit-test-shared-cache diff --git a/.tekton/on-push.yaml b/.tekton/on-push.yaml new file mode 100644 index 000000000..5da2cc106 --- /dev/null +++ b/.tekton/on-push.yaml @@ -0,0 +1,154 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: vulnerability-analysis-on-push + annotations: + # The event we are targeting as seen from the webhook payload + # this can be an array too, i.e: [pull_request, push] + pipelinesascode.tekton.dev/on-event: "[push]" + + # The branch or tag we are targeting (ie: main, refs/tags/*) + pipelinesascode.tekton.dev/on-target-branch: "[main]" + + # Fetch the git-clone task from hub, we are able to reference later on it + # with taskRef and it will automatically be embedded into our pipeline. + pipelinesascode.tekton.dev/task: "git-clone" + + # How many runs we want to keep. + pipelinesascode.tekton.dev/max-keep-runs: "5" +spec: + params: + # The variable with brackets are special to Pipelines as Code + # They will automatically be expanded with the events from Github. + - name: repo_url + value: "{{ repo_url }}" + - name: revision + value: "{{ revision }}" + - name: output-image + value: quay.io/ecosystem-appeng/agent-morpheus-rh:latest + - name: path-context + value: . + - name: dockerfile + value: ./Dockerfile + pipelineSpec: + params: + - name: repo_url + - name: revision + - name: output-image + description: Fully Qualified Output Image + type: string + - name: path-context + default: . + description: Path to the source code of an application's component from where to build image. + type: string + - name: dockerfile + default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter path-context + type: string + workspaces: + - name: source + - name: basic-auth + tasks: + - name: fetch-repository + taskRef: + resolver: cluster + params: + - name: kind + value: task + - name: name + value: git-clone + - name: namespace + value: openshift-pipelines + workspaces: + - name: output + workspace: source + - name: basic-auth + workspace: basic-auth + params: + - name: URL + value: $(params.repo_url) + - name: REVISION + value: $(params.revision) + - name: buildah-pvc + runAfter: + - fetch-repository + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: BUILDER_IMAGE + value: >- + registry.redhat.io/rhel8/buildah@sha256:6d2dcb651ba680cf4ec74331f8349dec43d071d420625a1703370acc8d984e9e + - name: STORAGE_DRIVER + value: overlay + - name: FORMAT + value: docker + - name: BUILD_EXTRA_ARGS + value: >- + --target base + --layers + --label=exploit_iq_vcs_branch={{source_branch}} + --label=exploit_iq_vcs_build_commit_id={{revision}} + --label=exploit_iq_vcs_repo={{repo_url}} + --build-arg=AGENT_GIT_COMMIT={{revision}} + --format=docker + taskRef: + resolver: cluster + params: + - name: kind + value: task + - name: name + value: buildah-pvc + - name: namespace + value: exploit-iq-tests + workspaces: + - name: source + workspace: source + + - name: dockerconfig + workspace: dockerconfig-ws + + - name: buildah-storage-dir + workspace: buildah-storage-dir + + - name: buildah-temp-cache + workspace: buildah-temp-cache + + workspaces: + - name: source + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + - name: buildah-storage-dir + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + + - name: buildah-temp-cache + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + # This workspace will inject secret to help the git-clone task to be able to + # checkout the private repositories + - name: basic-auth + secret: + secretName: "{{ git_auth_secret }}" + - name: dockerconfig-ws + secret: + secretName: ecosystem-appeng-morpheus-quay diff --git a/.tekton/on-tag.yaml b/.tekton/on-tag.yaml new file mode 100644 index 000000000..08718fd32 --- /dev/null +++ b/.tekton/on-tag.yaml @@ -0,0 +1,185 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: vulnerability-analysis-on-tag + annotations: + # The event we are targeting as seen from the webhook payload + # this can be an array too, i.e: [pull_request, push] + pipelinesascode.tekton.dev/on-event: "[push]" + + # The branch or tag we are targeting (ie: main, refs/tags/*) + pipelinesascode.tekton.dev/on-target-branch: "[refs/tags/*]" + + # Fetch the git-clone task from hub, we are able to reference later on it + # with taskRef and it will automatically be embedded into our pipeline. + pipelinesascode.tekton.dev/task: "git-clone" + + # How many runs we want to keep. + pipelinesascode.tekton.dev/max-keep-runs: "5" +spec: + params: + # The variable with brackets are special to Pipelines as Code + # They will automatically be expanded with the events from Github. + - name: repo_url + value: "{{ repo_url }}" + - name: revision + value: "{{ revision }}" + - name: output-image + value: 'quay.io/ecosystem-appeng/agent-morpheus-rh' + - name: tag-name + value: "{{ target_branch }}" + - name: path-context + value: . + - name: dockerfile + value: ./Dockerfile + pipelineSpec: + params: + - name: repo_url + - name: revision + - name: tag-name + - name: output-image + description: Fully Qualified Output Image + type: string + - name: path-context + default: . + description: Path to the source code of an application's component from where to build image. + type: string + - name: dockerfile + default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter path-context + type: string + workspaces: + - name: source + - name: basic-auth + tasks: + - name: fetch-repository + taskRef: + resolver: cluster + params: + - name: kind + value: task + - name: name + value: git-clone + - name: namespace + value: openshift-pipelines + workspaces: + - name: output + workspace: source + - name: basic-auth + workspace: basic-auth + params: + - name: URL + value: $(params.repo_url) + - name: REVISION + value: $(params.revision) + - name: format-image-name + params: + - name: image_name + value: $(params.output-image) + - name: tag_name + value: $(params.tag-name) + taskSpec: + params: + - name: image_name + type: string + - name: tag_name + type: string + results: + - name: target-image + steps: + - name: format-image + image: registry.redhat.io/ubi9/ubi-minimal:9.4 + script: | + #!/usr/bin/env bash + IMAGE="$(params.image_name):$(echo $(params.tag_name) | sed 's|refs/tags/||')" + echo "Building image: $IMAGE" + + # Set the formatted image as an output + echo -n "${IMAGE}" > "$(results.target-image.path)" + - name: buildah-pvc + runAfter: + - fetch-repository + - format-image-name + params: + - name: IMAGE + value: $(tasks.format-image-name.results.target-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: BUILDER_IMAGE + value: >- + registry.redhat.io/rhel8/buildah@sha256:6d2dcb651ba680cf4ec74331f8349dec43d071d420625a1703370acc8d984e9e + - name: STORAGE_DRIVER + value: overlay + - name: FORMAT + value: docker + - name: BUILD_EXTRA_ARGS + value: >- + --target base + --layers + --label=exploit_iq_vcs_branch={{source_branch}} + --label=exploit_iq_vcs_build_commit_id={{revision}} + --label=exploit_iq_vcs_repo={{repo_url}} + --env=EXPLOIT_IQ_VERSION={{git_tag}} + --build-arg=AGENT_GIT_COMMIT={{revision}} + --build-arg=AGENT_GIT_TAG={{git_tag}} + --format=docker + taskRef: + resolver: cluster + params: + - name: kind + value: task + - name: name + value: buildah-pvc + - name: namespace + value: exploit-iq-tests + workspaces: + - name: source + workspace: source + + - name: dockerconfig + workspace: dockerconfig-ws + + - name: buildah-storage-dir + workspace: buildah-storage-dir + + - name: buildah-temp-cache + workspace: buildah-temp-cache + + workspaces: + - name: source + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + - name: buildah-storage-dir + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + + - name: buildah-temp-cache + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + + # This workspace will inject secret to help the git-clone task to be able to + # checkout the private repositories + - name: basic-auth + secret: + secretName: "{{ git_auth_secret }}" + - name: dockerconfig-ws + secret: + secretName: ecosystem-appeng-morpheus-quay diff --git a/.tekton/post-integration-evaluation.yaml b/.tekton/post-integration-evaluation.yaml new file mode 100644 index 000000000..e99754f2c --- /dev/null +++ b/.tekton/post-integration-evaluation.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: post-integration-evaluation +spec: + steps: + - name: report + image: quay.io/ecosystem-appeng/cve-evaluation:llama_prompt + env: + - name: EXPLOIT_IQ_API_BASE + valueFrom: + configMapKeyRef: + name: evaluation-config + key: EXPLOIT_IQ_API_BASE + - name: EXPLOIT_IQ_API_TOKEN + valueFrom: + secretKeyRef: + name: evaluation-secret + key: EXPLOIT_IQ_API_TOKEN + - name: NGC_API_KEY + valueFrom: + secretKeyRef: + name: evaluation-secret + key: NGC_API_KEY + - name: JUDGE_BASE_URL + valueFrom: + configMapKeyRef: + name: evaluation-config + key: JUDGE_BASE_URL + - name: JUDGE_MODEL + valueFrom: + configMapKeyRef: + name: evaluation-config + key: JUDGE_MODEL + - name: JUDGE_TEMPERATURE + valueFrom: + configMapKeyRef: + name: evaluation-config + key: JUDGE_TEMPERATURE + - name: JUDGE_TOP_P + valueFrom: + configMapKeyRef: + name: evaluation-config + key: JUDGE_TOP_P + args: + - --mode + - api + - --stages + - all + - --submit diff --git a/.tests-automation b/.tests-automation new file mode 160000 index 000000000..ae5e9806c --- /dev/null +++ b/.tests-automation @@ -0,0 +1 @@ +Subproject commit ae5e9806cd15a20a170550fe4c1b69d5d425435f diff --git a/Dockerfile b/Dockerfile index 0f104b425..45f492fc9 100755 --- a/Dockerfile +++ b/Dockerfile @@ -24,18 +24,75 @@ FROM ${BASE_IMAGE_URL}:${BASE_IMAGE_TAG} AS base COPY --from=ghcr.io/astral-sh/uv:0.7.15 /uv /uvx /bin/ ARG VULN_ANALYSIS_VERSION ARG PYTHON_VERSION +ARG AGENT_GIT_COMMIT +ARG AGENT_GIT_TAG ENV PYTHONDONTWRITEBYTECODE=1 +ENV AGENT_GIT_COMMIT=${AGENT_GIT_COMMIT} +ENV AGENT_GIT_TAG=${AGENT_GIT_TAG} + +# System dependencies: +# - build-essential: Required for compiling Python packages with native C extensions +# (e.g., psutil, cryptography, cffi) during `uv sync`. Also needed for libkrb5-dev. +# - libarchive-tools: Provides bsdtar for extracting RPM/SRPM archives in the checker +# - libkrb5-dev: Kerberos headers for gssapi Python package (Brew authentication) RUN apt-get update && apt-get install -y \ + build-essential \ ca-certificates \ curl \ git \ git-lfs \ wget \ + skopeo \ + libarchive-tools \ + xz-utils \ + libatomic1 \ + libkrb5-dev \ && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ && update-ca-certificates + +RUN curl -L -X GET https://go.dev/dl/go1.24.1.linux-amd64.tar.gz -o /tmp/go1.24.1.linux-amd64.tar.gz \ + && tar -C /usr/local -xzf /tmp/go1.24.1.linux-amd64.tar.gz \ + && rm /tmp/go1.24.1.linux-amd64.tar.gz + +ENV GOTOOLCHAIN=auto + +# --- Node.js 25.2.0 and npm 11.6.2 --- +ARG NODE_VERSION=25.2.0 +RUN curl -fsSL -o /tmp/node.tar.xz \ + "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz" \ + && mkdir -p /opt/nodejs \ + && tar -C /opt/nodejs -xJf /tmp/node.tar.xz --strip-components=1 \ + && rm -f /tmp/node.tar.xz +ENV PATH="/opt/nodejs/bin:${PATH}" + +# Verify Node.js and npm installation +RUN node --version && npm --version + +# --- Temurin JDK 22 (amd64/x86_64) --- +ARG JDK_URL="https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.18%2B8/OpenJDK17U-jdk_x64_linux_hotspot_17.0.18_8.tar.gz" +ARG JDK_DIR="jdk-17.0.18+8" +RUN mkdir -p /opt/jdk \ + && curl -fsSL -o /tmp/jdk.tgz "${JDK_URL}" \ + && tar -C /opt/jdk -xzf /tmp/jdk.tgz \ + && rm -f /tmp/jdk.tgz +ENV JAVA_HOME=/opt/jdk/${JDK_DIR} +ENV PATH="${JAVA_HOME}/bin:${PATH}" + +# --- Maven 3.9.11 (optional) --- +ARG MVN_VER=3.9.11 +RUN curl -fsSL -o /tmp/maven.tgz \ + "https://archive.apache.org/dist/maven/maven-3/${MVN_VER}/binaries/apache-maven-${MVN_VER}-bin.tar.gz" \ + && tar -C /opt -xzf /tmp/maven.tgz \ + && rm -f /tmp/maven.tgz +ENV PATH="/opt/apache-maven-${MVN_VER}/bin:${PATH}" + +# Verify +RUN java -version && mvn -v + # Set SSL environment variables ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt @@ -53,29 +110,43 @@ WORKDIR /workspace # Copy the project into the container COPY ./ /workspace +RUN groupadd workspace-group && \ + useradd -u 1001 -m -g workspace-group user1001 && \ + chown -R :workspace-group /home/user1001 && \ + chmod -R g+wrx /home/user1001 + + + # Install the NeMo Agent toolkit package and vuln analysis package -RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv,sharing=locked \ +RUN --mount=type=cache,id=uv_cache,target=/home/user1001/.cache/uv,mode=0775,sharing=locked \ export SETUPTOOLS_SCM_PRETEND_VERSION=${VULN_ANALYSIS_VERSION} && \ uv venv --python ${PYTHON_VERSION} /workspace/.venv && \ - uv sync --prerelease=allow + uv sync && \ + chown -R :workspace-group /workspace && \ + chmod -R g+wrx /workspace && \ + chown -R :workspace-group /root && \ + chmod -R g+rx /root -# Activate the environment (make it default for subsequent commands) -RUN echo "source /workspace/.venv/bin/activate" >> ~/.bashrc +USER 1001 +# Activate the environment (make it default for subsequent commands) # Enivronment variables for the venv -ENV PATH="/workspace/.venv/bin:$PATH" +ENV PATH="/workspace/.venv/bin:/usr/local/go/bin:$PATH" # Mark all git repos as safe to avoid git errors RUN echo $'\ [safe]\n\ directory = *\n\ -'> /root/.gitconfig +'> ~/.gitconfig + + +# Activate the environment (make it default for subsequent commands) +RUN echo "source /workspace/.venv/bin/activate" >> ~/.bashrc # ===== Setup for development ===== FROM base AS runtime - RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv,sharing=locked \ - source /workspace/.venv/bin/activate && \ - uv pip install "jupyterlab>=4.2.5,<5" + source /workspace/.venv/bin/activate + -CMD ["jupyter-lab", "--no-browser", "--allow-root", "--ip='*'", "--port=8000", "--NotebookApp.token=''", "--NotebookApp.password=''"] +CMD ["jupyter-lab", "--no-browser", "--allow-root", "--ip='*'", "--port=8000", "--NotebookApp.token=''", "--NotebookApp.password=''"] \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..cf7335721 --- /dev/null +++ b/Makefile @@ -0,0 +1,85 @@ +# Ensure a consistent shell is used, avoiding potential issues with environment inheritance. +# https://www.gnu.org/software/make/manual/make.html#Makefile-Basics +SHELL = /bin/sh +# Clear the list of suffixes to disable built-in implicit rules. +.SUFFIXES: + +# Directory for the main source code (override via `make lint-all SRC_DIR=./app`) +SRC_DIR ?= src +# Path to the LLM metrics test file +LLM_METRICS_TEST_FILE ?= tests/integration/generate_metrics.py + +# Target branch for PR diffs in CI (override via `make lint-pr TARGET_BRANCH=develop`) +TARGET_BRANCH ?= rh-aiq-main + +# Git remote for PR diffs in CI (override via `make lint-pr GIT_REMOTE=upstream`) +GIT_REMOTE ?= origin + +# The pylint executable (override via `make lint PYLINT=pylint3`) +PYLINT ?= pylint +# The pytest executable (override via `make test PYTEST=pytest-xdist`) +PYTEST ?= pytest +# Extra options for pylint (override via `make lint PYLINT_OPTS="--disable=C0114"`) +PYLINT_OPTS ?= +# Extra options for pytest (override via `make test PYTEST_OPTS="-v -s --maxfail=1"`) +PYTEST_OPTS ?= + +.PHONY: help lint lint-all lint-staged lint-pr test test-unit test-llm-metrics test-integration check + +# Set the default target to "help" +.DEFAULT_GOAL := help + +help: ## Display this help message + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | \ + awk 'BEGIN {FS = ":.*## "}; {printf "\t%-20s %s\n", $$1, $$2}' + +lint: lint-staged ## Lint staged files (default). + +lint-all: ## Lint all source files. + @echo "--- Linting all source files ---" + @$(PYLINT) $(PYLINT_OPTS) $(SRC_DIR) + +lint-staged: ## Lint staged files. + @echo "--- Linting staged files ---" + @{ \ + FILES_TO_LINT=$$(git diff --name-only --cached --diff-filter=ACMRTUXB -- '*.py' ':(exclude)*/__init__.py'); \ + if [ -z "$$FILES_TO_LINT" ]; then \ + echo "No Python files staged for commit. Skipping."; \ + else \ + echo "Linting files:"; \ + echo "$$FILES_TO_LINT" | sed 's/^/ /'; \ + $(PYLINT) $(PYLINT_OPTS) $$FILES_TO_LINT; \ + fi \ + } + +# TODO: During migration, linting errors are ignored to allow the pipeline to pass. +# Remove "|| true" from the pylint command below once the codebase is clean. +lint-pr: ## Lint PR changes (for CI). + @echo "--- Linting changed files in PR against $(GIT_REMOTE)/$(TARGET_BRANCH) ---" + @{ \ + FILES_TO_LINT=$$(git diff --name-only "$(GIT_REMOTE)/$(TARGET_BRANCH)..." -- '*.py' ':(exclude)*/__init__.py'); \ + if [ -z "$$FILES_TO_LINT" ]; then \ + echo "No changed Python files in this PR. Skipping."; \ + else \ + echo "Linting files:"; \ + echo "$$FILES_TO_LINT" | sed 's/^/ /'; \ + $(PYLINT) $(PYLINT_OPTS) $$FILES_TO_LINT || true; \ + fi \ + } + +test: test-unit test-llm-metrics ## Run all tests. + @echo "All tests have been run." + +test-unit: ## Run unit tests. + @echo "Running unit tests in $(SRC_DIR) and tests/..." + @python -m pytest $(SRC_DIR) tests/ $(PYTEST_OPTS) + +test-llm-metrics: ## Run LLM metrics tests. + @echo "Running LLM metrics tests..." + @python -m pytest $(LLM_METRICS_TEST_FILE) $(PYTEST_OPTS) + +test-integration: ## Run integration tests (placeholder). + @echo "--- SKIPPING: Integration tests are not yet implemented. ---" + +check: lint-all test ## Lint all files and run all tests. + @echo "Lint and all tests have been run." diff --git a/README.md b/README.md index aaf842b1d..4aff86137 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,8 @@ limitations under the License. - [Customizing the embedding model](#customizing-the-embedding-model) - [Steps to configure an alternate embedding provider](#steps-to-configure-an-alternate-embedding-provider) - [Customizing the Output](#customizing-the-output) +- [RPM CVE Checker](#rpm-cve-checker) +- [Limitations](#limitations) - [Troubleshooting](#troubleshooting) - [Git LFS issues](#git-lfs-issues) - [Container build issues](#container-build-issues) @@ -191,6 +193,7 @@ The current default embedding NIM model is `nv-embedqa-e5-v5`, which was selecte * [git](https://git-scm.com/) * [git-lfs](https://git-lfs.com/) +* bsdtar (from libarchive) - Used for extracting RPM source archives. Install via `apt install libarchive-tools` (Debian/Ubuntu) or `dnf install bsdtar` (Fedora/RHEL). * Since the workflow uses [NVIDIA NeMo Agent Toolkit](https://docs.nvidia.com/aiqtoolkit), the [NeMo Agent toolkit requirements](https://docs.nvidia.com/aiqtoolkit/latest/quick-start/installing.html#prerequisites) also need to be installed. ### Obtain API keys @@ -216,6 +219,15 @@ To run the workflow you need to obtain API keys for the following APIs. These wi - Click on your account in the top right, select "Setup" from the dropdown. - Click the "Generate Personal Key" option and then the "+ Generate Personal Key" button to create your API key. - This will be used in the `NVIDIA_API_KEY` environment variable. + - REDHAT container registry (Recommended but not compulsory) + - To get source images from a Red Hat container registry using registry service account tokens. You will need to create a [registry service account](https://access.redhat.com/terms-based-registry/) + - Steps: + - Sign in to the registry service accout + - Press on New service Account Button + - Fill the Name (ex. 'test-case') and Description fields and Click the Create botton + - Token user name for example '11008101|test-case' for REGISTRY_REDHAT_USERNAME environment variable + - Token password a long string for REGISTRY_REDHAT_PASSWORD environment variable + The workflow can be configured to use other LLM services as well, see the [Customizing the LLM models](#customizing-the-llm-models) section for more info. @@ -535,7 +547,8 @@ The configuration defines how the workflow operates, including functions, LLMs, - `base_code_index_dir`: The directory used for storing code index files. Default is `./cache/am_cache/code_index`. - `cve_fetch_intel`: Fetches details about CVEs from NIST and CVE Details websites. - `cve_process_sbom`: Prepares and validates input SBOM. - - `cve_check_vuln_deps`: Cross-references every entry in the SBOM for known vulnerabilities. + - `cve_check_vuln_deps`: Cross-references every entry in the SBOM for known vulnerabilities. If none are found, the agent is skipped for that CVE. + - `skip`: If true, skips this check and runs all CVEs through the agent. Useful when the input CVEs to the workflow have been validated by a dedicated vulnerability scanner. Default is false. - Core LLM engine functions: - `cve_checklist`: Generates tailored, context-sensitive task checklist for impact analysis. - `Container Image Code QA System`: Retriever tool used by `cve_agent_executor` to query source code vector database. @@ -552,6 +565,16 @@ The configuration defines how the workflow operates, including functions, LLMs, - `replace_exceptions_value`: If `replace_exceptions` is `true`, use this message. Default is `I do not have a definitive answer for this checklist item."` - `return_intermediate_steps`: Controls whether to return intermediate steps taken by the agent, and include them in the output file. Helpful for troubleshooting agent responses. Default is `false`. - `verbose`: Set to true for verbose output. Default is `false`. + - `cve_generate_cvss`: generates and calculates the CVSS (Common Vulnerability Scoring System) vector string and score for the vulnerability analysis. + - `llm_name`: Name of LLM (`generate_cvss_llm`) configured in `llms` section. + - `tool_names`: Container Image Code QA System, Container Image Developer Guide QA System, (Optional) Lexical Search Container Image Code QA System, Container Image Analysis Data + - `max_concurrency`: Controls the maximum number of concurrent requests to the LLM. Default is `null`, which doesn't limit concurrency. + - `max_iterations`: The maximum number of iterations for the agent. Default is 10. + - `prompt_examples`: Whether to include examples in agent prompt. Default is `true`. + - `replace_exceptions`: Whether to replace exception message with custom message. Default is `false`. + - `replace_exceptions_value`: If `replace_exceptions` is `true`, use this message. Default is `Failed to generate CVSS for this analysis."` + - `return_intermediate_steps`: Controls whether to return intermediate steps taken by the agent, and include them in the output file. Helpful for troubleshooting agent responses. Default is `false`. + - `verbose`: Set to true for verbose output. Default is `false`. - `cve_summarize`: Generates concise, human-readable summarization paragraph from agent results. - `llm_name`: Name of LLM (`summarize_llm`) configured in `llms` section. - `cve_justify`: Assigns justification label and reason to each CVE based on summary. @@ -562,7 +585,7 @@ The configuration defines how the workflow operates, including functions, LLMs, - `markdown_dir`: Defines the path to the directory where the output will be saved in individual navigable markdown files per CVE-ID. - `overwrite`: Indicates whether the output file should be overwritten when the workflow starts if it already exists. Will throw an error if set to `False` and the file already exists. Note that the overwrite behavior only occurs on workflow initialization. For pipelines started in HTTP mode, each new request will append the existing file until the workflow is restarted. 3. **LLMs** (`llms`): The `llms` section contains the LLMs used by the workflow. Functions can reference LLMs in this section to use. The supported LLM API types in NeMo Agent toolkit are `nim` and `openai` The models in this workflow use `nim`. - - Configured models in this workflow: `checklist_llm`, `code_vdb_retriever_llm`, `doc_vdb_retriever_llm`, `cve_agent_executor_llm`, `summarize_llm`, `justify_llm` + - Configured models in this workflow: `checklist_llm`, `code_vdb_retriever_llm`, `doc_vdb_retriever_llm`, `cve_agent_executor_llm`, `generate_cvss_llm`, `summarize_llm`, `justify_llm` - Each `nim` model is configured with the following attributes defined in the NeMo Agent toolkit's [NimModelConfig](https://docs.nvidia.com/aiqtoolkit/latest/api/aiq/llm/nim_llm/index.html#aiq.llm.nim_llm.NIMModelConfig). Use [OpenAIModelConfig](https://docs.nvidia.com/aiqtoolkit/latest/api/aiq/llm/openai_llm/index.html#aiq.llm.openai_llm.OpenAIModelConfig) for `openai` models. - `base_url`: Optional attribute to override `https://integrate.api.nvidia.com/v1` - `model_name`: The name of the LLM model used by the node. @@ -660,7 +683,7 @@ In any configuration file, locate the `llms` section to see the current settings #### Steps to configure an LLM model 1. Obtain an API key and any other required auth info for the selected service. -2. Update the `.env` file with the auth and base URL environment variables for the service as indicated in the [Supported LLM APIs](#supported-llm-apis) table. If you choose not to use the default LLM models in your workflow (`meta/llama-3.1-70b-instruct`), you can also add environment variables to override the model names to your `.env` file. In addition to `CHECKLIST_MODEL_NAME`, you can also set `model_name` for the other LLM models using `CODE_VDB_RETRIEVER_MODEL_NAME`, `DOC_VDB_RETRIEVER_MODEL_NAME`, `CVE_AGENT_EXECUTOR_MODEL_NAME`, `SUMMARIZE_MODEL_NAME`, and `JUSTIFY_MODEL_NAME`. +2. Update the `.env` file with the auth and base URL environment variables for the service as indicated in the [Supported LLM APIs](#supported-llm-apis) table. If you choose not to use the default LLM models in your workflow (`meta/llama-3.1-70b-instruct`), you can also add environment variables to override the model names to your `.env` file. In addition to `CHECKLIST_MODEL_NAME`, you can also set `model_name` for the other LLM models using `CODE_VDB_RETRIEVER_MODEL_NAME`, `DOC_VDB_RETRIEVER_MODEL_NAME`, `CVE_AGENT_EXECUTOR_MODEL_NAME`, `GENERATE_CVSS_MODEL_NAME`, `SUMMARIZE_MODEL_NAME`, and `JUSTIFY_MODEL_NAME`. 3. Update the config file as described above. For example, if you want to use OpenAI's `gpt-4o` model for checklist generation, update `checklist_llm` in the `llms` section to: ``` checklist_llm: @@ -750,6 +773,37 @@ The workflow can then be updated to use the new function: Additional output options will be added in the future. +## RPM CVE Checker + +Determine if an RPM package is vulnerable to a specific CVE by examining source patches, changelogs, and build artifacts. + +**When to use:** +- Investigating a specific CVE in a single RHEL or Fedora RPM package +- Need source-level evidence (patches, changelogs) for vulnerability status +- Targeted package analysis vs. container-wide scanning + +**Quick example:** +```json +{ + "pipeline_mode": "package_checker", + "target_package": { + "name": "libarchive", + "version": "3.1.2", + "release": "14.el7_9.2", + "arch": "x86_64", + "cve_id": "CVE-2026-5121" + } +} +``` + +For detailed usage, input/output formats, and verdict explanations, see [RPM CVE Checker Documentation](RPM-CVE-CHECKER.md). + +## Limitations + +The following limitations apply to this workflow: + +- **Python 2 Support**: Python 2 scripts and source code are not supported for analysis. The workflow only processes Python 3 code. Python 2 code will be automatically skipped during code repository processing. This limitation exists because Python 2 reached end-of-life in January 2020 and modern AST parsing libraries used by the workflow only support Python 3. + ## Troubleshooting Several common issues can arise when running the workflow. Here are some common issues and their solutions. diff --git a/RPM-CVE-CHECKER.md b/RPM-CVE-CHECKER.md new file mode 100644 index 000000000..0b245c963 --- /dev/null +++ b/RPM-CVE-CHECKER.md @@ -0,0 +1,246 @@ +# RPM CVE Checker + +Determine if an RPM package is vulnerable to a specific CVE by examining source patches, changelogs, and build artifacts. + +This mode provides targeted, single-package vulnerability analysis for RHEL and Fedora RPMs, producing detailed evidence-based verdicts. + +## Table of Contents + +- [Supported Distributions](#supported-distributions) +- [How It Works](#how-it-works) +- [Input Format](#input-format) +- [Configuration](#configuration) +- [Verdicts](#verdicts) +- [Output Format](#output-format) +- [Example](#example) +- [Limitations](#limitations) + +## Supported Distributions + +| Distribution | L1 Source Analysis | L2 Compilation Check | L2 Hardening Check | +|--------------|-------------------|---------------------|-------------------| +| **RHEL** | Full | Full | Full | +| **Fedora** | Full | Full | Not available | + +**RHEL** receives full analysis across both levels: +- L1: Source analysis (spec files, changelogs, patches) +- L2: Compilation check (detect disabled features) + hardening flag analysis (compiler protections) + +**Fedora** receives L1 + partial L2: +- L1: Source analysis (spec files, changelogs, patches) +- L2: Compilation check only (can detect if features like LDAP/OpenSSL are disabled via configure/makefiles) +- No hardening flag analysis (different build system, no Brew build logs) + +## How It Works + +The checker performs a two-level investigation to determine vulnerability status: + +```mermaid +flowchart TD + A["Input: CVE + RPM Package"] --> B["L1: Source Analysis"] + B --> C{"Backport found?"} + C -->|Yes| D["NOT_VULNERABLE"] + C -->|No| E["L2: Compilation Check"] + E --> F{"Code compiled?"} + F -->|No| D + F -->|Yes| G["L2: Hardening Check (RHEL only)"] + G --> H{"Hardening mitigates?"} + H -->|Yes| I["VULNERABLE_MITIGATED"] + H -->|No| J["VULNERABLE"] +``` + +### L1: Source Analysis + +Examines package source artifacts in three phases: + +1. **IDENTIFY** — Is this package affected by the CVE? (version range, architecture) +2. **LOCATE** — Is the vulnerable code present in the source tree? +3. **VERIFY** — Has a fix been applied? (backport patches, changelog entries, spec file references) + +### L2: Build Analysis + +When L1 finds no source-level fix: + +1. **Compilation Check** — Is the vulnerable code actually compiled into the binary? (feature flags, configure options) +2. **Hardening Check** (RHEL only) — Do compiler flags mitigate the vulnerability? (`-fstack-protector`, RELRO, PIE, etc.) + +## Input Format + +Provide the target package and CVE via JSON input with `pipeline_mode: "package_checker"`. + +### RHEL Example + +```json +{ + "pipeline_mode": "package_checker", + "target_package": { + "name": "libarchive", + "version": "3.1.2", + "release": "14.el7_9.2", + "arch": "x86_64", + "cve_id": "CVE-2026-5121" + } +} +``` + +### Fedora Example + +```json +{ + "pipeline_mode": "package_checker", + "target_package": { + "name": "openssl", + "version": "3.2.1", + "release": "1.fc40", + "arch": "x86_64", + "cve_id": "CVE-2024-2511" + } +} +``` + +### Required Fields + +| Field | Description | +|-------|-------------| +| `name` | RPM package name | +| `version` | Package version | +| `release` | Release string (e.g., `14.el7_9.2` for RHEL, `1.fc40` for Fedora) | +| `arch` | Architecture (e.g., `x86_64`) | +| `cve_id` | CVE identifier to investigate | + +## Configuration + +The RPM CVE Checker uses these configuration options in the workflow YAML: + +### Brew Profile (`rpm_user_type`) + +Controls which build system is used for fetching SRPMs and build logs: + +```yaml +cve_source_acquisition: + _type: cve_source_acquisition + rpm_user_type: ${RPM_USER_TYPE:-internal} + +cve_package_code_agent: + _type: cve_package_code_agent + rpm_user_type: ${RPM_USER_TYPE:-internal} +``` + +| Profile | Hub | Build Logs | Use Case | +|---------|-----|------------|----------| +| `internal` | Red Hat Brew (`brewhub.engineering.redhat.com`) | Auto-fetched | RHEL packages (requires VPN) | +| `external` | Fedora Koji (`koji.fedoraproject.org`) | Not fetched | Fedora packages (public access) | + +**Environment variable override:** Set `RPM_USER_TYPE=external` to override the config file value. + +### Cache Directories + +Configure where artifacts are stored: + +```yaml +cve_source_acquisition: + base_rpm_dir: .cache/am_cache/rpms # Shared SRPM cache + base_checker_dir: .cache/am_cache/checker # Checker-specific artifacts + +cve_checker_segmentation: + base_checker_dir: .cache/am_cache/checker + base_code_index_dir: .cache/am_cache/code_index # Tantivy search index +``` + +| Directory | Contents | +|-----------|----------| +| `base_rpm_dir` | Downloaded SRPMs (shared cache) | +| `base_checker_dir` | Extracted sources, build logs, patches, reports | +| `base_code_index_dir` | Tantivy full-text search index for source code | + +### Profile Details + +**Internal profile** (`internal-user-profile.yml`): +- Connects to Red Hat Brew hub +- Fetches build logs automatically (`build_log.auto_fetch: true`) +- Enables L2 hardening check via build log analysis +- Requires Red Hat VPN connection + +**External profile** (`external-user-profile.yml`): +- Connects to Fedora Koji hub +- Does not fetch build logs (`build_log.auto_fetch: false`) +- L2 hardening check unavailable (no build log) +- Public access, no authentication required + +For other configuration options (LLM models, output paths), see the main [Configuration file reference](README.md#configuration-file-reference). + +## Verdicts + +The checker produces one of three verdicts with a justification label explaining the reasoning: + +| Verdict | Justification Label | When | +|---------|---------------------|------| +| `NOT_VULNERABLE` | `code_not_present` | Package version outside affected range, architecture not applicable, or vulnerable code not compiled (feature disabled) | +| `NOT_VULNERABLE` | `protected_by_mitigating_control` | Downstream backport patch applied, confirmed in changelog/spec/build log | +| `VULNERABLE_MITIGATED` | `protected_by_compiler` | No source fix, but compiler hardening flags mitigate the exploit mechanism (RHEL only) | +| `VULNERABLE` | `vulnerable` | Package is affected, no fix or relevant mitigation found | + +Each verdict includes a **confidence score** (0-100%) indicating the strength of evidence. + +## Output Format + +The checker produces a structured report with these sections: + +### Executive Summary +One-paragraph assessment of vulnerability status with key findings. + +### Verdict and Justification +- `justification_label`: The classification (see Verdicts table) +- `confidence`: Percentage confidence in the verdict +- Status: `TRUE` (vulnerable) or `FALSE` (not vulnerable) + +### Evidence Chain +Bullet-point list of evidence supporting the verdict: +- Patch files found/not found +- Changelog mentions +- Build log confirmations +- Version comparisons + +### Extracted Facts +Verbatim excerpts from package artifacts: +- Spec `PatchN:` lines matching the CVE +- `%changelog` entries mentioning the CVE +- Build log lines showing patch application + +### Code Snippets +When available, shows vulnerable code and fix code side-by-side. + +## Example + +**Input:** CVE-2026-5121 in `libarchive-3.1.2-14.el7_9.2` + +**Verdict:** `NOT_VULNERABLE` (`protected_by_mitigating_control`, 90% confidence) + +**Executive Summary:** +The package is protected by a downstream patch that mitigates the vulnerability. The fix is carried as a separate %patch directive. + +**Evidence Chain:** +- Downstream patch file `libarchive-3.3.3-Fix-CVE-2026-5121.patch` found and applied +- Spec file contains: `Patch33: %{name}-3.3.3-Fix-CVE-2026-5121.patch` +- Changelog contains: `Security fix for CVE-2026-4424 CVE-2026-5121` +- Build log confirms: `Patch #33 (libarchive-3.3.3-Fix-CVE-2026-5121.patch)` + +**Affected File:** `libarchive/archive_read_support_format_iso9660.c` + +**Fix Applied:** Added bounds check to prevent integer overflow in zisofs block pointer allocation: +```c +if (file->pz_log2_bs < 15 || file->pz_log2_bs > 17) { + file->pz = 0; + return; +} +``` + +## Limitations + +1. **Fedora: No hardening check** — L2 compilation check works (can detect disabled features), but no compiler flag analysis due to different build system. + +2. **Requires source access** — Needs SRPM via Brew (RHEL) or Koji (Fedora). Packages without accessible source cannot be fully analyzed. + +3. **LLM-based analysis** — Verdicts are generated using LLM reasoning and should be reviewed by security analysts before action. + +4. **Single-package scope** — Designed for targeted investigation of one CVE in one package. For container-wide scanning, use the main workflow. diff --git a/SECURITY.md b/SECURITY.md index 351809626..e9348936c 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,4 +1,21 @@ - ## Security + + +## Security NVIDIA is dedicated to the security and trust of our software products and services, including all source code repositories managed through our organization. diff --git a/ci/it/integration-tests-input.json b/ci/it/integration-tests-input.json new file mode 100644 index 000000000..99238da9a --- /dev/null +++ b/ci/it/integration-tests-input.json @@ -0,0 +1,213 @@ +{ + "origin": "Agent Repository", + "iterations": 1, + "tests": [ + { + "language": "go", + "vuln_id": "CVE-2024-51744", + "git": { + "repo": "https://github.com/openshift/assisted-installer", + "ref": "ab9e2ade2f45890033a140b314ef87e367317b51" + }, + "use_sbom": false, + "expected_label": "code_not_reachable", + "expected_result": "Not Exploitable", + "allowed_deviation_labels" : ["code_not_reachable", "code_not_present"], + "skip": false + }, + { + "language": "c", + "vuln_id": "CVE-2025-1094", + "image": { + "name": "registry.redhat.io/rhel8/postgresql-13", + "tag": "1-196.1724180180" + }, + "git": { + "repo": "https://github.com/postgres/postgres", + "ref": "REL_13_14" + }, + "use_sbom": true, + "sbom_file": "sboms/postgresql-13-1-196.1724180180.sbom", + "expected_label": "vulnerable", + "expected_result": "Exploitable", + "allowed_deviation_labels" : ["vulnerable"], + "skip": false + }, + { + "language": "go", + "vuln_id": "CVE-2025-22865", + "git": { + "repo": "https://github.com/kuadrant/authorino", + "ref": "f792cd138891dc1ead99fd089aa757fbca3aace9" + }, + "use_sbom": false, + "expected_label": "false_positive", + "expected_result": "Not Exploitable", + "allowed_deviation_labels" : ["false_positive"], + "skip": false + }, + { + "language": "python", + "vuln_id": "CVE-2024-49767", + "git": { + "repo": "https://github.com/TamarW0/example-repo", + "ref": "714b9fc466d1f17b032a6d40707ee5a143cfc1ab" + }, + "use_sbom": false, + "expected_label": "vulnerable", + "expected_result": "Exploitable", + "allowed_deviation_labels" : ["vulnerable"], + "skip": false + }, + { + "language": "go", + "vuln_id": "CVE-2024-28180", + "git": { + "repo": "https://github.com/openshift/oc-mirror", + "ref": "9621d8f72ecc7a0a13e40b9709b5e19cc621117b" + }, + "use_sbom": false, + "expected_label": "code_not_reachable", + "expected_result": "Not Exploitable", + "allowed_deviation_labels" : ["code_not_reachable", "code_not_present","protected_by_mitigating_control"], + "skip": true + }, + { + "language": "java", + "vuln_id": "CVE-2025-48734", + "git": { + "repo": "https://github.com/cryostatio/cryostat", + "ref": "v4.0.1" + }, + "use_sbom": false, + "expected_label": "code_not_reachable", + "expected_result": "Not Exploitable", + "allowed_deviation_labels" : ["code_not_present","code_not_reachable"], + "skip": true + }, + { + "language": "javascript", + "vuln_id": "CVE-2021-23369", + "git": { + "repo": "https://github.com/TamarW0/node-example-project", + "ref": "c0223de250a6c06b5bab660f3c7057a53df8921a" + }, + "use_sbom": false, + "expected_label": "vulnerable", + "expected_result": "Exploitable", + "allowed_deviation_labels" : ["vulnerable"], + "skip": false + }, + { + "language": "javascript", + "vuln_id": "CVE-2019-10744", + "git": { + "repo": "https://github.com/TamarW0/node-example-project", + "ref": "aec80a8feef62f25ff8ff200b0d1ad2244b2a1e0" + }, + "use_sbom": false, + "expected_label": "false_positive", + "expected_result": "Not Exploitable", + "allowed_deviation_labels" : ["code_not_reachable", "code_not_present","false_positive"], + "skip": false + }, + { + "language": "javascript", + "vuln_id": "CVE-2019-10744", + "git": { + "repo": "https://github.com/TamarW0/node-example-project", + "ref": "551e89592bf42c4d5d5dd82009abc7f5b4e0c601" + }, + "use_sbom": false, + "expected_label": "code_not_present", + "expected_result": "Not Exploitable", + "allowed_deviation_labels" : ["code_not_reachable", "code_not_present","false_positive"], + "skip": false + }, + { + "scan": { "vulns": [{ "vuln_id": "CVE-2026-5121" }] }, + "image": { + "analysis_type": "source", + "pipeline_mode": "rpm_package_checker", + "target_package": { + "name": "libarchive", + "version": "3.1.2", + "release": "14.el7_9.2", + "arch": "x86_64" + } + }, + "expected_label": "protected_by_mitigating_control", + "expected_result": "Not Exploitable", + "allowed_deviation_labels": ["protected_by_mitigating_control"], + "skip": true + }, + { + "scan": { "vulns": [{ "vuln_id": "CVE-2024-2511" }] }, + "image": { + "analysis_type": "source", + "pipeline_mode": "rpm_package_checker", + "target_package": { + "name": "openssl", + "version": "3.5.1", + "release": "7.el9_7", + "arch": "x86_64" + } + }, + "expected_label": "protected_by_mitigating_control", + "expected_result": "Not Exploitable", + "allowed_deviation_labels": ["protected_by_mitigating_control"], + "skip": true + }, + { + "scan": { "vulns": [{ "vuln_id": "CVE-2024-2511" }] }, + "image": { + "analysis_type": "source", + "pipeline_mode": "rpm_package_checker", + "target_package": { + "name": "openssl", + "version": "3.2.1", + "release": "1.fc40", + "arch": "x86_64" + } + }, + "expected_label": "vulnerable", + "expected_result": "Exploitable", + "allowed_deviation_labels": ["vulnerable"], + "skip": false + }, + { + "scan": { "vulns": [{ "vuln_id": "CVE-2024-48957" }] }, + "image": { + "analysis_type": "source", + "pipeline_mode": "rpm_package_checker", + "target_package": { + "name": "libarchive", + "version": "3.7.2", + "release": "4.fc40", + "arch": "x86_64" + } + }, + "expected_label": "vulnerable", + "expected_result": "Exploitable", + "allowed_deviation_labels": ["vulnerable"], + "skip": false + }, + { + "scan": { "vulns": [{ "vuln_id": "CVE-2024-48957" }] }, + "image": { + "analysis_type": "source", + "pipeline_mode": "rpm_package_checker", + "target_package": { + "name": "libarchive", + "version": "3.7.2", + "release": "7.fc40", + "arch": "x86_64" + } + }, + "expected_label": "protected_by_mitigating_control", + "expected_result": "Not Exploitable", + "allowed_deviation_labels": ["protected_by_mitigating_control"], + "skip": false + } + ] +} diff --git a/ci/scripts/HttpProvider.py b/ci/scripts/HttpProvider.py new file mode 100644 index 000000000..2a9b2370e --- /dev/null +++ b/ci/scripts/HttpProvider.py @@ -0,0 +1,169 @@ +""" +HTTP provider for dispatching payloads to a configurable endpoint with optional Bearer auth. +""" +import json +import time +import typing +from enum import Enum +from http import HTTPStatus +from pathlib import Path + +import requests +from urllib3.util.url import parse_url + +# Default dispatch endpoint configuration +DEFAULT_URL = "https://exploit-iq-client.exploit-iq-tests.svc.cluster.local:8443" +DEFAULT_AUTH_TYPE = "Bearer" +DEFAULT_TOKEN_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/token" +DEFAULT_VERIFY_PATH = "/app/certs/service-ca.crt" +DEFAULT_ENABLE_VERIFY = False + + +class AuthType(str, Enum): + BEARER = "bearer" + NONE = "none" + + +class HttpProvider: + """ + Sends HTTP POST requests to a configurable endpoint with optional Bearer token auth + and retry logic. + """ + + def __init__( + self, + url: str = DEFAULT_URL, + auth_type: AuthType = AuthType.NONE, + token_path: str = DEFAULT_TOKEN_PATH, + verify_path: str = DEFAULT_VERIFY_PATH, + enable_verify: bool = DEFAULT_ENABLE_VERIFY, + ): + self.url = self._prepare_url(url) + self.auth_type = auth_type + self.token_path = token_path + self.verify_path = verify_path + self.enable_verify = enable_verify + self.session = requests.Session() + self.headers = self._build_headers() + + def send_post( + self, + json_data: str, + endpoint_url: str | None = None, + ) -> HTTPStatus: + """ + Send a POST request with JSON body to the endpoint. + + Args: + json_data: JSON string body to send. + endpoint_url: Override URL for this request; if None, uses the instance URL. + + Returns: + HTTP status code of the response, or INTERNAL_SERVER_ERROR on failure. + """ + target_url = endpoint_url if endpoint_url is not None else self.url + if self.enable_verify: + verify = self.verify_path + else: + verify = False + try: + _, response = self._request_with_retry( + request_kwargs={ + "url": target_url, + "method": "POST", + "data": json_data.encode("utf-8"), + "headers": self.headers, + "verify": verify, + }, + requests_session=self.session, + accept_status_codes=(HTTPStatus.OK, HTTPStatus.CREATED, HTTPStatus.ACCEPTED), + ) + return HTTPStatus(response.status_code) + except Exception as e: + print("Unable to send output response to %s. Error: %s", target_url, e) + return HTTPStatus.INTERNAL_SERVER_ERROR + + def _build_headers(self) -> dict[str, str]: + """Build the headers for the request.""" + if self.auth_type == AuthType.BEARER: + return { + "Content-Type": "application/json", + "Authorization": f"Bearer {self._get_token()}", + "Bypass-Referential-Integrity-Check": "true", + } + return {"Content-Type": "application/json", "Bypass-Referential-Integrity-Check": "true"} + + def _get_token(self) -> str: + """Read Bearer token from token_path.""" + with open(self.token_path, encoding="utf-8") as f: + return f.read().strip() + + def _prepare_url(self, url: str) -> str: + """ + Verifies that `url` contains a protocol scheme and a host and returns the url. + If no protocol scheme is provided, `http` is used. + """ + parsed = parse_url(url) + if parsed.scheme is None or parsed.host is None: + url = f"http://{url}" + parsed = parse_url(url) + if parsed.scheme is None or parsed.host is None: + raise ValueError(f"Invalid URL: {url}") + return url + + def _request_with_retry( + self, + request_kwargs: dict, + requests_session: requests.Session | None = None, + max_retries: int = 10, + sleep_time: float = 0.1, + accept_status_codes: typing.Iterable[int] = (HTTPStatus.OK,), + respect_retry_after_header: bool = True, + on_success_fn: typing.Callable[[requests.Response], typing.Any] | None = None, + ) -> tuple[requests.Session, requests.Response | typing.Any]: + """ + Wrapper around requests.request that retries on failure. + """ + try_count = 0 + codes = tuple(accept_status_codes) + while try_count <= max_retries: + if requests_session is None: + requests_session = requests.Session() + retry_after_header: int | None = None + try: + response = requests_session.request(**request_kwargs) + if response.status_code in codes: + if on_success_fn is not None: + return (requests_session, on_success_fn(response)) + return (requests_session, response) + if respect_retry_after_header and "Retry-After" in response.headers: + retry_after_header = int(response.headers["Retry-After"]) + raise RuntimeError( + f"Received unexpected status code {response.status_code}: {response.text}" + ) + except Exception as e: + if isinstance(e, requests.exceptions.RequestException): + try: + requests_session.close() + finally: + requests_session = None + try_count += 1 + if try_count > max_retries: + print("Failed after %s retries: %s", max_retries, e) + raise + actual_sleep = ( + retry_after_header + if retry_after_header is not None + else (2 ** (try_count - 1)) * sleep_time + ) + print( + "Error performing %s to %s: %s", + request_kwargs.get("method"), + request_kwargs.get("url"), + e, + ) + print("Sleeping %s seconds before retry", actual_sleep) + time.sleep(actual_sleep) + raise RuntimeError("Unreachable") + + diff --git a/ci/scripts/analyze_traces.py b/ci/scripts/analyze_traces.py new file mode 100644 index 000000000..81d51d6a0 --- /dev/null +++ b/ci/scripts/analyze_traces.py @@ -0,0 +1,762 @@ +""" +Trace analyzer for LLM reasoning quality. + +Reads per-trace JSON files produced by collect_traces_local.py, +reconstructs agent investigation flows, and flags quality issues. + +Usage: + python ci/scripts/analyze_traces.py --input traces_output/ + python ci/scripts/analyze_traces.py --input traces_output/ --json + python ci/scripts/analyze_traces.py --input traces_output/single_trace.json +""" +import argparse +import json +import sys +from collections import Counter +from dataclasses import dataclass, field, asdict +from enum import Enum +from pathlib import Path + + +REACHABILITY_TOOLS = {"Function Locator", "Call Chain Analyzer", "Function Caller Finder"} +SEARCH_ONLY_TOOLS = {"Code Keyword Search", "Code Semantic Search", "Docs Semantic Search"} +NODE_FUNCTION_NAMES = {"thought node", "observation node", "pre_process node", "forced_finish node"} +TOKEN_BUDGET_THRESHOLD = 6000 # 75% of 8k window +TOKEN_BUDGET_WARNING = 5800 # early warning before hitting threshold +CONTEXT_WINDOW_LIMIT = 8192 +MAX_COMPLETION_TOKENS = 2000 + + +class Severity(str, Enum): + HIGH = "HIGH" + MEDIUM = "MEDIUM" + LOW = "LOW" + + +@dataclass +class QualityFlag: + name: str + severity: Severity + detail: str + + +@dataclass +class InvestigationStep: + step_num: int + span_kind: str # LLM, TOOL, or FUNCTION + name: str + input_preview: str + output_preview: str + token_prompt: int = 0 + token_completion: int = 0 + token_total: int = 0 + llm_role: str = "" # "thought" or "observation" for LLM spans + thought_mode: str = "" # "act" or "finish" for thought spans + has_error: bool = False + error_detail: str = "" + + +@dataclass +class InvestigationFlow: + """One agent investigation thread (one checklist question).""" + trace_id: str + job_id: str + parent_span_id: str + steps: list[InvestigationStep] = field(default_factory=list) + flags: list[QualityFlag] = field(default_factory=list) + tools_used: list[str] = field(default_factory=list) + llm_call_count: int = 0 + tool_call_count: int = 0 + token_progression: list[int] = field(default_factory=list) + cve_id: str = "" + ecosystem: str = "" + question: str = "" + is_reachability: str = "" + + +def _preview(text: str, max_len: int = 120) -> str: + if not text: + return "" + clean = text.replace("\n", " ").strip() + return clean[:max_len] + "..." if len(clean) > max_len else clean + + +def _extract_cve_from_input(input_value: str) -> tuple[str, str]: + """Try to extract CVE ID and ecosystem from an agent executor's LLM input.""" + cve_id = "" + ecosystem = "" + try: + messages = json.loads(input_value) + for msg in messages: + content = msg.get("content", "") if isinstance(msg, dict) else "" + if "CVE-" in content: + import re + match = re.search(r"(CVE-\d{4}-\d+)", content) + if match: + cve_id = match.group(1) + if "ecosystem" in content.lower(): + for eco in ("go", "python", "java", "javascript", "c"): + if f'"{eco}"' in content.lower() or f"'{eco}'" in content.lower(): + ecosystem = eco + break + except (json.JSONDecodeError, TypeError): + pass + return cve_id, ecosystem + + +def _extract_question(input_value: str) -> str: + """Extract the human/checklist question from the first LLM call's input.""" + try: + messages = json.loads(input_value) + for msg in messages: + if isinstance(msg, dict) and msg.get("type") == "human": + return _preview(msg.get("content", ""), 200) + except (json.JSONDecodeError, TypeError): + pass + return "" + + +# --------------------------------------------------------------------------- +# Parsing +# --------------------------------------------------------------------------- + +def load_trace_files(input_path: str) -> list[dict]: + """Load trace documents from a file or directory.""" + p = Path(input_path) + traces = [] + if p.is_file(): + with open(p, "r", encoding="utf-8") as f: + traces.append(json.load(f)) + elif p.is_dir(): + for fp in sorted(p.glob("*.json")): + with open(fp, "r", encoding="utf-8") as f: + try: + traces.append(json.load(f)) + except json.JSONDecodeError as e: + print(f"Warning: skipping {fp}: {e}", file=sys.stderr) + else: + print(f"Error: {input_path} is not a file or directory", file=sys.stderr) + sys.exit(1) + return traces + + +def _check_span_error(span: dict) -> tuple[bool, str]: + """Check if a span's output_value contains an error recorded by try/except tracing.""" + output = span.get("output_value", "") + if not output: + return False, "" + try: + parsed = json.loads(output) + if isinstance(parsed, dict) and "error" in parsed: + exc_type = parsed.get("exception_type", "Exception") + error_msg = parsed["error"] + return True, f"{exc_type}: {error_msg}" + except (json.JSONDecodeError, TypeError): + pass + return False, "" + + +def _classify_llm_span(span: dict) -> str: + """Classify an LLM span as 'thought' or 'observation' based on output format.""" + output = span.get("output_value", "") + stripped = output.lstrip() + if stripped.startswith('{"thought"') or stripped.startswith('{"thought'): + return "thought" + return "observation" + + +def _extract_thought_mode(output: str) -> str: + """Extract the mode ('act' or 'finish') from a thought-node JSON output.""" + try: + parsed = json.loads(output) + return parsed.get("mode", "") + except (json.JSONDecodeError, TypeError): + pass + import re + m = re.search(r'"mode"\s*:\s*"(act|finish)"', output) + return m.group(1) if m else "" + + +def _extract_goal_question(input_value: str) -> str: + """Extract the GOAL question from an observation-node input.""" + try: + messages = json.loads(input_value) + for msg in messages: + content = msg.get("content", "") if isinstance(msg, dict) else "" + if "GOAL:" in content: + goal_line = content.split("GOAL:")[1].split("\n")[0].strip() + return _preview(goal_line, 200) + except (json.JSONDecodeError, TypeError): + pass + return "" + + +def _split_by_question(spans: list[dict]) -> dict[str, list[dict]]: + """ + Split interleaved agent spans into per-question groups. + + Concurrent checklist questions share a parent_span_id. We separate them + by extracting the question from thought-node (human message) and + observation-node (GOAL field) LLM spans. TOOL spans are assigned to the + question whose thought span most recently preceded them. + """ + spans_sorted = sorted(spans, key=lambda s: int(s.get("start_time") or 0)) + + questions_seen: list[str] = [] + for span in spans_sorted: + if span.get("span_kind") == "LLM" and _classify_llm_span(span) == "thought": + q = _extract_question(span.get("input_value", "")) + if q and q not in questions_seen: + questions_seen.append(q) + + if len(questions_seen) <= 1: + return {} + + groups: dict[str, list[dict]] = {q: [] for q in questions_seen} + + last_thought_question: str | None = None + + for span in spans_sorted: + kind = span.get("span_kind", "") + + if kind == "LLM": + role = _classify_llm_span(span) + if role == "thought": + q = _extract_question(span.get("input_value", "")) + if q in groups: + last_thought_question = q + groups[q].append(span) + elif role == "observation": + goal_q = _extract_goal_question(span.get("input_value", "")) + matched = None + if goal_q: + for known_q in questions_seen: + if goal_q[:60] in known_q or known_q[:60] in goal_q: + matched = known_q + break + if matched: + groups[matched].append(span) + elif last_thought_question: + groups[last_thought_question].append(span) + + elif kind in ("TOOL", "FUNCTION"): + if last_thought_question: + groups[last_thought_question].append(span) + + return groups + + +def _build_flow_from_spans( + group_spans: list[dict], + trace_id: str, + job_id: str, + parent_id: str, +) -> InvestigationFlow: + """Build a single InvestigationFlow from an ordered list of spans.""" + group_spans.sort(key=lambda s: int(s.get("start_time") or 0)) + + flow = InvestigationFlow( + trace_id=trace_id, + job_id=job_id, + parent_span_id=parent_id, + ) + + for i, span in enumerate(group_spans): + kind = span.get("span_kind", "") + span_name = span.get("name", "") + role = "" + mode = "" + if kind == "LLM": + role = _classify_llm_span(span) + if role == "thought": + mode = _extract_thought_mode(span.get("output_value", "")) + elif kind == "FUNCTION" and span_name == "thought node": + role = "thought" + mode = _extract_thought_mode(span.get("output_value", "")) + + has_error, error_detail = _check_span_error(span) + + step = InvestigationStep( + step_num=i + 1, + span_kind=kind, + name=span_name, + input_preview=_preview(span.get("input_value", "")), + output_preview=_preview(span.get("output_value", "")), + token_prompt=span.get("token_prompt", 0), + token_completion=span.get("token_completion", 0), + token_total=span.get("token_total", 0), + llm_role=role, + thought_mode=mode, + has_error=has_error, + error_detail=error_detail, + ) + flow.steps.append(step) + + if kind == "LLM": + flow.llm_call_count += 1 + if step.token_prompt > 0: + flow.token_progression.append(step.token_prompt) + if i == 0: + cve_id, eco = _extract_cve_from_input(span.get("input_value", "")) + flow.cve_id = cve_id + flow.ecosystem = eco + flow.question = _extract_question(span.get("input_value", "")) + elif kind == "FUNCTION" and span_name == "thought node": + flow.llm_call_count += 1 + if step.token_prompt > 0: + flow.token_progression.append(step.token_prompt) + elif kind == "TOOL": + flow.tool_call_count += 1 + flow.tools_used.append(span_name) + elif kind == "FUNCTION" and span_name == "pre_process node": + try: + output = json.loads(span.get("output_value", "{}")) + if isinstance(output, dict) and "reachability_question" in output: + flow.is_reachability = output["reachability_question"] + except (json.JSONDecodeError, TypeError): + pass + + return flow + + +def _build_flows_from_workflow_output( + spans: list[dict], + trace_id: str, + job_id: str, +) -> list[InvestigationFlow]: + """Fallback: build flows from the span's output when agent-level + spans (thought node, pre_process node, etc.) are absent. + + This handles the ``cve_justify`` pipeline variant where the agent's + internal LangGraph spans are not exported individually. + """ + workflow_span = next( + (s for s in spans if s.get("function_name") == "" and s.get("span_kind") == "FUNCTION"), + None, + ) + if not workflow_span: + return [] + + output_raw = workflow_span.get("output_value", "") + try: + output = json.loads(output_raw) + except (json.JSONDecodeError, TypeError): + return [] + + analyses = (output.get("output", {}) or {}).get("analysis", []) + if not analyses: + return [] + + input_raw = workflow_span.get("input_value", "") + ecosystem = "" + try: + inp = json.loads(input_raw) + ecosystem = (inp.get("image", {}) or {}).get("ecosystem", "") + except (json.JSONDecodeError, TypeError): + pass + + llm_spans = [s for s in spans if s.get("span_kind") == "LLM"] + total_prompt = sum(s.get("token_prompt", 0) for s in llm_spans) + total_completion = sum(s.get("token_completion", 0) for s in llm_spans) + + flows: list[InvestigationFlow] = [] + for analysis in analyses: + vuln_id = analysis.get("vuln_id", "") + checklist = analysis.get("checklist", []) + justification = analysis.get("justification", {}) or {} + label = justification.get("label", "") + reason = justification.get("reason", "") + + flow = InvestigationFlow( + trace_id=trace_id, + job_id=job_id, + parent_span_id=workflow_span.get("span_id", ""), + cve_id=vuln_id, + ecosystem=ecosystem, + ) + + for i, item in enumerate(checklist): + question = item.get("input", "") + response = item.get("response", "") + flow.steps.append(InvestigationStep( + step_num=i + 1, + span_kind="SUMMARY", + name="checklist_qa", + input_preview=_preview(question, 200), + output_preview=_preview(response, 200), + )) + + flow.steps.append(InvestigationStep( + step_num=len(checklist) + 1, + span_kind="SUMMARY", + name="justification", + input_preview=f"label={label}", + output_preview=_preview(reason, 200), + )) + + flow.llm_call_count = len(llm_spans) + if total_prompt > 0: + flow.token_progression.append(total_prompt) + + flows.append(flow) + + return flows + + +def build_investigation_flows(trace_doc: dict) -> list[InvestigationFlow]: + """ + From a single trace document, reconstruct per-question investigation flows. + + When concurrent checklist questions share one parent_span_id (LangGraph), + we split by extracting the human question from each thought-node span. + Falls back to parent_span_id grouping when questions aren't interleaved. + """ + trace_id = trace_doc.get("trace_id", "") + job_id = trace_doc.get("job_id", "") + spans = trace_doc.get("spans", []) + + llm_tokens_by_parent: dict[str, dict[str, int]] = {} + for s in spans: + if ( + s.get("span_kind") == "LLM" + and s.get("function_name") in NODE_FUNCTION_NAMES + ): + pid = s.get("parent_span_id", "") + if pid and s.get("token_prompt", 0) > 0: + llm_tokens_by_parent[pid] = { + "token_prompt": s.get("token_prompt", 0), + "token_completion": s.get("token_completion", 0), + "token_total": s.get("token_total", 0), + } + + agent_spans = [ + s for s in spans + if ( + s.get("function_name") in ("cve_agent_executor", "checklist_question") + and s.get("span_kind") in ("LLM", "TOOL") + ) or ( + s.get("function_name") in NODE_FUNCTION_NAMES + and s.get("span_kind") == "FUNCTION" + ) + ] + + for s in agent_spans: + sid = s.get("span_id", "") + if sid in llm_tokens_by_parent: + s.update(llm_tokens_by_parent[sid]) + + parent_groups: dict[str, list[dict]] = {} + for span in agent_spans: + pid = span.get("parent_span_id", "unknown") + parent_groups.setdefault(pid, []).append(span) + + flows = [] + for parent_id, group_spans in parent_groups.items(): + question_groups = _split_by_question(group_spans) + + if question_groups: + for q_text, q_spans in question_groups.items(): + if q_spans: + flow = _build_flow_from_spans(q_spans, trace_id, job_id, parent_id) + if not flow.question: + flow.question = q_text + flows.append(flow) + else: + flow = _build_flow_from_spans(group_spans, trace_id, job_id, parent_id) + flows.append(flow) + + if not flows: + flows = _build_flows_from_workflow_output(spans, trace_id, job_id) + + return flows + + +# --------------------------------------------------------------------------- +# Quality checks +# --------------------------------------------------------------------------- + +def run_quality_checks(flow: InvestigationFlow) -> list[QualityFlag]: + flags = [] + + is_summary_only = all(s.span_kind == "SUMMARY" for s in flow.steps) if flow.steps else False + if is_summary_only: + return flags + + error_steps = [s for s in flow.steps if s.has_error] + for es in error_steps: + flags.append(QualityFlag( + name="NODE_EXCEPTION", + severity=Severity.HIGH, + detail=f"Exception in {es.name} (step {es.step_num}): {es.error_detail}", + )) + + tool_set = set(flow.tools_used) + + is_reachability_question = flow.is_reachability != "no" + + if not tool_set: + flags.append(QualityFlag( + name="NO_TOOL_CALLS", + severity=Severity.HIGH, + detail="Agent concluded without any tool calls.", + )) + elif is_reachability_question and not tool_set & REACHABILITY_TOOLS: + flags.append(QualityFlag( + name="NO_REACHABILITY_TOOL", + severity=Severity.HIGH, + detail=f"No reachability tool used. Tools: {', '.join(tool_set)}", + )) + + if is_reachability_question and tool_set and tool_set <= SEARCH_ONLY_TOOLS: + flags.append(QualityFlag( + name="KEYWORD_SEARCH_ONLY", + severity=Severity.HIGH, + detail="Only search tools used, never traced call chain.", + )) + + if flow.llm_call_count <= 2 and flow.tool_call_count > 0: + flags.append(QualityFlag( + name="PREMATURE_FINISH", + severity=Severity.MEDIUM, + detail=f"Agent finished in {flow.llm_call_count} LLM iterations.", + )) + + budget_flagged = False + for tok in flow.token_progression: + if tok > TOKEN_BUDGET_THRESHOLD: + flags.append(QualityFlag( + name="TOKEN_BUDGET_HIGH", + severity=Severity.MEDIUM, + detail=f"LLM call used {tok} prompt tokens (>{TOKEN_BUDGET_THRESHOLD}).", + )) + budget_flagged = True + break + if tok > TOKEN_BUDGET_WARNING: + flags.append(QualityFlag( + name="TOKEN_BUDGET_APPROACHING", + severity=Severity.LOW, + detail=f"LLM call used {tok} prompt tokens, approaching {TOKEN_BUDGET_THRESHOLD} limit.", + )) + budget_flagged = True + break + + if "Function Caller Finder" in tool_set and flow.ecosystem and flow.ecosystem != "go": + flags.append(QualityFlag( + name="WRONG_TOOL_ECOSYSTEM", + severity=Severity.HIGH, + detail=f"Function Caller Finder used on {flow.ecosystem} (Go-only tool).", + )) + + seen_tool_calls: set[tuple[str, str]] = set() + duplicate_count = 0 + for step in flow.steps: + if step.span_kind == "TOOL": + key = (step.name, step.input_preview) + if key in seen_tool_calls: + duplicate_count += 1 + else: + seen_tool_calls.add(key) + if duplicate_count > 0: + flags.append(QualityFlag( + name="DUPLICATE_TOOL_CALL", + severity=Severity.MEDIUM, + detail=f"{duplicate_count} duplicate tool call(s) with identical inputs.", + )) + + thought_steps = [s for s in flow.steps if s.llm_role == "thought"] + if thought_steps: + last_thought = thought_steps[-1] + ended_with_finish = last_thought.thought_mode == "finish" + peak_tokens = max(flow.token_progression) if flow.token_progression else 0 + available_prompt_space = CONTEXT_WINDOW_LIMIT - MAX_COMPLETION_TOKENS + + if not ended_with_finish and last_thought.thought_mode == "act": + if peak_tokens > available_prompt_space * 0.65: + flags.append(QualityFlag( + name="CONTEXT_WINDOW_EXCEEDED", + severity=Severity.HIGH, + detail=( + f"Flow ended mid-investigation (last thought: mode='act', peak tokens: {peak_tokens}, " + f"available prompt space: {available_prompt_space}). " + f"Next LLM call likely exceeded {CONTEXT_WINDOW_LIMIT} context window." + ), + )) + else: + flags.append(QualityFlag( + name="INCOMPLETE_FLOW", + severity=Severity.HIGH, + detail=( + f"Flow ended without mode='finish' (last thought: mode='act', " + f"peak tokens: {peak_tokens}). Possible unrecorded exception." + ), + )) + + return flags + + +# --------------------------------------------------------------------------- +# Report +# --------------------------------------------------------------------------- + +def print_flow_report(flow: InvestigationFlow): + is_summary = all(s.span_kind == "SUMMARY" for s in flow.steps) if flow.steps else False + tag = " [SUMMARY]" if is_summary else "" + header = f"=== Trace: {flow.trace_id} | Parent: {flow.parent_span_id[:12]}...{tag} ===" + print(header) + if flow.cve_id or flow.ecosystem: + print(f"CVE: {flow.cve_id or 'unknown'} | Ecosystem: {flow.ecosystem or 'unknown'}") + if flow.question: + print(f"Question: {flow.question}") + print(f"Steps: {flow.llm_call_count} LLM calls, {flow.tool_call_count} tool calls") + + if flow.token_progression: + tokens_str = " -> ".join(str(t) for t in flow.token_progression) + print(f"Token usage: {tokens_str}") + + if flow.tools_used: + tool_counts = Counter(flow.tools_used) + tools_str = ", ".join(f"{name} (x{count})" for name, count in tool_counts.items()) + print(f"Tools used: {tools_str}") + reachability_used = set(flow.tools_used) & REACHABILITY_TOOLS + if reachability_used: + print(f"Reachability tools: YES ({', '.join(reachability_used)})") + else: + print("Reachability tools: NO") + else: + print("Tools used: NONE") + + if flow.flags: + flag_strs = [f"[{f.severity.value}] {f.name}: {f.detail}" for f in flow.flags] + print(f"Flags: {len(flow.flags)}") + for fs in flag_strs: + print(f" {fs}") + else: + print("Flags: NONE") + + print() + for step in flow.steps: + error_marker = " *** ERROR ***" if step.has_error else "" + if step.span_kind == "LLM": + role_label = f" [{step.llm_role}]" if step.llm_role else "" + mode_label = f" mode={step.thought_mode}" if step.thought_mode else "" + print(f"--- Step {step.step_num}: LLM{role_label}{mode_label}{error_marker} ---") + if step.token_prompt: + print(f" Prompt tokens: {step.token_prompt} | Completion: {step.token_completion}") + print(f" Output: \"{step.output_preview}\"") + elif step.span_kind == "TOOL": + print(f"--- Step {step.step_num}: TOOL ({step.name}){error_marker} ---") + print(f" Input: {step.input_preview}") + print(f" Output: \"{step.output_preview}\"") + elif step.span_kind == "SUMMARY": + print(f"--- Step {step.step_num}: {step.name} ---") + print(f" Q: {step.input_preview}") + print(f" A: \"{step.output_preview}\"") + elif step.span_kind == "FUNCTION": + print(f"--- Step {step.step_num}: NODE ({step.name}){error_marker} ---") + if step.has_error: + print(f" Exception: {step.error_detail}") + else: + print(f" Output: \"{step.output_preview}\"") + + print() + + +def print_summary(all_flows: list[InvestigationFlow]): + total = len(all_flows) + flagged = [f for f in all_flows if f.flags] + passed = total - len(flagged) + + print("=" * 60) + print("SUMMARY") + print("=" * 60) + print(f"Total investigation flows: {total}") + print(f"Passed (no flags): {passed}") + print(f"Flagged: {len(flagged)}") + + if flagged: + all_flag_counts: Counter = Counter() + for flow in flagged: + for flag in flow.flags: + all_flag_counts[flag.name] += 1 + for flag_name, count in all_flag_counts.most_common(): + print(f" - {flag_name}: {count}") + + if all_flows: + avg_llm = sum(f.llm_call_count for f in all_flows) / total + avg_tool = sum(f.tool_call_count for f in all_flows) / total + all_tokens = [t for f in all_flows for t in f.token_progression] + avg_tokens = sum(all_tokens) / len(all_tokens) if all_tokens else 0 + print(f"\nAvg LLM calls per flow: {avg_llm:.1f}") + print(f"Avg tool calls per flow: {avg_tool:.1f}") + print(f"Avg prompt tokens per LLM call: {avg_tokens:.0f}") + + print() + + +def build_json_report(all_flows: list[InvestigationFlow]) -> dict: + flows_data = [] + for flow in all_flows: + is_summary = all(s.span_kind == "SUMMARY" for s in flow.steps) if flow.steps else False + flows_data.append({ + "trace_id": flow.trace_id, + "job_id": flow.job_id, + "cve_id": flow.cve_id, + "ecosystem": flow.ecosystem, + "question": flow.question, + "summary_only": is_summary, + "llm_call_count": flow.llm_call_count, + "tool_call_count": flow.tool_call_count, + "token_progression": flow.token_progression, + "tools_used": flow.tools_used, + "flags": [asdict(f) for f in flow.flags], + "steps": [asdict(s) for s in flow.steps], + }) + + total = len(all_flows) + flagged_count = sum(1 for f in all_flows if f.flags) + return { + "total_flows": total, + "passed": total - flagged_count, + "flagged": flagged_count, + "flows": flows_data, + } + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser(description="Analyze agent trace files for LLM reasoning quality.") + parser.add_argument("--input", required=True, + help="Path to traces_output/ directory or a single trace JSON file") + parser.add_argument("--json", action="store_true", + help="Output structured JSON report instead of console text") + parser.add_argument("--summary-only", action="store_true", + help="Print only the summary, skip per-flow details") + args = parser.parse_args() + + trace_docs = load_trace_files(args.input) + if not trace_docs: + print("No trace files found.", file=sys.stderr) + sys.exit(1) + + all_flows: list[InvestigationFlow] = [] + for doc in trace_docs: + flows = build_investigation_flows(doc) + for flow in flows: + flow.flags = run_quality_checks(flow) + all_flows.extend(flows) + + if args.json: + report = build_json_report(all_flows) + print(json.dumps(report, indent=2)) + else: + if not args.summary_only: + for flow in all_flows: + print_flow_report(flow) + print_summary(all_flows) + + +if __name__ == "__main__": + main() diff --git a/ci/scripts/collect_and_dispatch_traces.py b/ci/scripts/collect_and_dispatch_traces.py new file mode 100755 index 000000000..bbf8723cc --- /dev/null +++ b/ci/scripts/collect_and_dispatch_traces.py @@ -0,0 +1,292 @@ +from kafka import KafkaConsumer +import argparse +import json +from HttpProvider import HttpProvider, AuthType +from http import HTTPStatus +from datetime import datetime +from typing import Any +from pydantic import BaseModel, Field, constr, RootModel +import signal +import time +from kafka.errors import NoBrokersAvailable +# Define trace class +TRACE_VERSION = 1 + +# Default dispatch endpoint configuration +DEFAULT_URL = "https://exploit-iq-client.exploit-iq-tests.svc.cluster.local:8443" +DEFAULT_AUTH_TYPE = "Bearer" +DEFAULT_TOKEN_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/token" +DEFAULT_VERIFY_PATH = "/app/certs/service-ca.crt" +DEFAULT_ENABLE_VERIFY = False + +# Tempo configuration +TEMPO_URL = "http://tempo.grafana-tempo-poc.svc.cluster.local:4318/v1/traces" + +class LocalDateTime(RootModel[datetime]): + root: datetime = Field(..., examples=['2022-03-10T12:15:50']) + +class Trace(BaseModel): + format_version: int | None = None + job_id: constr(min_length=1) = Field( + ..., description='The job run id in which the trace and spans were instrumented' + ) + execution_start_timestamp: LocalDateTime | None = None + trace_id: constr(min_length=1) = Field( + ..., + description='The id corresponding to one agent stage, grouping underneath all spans of sub-tasks embedded in the agent stage', + ) + span_id: constr(min_length=1) = Field( + ..., description='The id corresponding to one sub task inside one agent stage.' + ) + span_payload: dict[str, Any] | None = Field( + None, + description='free style span payload without restrictions, containing all span metadata and istrumentation data', + ) + +# wrapper around jobs +class TracesContainer(RootModel[list[Trace]]): + pass + +# --- CONFIGURATION --- +KAFKA_TOPIC = 'otel-traces' +KAFKA_BROKER = 'localhost:9092' +OUTPUT_FILE = 'cve_research_log.jsonl' + +def truncate_value(value, max_length=100): + """Recursively truncate string values in OTLP attribute value objects.""" + if isinstance(value, dict): + # Handle OTLP value objects like {"stringValue": "...", "intValue": 123} + truncated = {} + for key, val in value.items(): + if key.endswith('Value') and isinstance(val, str): + # Truncate string values + truncated[key] = val[:max_length] if len(val) > max_length else val + else: + truncated[key] = truncate_value(val, max_length) + return truncated + elif isinstance(value, str): + # Direct string value + return value[:max_length] if len(value) > max_length else value + elif isinstance(value, list): + # Handle lists + return [truncate_value(item, max_length) for item in value] + else: + # Numbers, booleans, None - return as-is + return value + + +def get_metadata(span: dict) -> dict: + return { + 'trace_id': span.get('traceId'), + 'span_id': span.get('spanId'), + 'parent_span_id': span.get('parentSpanId'), + 'name': span.get('name'), + 'start_time': span.get('startTimeUnixNano'), + 'end_time': span.get('endTimeUnixNano'), + 'status': span.get('status'), + } + +def find_job_id(attributes: dict) -> str: + try: + input_value = attributes.get("input.value") + str_json = input_value.get("stringValue") + scan = json.loads(str_json) + if "scan" in scan: + return scan["scan"]["id"] + if "input" in scan: + return scan["input"]["scan"]["id"] + else: + print(f"Warning: Scan not found in attributes:") + return "" + except Exception as e: + print(f"Error finding job id (scan id): {e}") + print(f"Attributes: {attributes}") + return "" + +def process_message(http_provider, tempo_provider, traces_table, map_trace_id_to_job_id, data): + + # --- NEW: Forward original trace to Tempo --- + try: + for rs in data.get('resourceSpans', []): + # Requirement B: Ensure service.name is present in the resource + # (Checking if it's missing and adding it) + attrs = rs.get('resource', {}).get('attributes', []) + if not any(a['key'] == 'service.name' for a in attrs): + attrs.append({"key": "service.name", "value": {"stringValue": "cve-agent"}}) + + for ss in rs.get('scopeSpans', []): + for span in ss.get('spans', []): + # Requirement A: If it's the workflow, explicitly null the parent + if span.get('name') == "": + span['parentSpanId'] = "" + # Tempo expects the raw OTLP JSON structure as-is + tempo_status = tempo_provider.send_post(json.dumps(data)) + # Optional: logging for debug + # print(f"Sent copy to Tempo, status: {tempo_status}") + except Exception as e: + print(f"Failed to forward trace to Tempo: {e}") + # Traverse the OTLP structure + for rs in data.get('resourceSpans', []): + for ss in rs.get('scopeSpans', []): + for span in ss.get('spans', []): + metadata = get_metadata(span) + attrs = { + a['key']: a.get('value', {}) + for a in span.get('attributes', []) + } + trace_id = metadata.get('trace_id') + if trace_id not in traces_table: + print("add trace_id to traces_table: ", trace_id) + traces_table[trace_id] = [] + if trace_id in map_trace_id_to_job_id: + job_id = map_trace_id_to_job_id[trace_id] + else: + job_id = find_job_id(attrs) + if len(job_id) > 0: + trace_collection = traces_table.get(trace_id, []) + if trace_id not in map_trace_id_to_job_id: + map_trace_id_to_job_id[trace_id] = job_id + print(f"add trace_id to map_trace_id_to_job_id: {trace_id} --> {job_id}") + print(map_trace_id_to_job_id) + dict_payload = {} + dict_payload["metadata"] = metadata + dict_payload["attributes"] = attrs + trace_obj = Trace( + format_version=TRACE_VERSION, + job_id=job_id, + execution_start_timestamp=LocalDateTime(datetime.fromtimestamp(int(metadata.get('start_time',"0"))/1e9)), + trace_id=metadata.get('trace_id'), + span_id=metadata.get('span_id'), + span_payload=dict_payload + ) + trace_collection.append(trace_obj) + traces_table[trace_id] = trace_collection + span_name = metadata.get('name') + if span_name == "": + print("finsh scan workflow- need to send mlops") + traces = traces_table.get(trace_id, []) + traces_container = TracesContainer(traces) + status = http_provider.send_post(traces_container.model_dump_json()) + print(f"send traces(job_id: {job_id}, trace_id: {trace_id}) to endpoint status: {status}") + print(traces_table.keys()) + print(("del trace_id: ", trace_id)) + del traces_table[trace_id] + + +def run_worker( + url: str = DEFAULT_URL, + auth_type: str = DEFAULT_AUTH_TYPE, + token_path: str = DEFAULT_TOKEN_PATH, + verify_path: str = DEFAULT_VERIFY_PATH, + enable_verify: bool = DEFAULT_ENABLE_VERIFY, +): + # Logic to stop the loop on SIGTERM + stop_triggered = False + def signal_handler(sig, frame): + nonlocal stop_triggered + stop_triggered = True + + signal.signal(signal.SIGTERM, signal_handler) + # --- ADD THIS WAIT LOOP --- + print(f"Waiting for Kafka at {KAFKA_BROKER}...") + consumer = None + while consumer is None: + try: + consumer = KafkaConsumer( + KAFKA_TOPIC, + bootstrap_servers=[KAFKA_BROKER], + auto_offset_reset='earliest', + group_id='cve-file-writer-group', + value_deserializer=lambda m: json.loads(m.decode('utf-8')), + # Optional: reduce metadata timeout for faster retries + request_timeout_ms=20000, + session_timeout_ms=10000 + ) + print("Successfully connected to Redpanda!") + except NoBrokersAvailable: + print("Kafka not ready yet... retrying in 2 seconds.") + time.sleep(2) + # -------------------------- + print(f"📡 Worker Active. Writing traces to: {OUTPUT_FILE}") + traces_table = {} + map_trace_id_to_job_id = {} + + #items_count = 0 + enpoint_url = url + "/api/v1/traces" + EndpointAuthType = AuthType.BEARER + if auth_type != DEFAULT_AUTH_TYPE: + EndpointAuthType = AuthType.NONE + http_provider = HttpProvider( + url=enpoint_url, + auth_type=EndpointAuthType, + token_path=token_path, + verify_path=verify_path, + enable_verify=enable_verify, + ) + + # Initialize Tempo Provider (Insecure/Internal) + tempo_provider = HttpProvider( + url=TEMPO_URL, + auth_type=AuthType.NONE, + enable_verify=False + ) + + try: + with open(OUTPUT_FILE, 'a', encoding='utf-8') as f: + while not stop_triggered: + messages = consumer.poll(timeout_ms=1000) + for tp, records in messages.items(): + for message in records: + data = message.value + #f.write(f"Item: {items_count}\n") + #f.write(json.dumps(data) + '\n') + #f.flush() + process_message(http_provider, tempo_provider, traces_table, map_trace_id_to_job_id, data) + #items_count += 1 + finally: + consumer.close() + print("\nStopping worker...") + +def parse_args(): + parser = argparse.ArgumentParser(description="Collect traces from Kafka and optionally dispatch to an endpoint.") + parser.add_argument( + "--url", + default=DEFAULT_URL, + help="Dispatch endpoint URL (default: %(default)s)", + ) + parser.add_argument( + "--auth-type", + default=DEFAULT_AUTH_TYPE, + help="Auth type for the endpoint, e.g. Bearer (default: %(default)s)", + ) + parser.add_argument( + "--token-path", + default=DEFAULT_TOKEN_PATH, + help="Path to auth token file (default: %(default)s)", + ) + parser.add_argument( + "--verify-path", + default=DEFAULT_VERIFY_PATH, + help="Path to TLS CA cert for verification (default: %(default)s)", + ) + parser.add_argument( + "--enable-verify", + action="store_true", + default=DEFAULT_ENABLE_VERIFY, + help="Enable TLS verification (default: False)", + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + run_worker( + url=args.url, + auth_type=args.auth_type, + token_path=args.token_path, + verify_path=args.verify_path, + enable_verify=args.enable_verify, + ) + + + diff --git a/ci/scripts/collect_traces_local.py b/ci/scripts/collect_traces_local.py new file mode 100644 index 000000000..f299c2738 --- /dev/null +++ b/ci/scripts/collect_traces_local.py @@ -0,0 +1,353 @@ +""" +Local trace collector -- copy of collect_and_dispatch_traces.py with per-trace +flattened JSON file output for offline analysis. + +Writes one JSON file per completed trace to traces_output/{job_id}_{trace_id}.json. +Original Tempo/endpoint dispatch is preserved. +""" +from kafka import KafkaConsumer +import argparse +import json +import os +from HttpProvider import HttpProvider, AuthType +from datetime import datetime +from typing import Any +from pydantic import BaseModel, Field, constr, RootModel +import signal +import time +from kafka.errors import NoBrokersAvailable + +TRACE_VERSION = 1 + +DEFAULT_URL = "http://localhost:8080" +DEFAULT_AUTH_TYPE = "none" +DEFAULT_TOKEN_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/token" +DEFAULT_VERIFY_PATH = "/app/certs/service-ca.crt" +DEFAULT_ENABLE_VERIFY = False +TEMPO_URL = "http://localhost:8080" + +KAFKA_TOPIC = 'otel-traces' +KAFKA_BROKER = 'localhost:9092' +TRACES_OUTPUT_DIR = 'traces_output' +FLUSH_GRACE_PERIOD_SEC = 15 + + +class LocalDateTime(RootModel[datetime]): + root: datetime = Field(..., examples=['2022-03-10T12:15:50']) + + +class Trace(BaseModel): + format_version: int | None = None + job_id: constr(min_length=1) = Field( + ..., description='The job run id in which the trace and spans were instrumented' + ) + execution_start_timestamp: LocalDateTime | None = None + trace_id: constr(min_length=1) = Field( + ..., + description='Trace id grouping all spans of sub-tasks in one agent stage', + ) + span_id: constr(min_length=1) = Field( + ..., description='The id corresponding to one sub task inside one agent stage.' + ) + span_payload: dict[str, Any] | None = Field( + None, + description='Span payload with metadata and instrumentation data', + ) + + +class TracesContainer(RootModel[list[Trace]]): + pass + + +def _otlp_str(val) -> str: + """Extract plain string from an OTLP attribute value like {"stringValue": "..."}.""" + if isinstance(val, dict): + return val.get("stringValue", "") + return str(val) if val else "" + + +def _otlp_int(val) -> int: + """Extract integer from an OTLP attribute value like {"intValue": "123"}.""" + if isinstance(val, dict): + raw = val.get("intValue", 0) + try: + return int(raw) + except (ValueError, TypeError): + return 0 + try: + return int(val) + except (ValueError, TypeError): + return 0 + + +def get_metadata(span: dict) -> dict: + return { + 'trace_id': span.get('traceId'), + 'span_id': span.get('spanId'), + 'parent_span_id': span.get('parentSpanId'), + 'name': span.get('name'), + 'start_time': span.get('startTimeUnixNano'), + 'end_time': span.get('endTimeUnixNano'), + 'status': span.get('status'), + } + + +def find_job_id(attributes: dict) -> str: + try: + input_value = attributes.get("input.value") + str_json = input_value.get("stringValue") + scan = json.loads(str_json) + if "scan" in scan: + return scan["scan"]["id"] + if "input" in scan: + return scan["input"]["scan"]["id"] + else: + print("Warning: Scan not found in attributes") + return "" + except Exception as e: + print(f"Error finding job id (scan id): {e}") + return "" + + +def flatten_span(metadata: dict, attrs: dict) -> dict: + """Convert a raw OTLP span (metadata + attributes) into a flat dict.""" + return { + "span_id": metadata.get("span_id", ""), + "parent_span_id": metadata.get("parent_span_id", ""), + "name": metadata.get("name", ""), + "start_time": metadata.get("start_time", ""), + "end_time": metadata.get("end_time", ""), + "span_kind": _otlp_str(attrs.get("nat.span.kind")), + "function_name": _otlp_str(attrs.get("nat.function.name")), + "input_value": _otlp_str(attrs.get("input.value")), + "output_value": _otlp_str(attrs.get("output.value")), + "token_prompt": _otlp_int(attrs.get("llm.token_count.prompt")), + "token_completion": _otlp_int(attrs.get("llm.token_count.completion")), + "token_total": _otlp_int(attrs.get("llm.token_count.total")), + } + + +def write_trace_file(trace_id: str, job_id: str, flat_spans: list[dict], output_dir: str): + """Write a completed trace as a single JSON file, spans sorted by start_time.""" + os.makedirs(output_dir, exist_ok=True) + flat_spans.sort(key=lambda s: int(s.get("start_time") or 0)) + trace_doc = { + "trace_id": trace_id, + "job_id": job_id, + "completed_at": datetime.utcnow().isoformat(), + "spans": flat_spans, + } + filename = f"{job_id}_{trace_id}.json" + filepath = os.path.join(output_dir, filename) + with open(filepath, "w", encoding="utf-8") as f: + json.dump(trace_doc, f, indent=2) + print(f"Wrote trace file: {filepath} ({len(flat_spans)} spans)") + + +def process_message(http_provider, tempo_provider, traces_table, flat_spans_table, + map_trace_id_to_job_id, pending_flushes, data, output_dir): + # Forward original trace to Tempo + try: + for rs in data.get('resourceSpans', []): + attrs = rs.get('resource', {}).get('attributes', []) + if not any(a['key'] == 'service.name' for a in attrs): + attrs.append({"key": "service.name", "value": {"stringValue": "cve-agent"}}) + for ss in rs.get('scopeSpans', []): + for span in ss.get('spans', []): + if span.get('name') == "": + span['parentSpanId'] = "" + #tempo_status = tempo_provider.send_post(json.dumps(data)) + except Exception as e: + print(f"Failed to forward trace to Tempo: {e}") + + for rs in data.get('resourceSpans', []): + for ss in rs.get('scopeSpans', []): + for span in ss.get('spans', []): + metadata = get_metadata(span) + attrs = { + a['key']: a.get('value', {}) + for a in span.get('attributes', []) + } + trace_id = metadata.get('trace_id') + if trace_id not in traces_table: + print("add trace_id to traces_table: ", trace_id) + traces_table[trace_id] = [] + flat_spans_table[trace_id] = [] + if trace_id in map_trace_id_to_job_id: + job_id = map_trace_id_to_job_id[trace_id] + else: + job_id = find_job_id(attrs) + if len(job_id) > 0: + trace_collection = traces_table.get(trace_id, []) + if trace_id not in map_trace_id_to_job_id: + map_trace_id_to_job_id[trace_id] = job_id + print(f"add trace_id to map_trace_id_to_job_id: {trace_id} --> {job_id}") + + dict_payload = {"metadata": metadata, "attributes": attrs} + trace_obj = Trace( + format_version=TRACE_VERSION, + job_id=job_id, + execution_start_timestamp=LocalDateTime( + datetime.fromtimestamp(int(metadata.get('start_time', "0")) / 1e9) + ), + trace_id=metadata.get('trace_id'), + span_id=metadata.get('span_id'), + span_payload=dict_payload, + ) + trace_collection.append(trace_obj) + traces_table[trace_id] = trace_collection + + flat_spans_table.setdefault(trace_id, []).append( + flatten_span(metadata, attrs) + ) + + span_name = metadata.get('name') + func_name = _otlp_str(attrs.get("nat.function.name")) + if span_name == "" or func_name == "agent_finish": + if trace_id not in traces_table: + print(f"Trace {trace_id} already flushed, skipping duplicate trigger.") + continue + if trace_id not in pending_flushes: + trigger = span_name if span_name == "" else f"agent_finish (func={func_name})" + pending_flushes[trace_id] = time.time() + print(f"Flush scheduled for trace {trace_id} (trigger: {trigger}, " + f"grace period: {FLUSH_GRACE_PERIOD_SEC}s, " + f"spans so far: {len(flat_spans_table.get(trace_id, []))})") + + +def flush_pending_traces(pending_flushes, traces_table, flat_spans_table, + map_trace_id_to_job_id, output_dir, force=False): + """Flush traces whose grace period has expired (or all if force=True).""" + now = time.time() + flushed = [] + for trace_id, trigger_time in list(pending_flushes.items()): + if not force and (now - trigger_time) < FLUSH_GRACE_PERIOD_SEC: + continue + if trace_id not in traces_table: + flushed.append(trace_id) + continue + job_id = map_trace_id_to_job_id.get(trace_id, "unknown") + waited = now - trigger_time + span_count = len(flat_spans_table.get(trace_id, [])) + print(f"Flushing trace {trace_id} (waited {waited:.1f}s, {span_count} spans)") + + write_trace_file( + trace_id, job_id, + flat_spans_table.get(trace_id, []), + output_dir, + ) + + del traces_table[trace_id] + del flat_spans_table[trace_id] + flushed.append(trace_id) + for trace_id in flushed: + del pending_flushes[trace_id] + + +def run_worker( + url: str = DEFAULT_URL, + auth_type: str = DEFAULT_AUTH_TYPE, + token_path: str = DEFAULT_TOKEN_PATH, + verify_path: str = DEFAULT_VERIFY_PATH, + enable_verify: bool = DEFAULT_ENABLE_VERIFY, + output_dir: str = TRACES_OUTPUT_DIR, +): + stop_triggered = False + + def signal_handler(sig, frame): + nonlocal stop_triggered + stop_triggered = True + + signal.signal(signal.SIGTERM, signal_handler) + + print(f"Waiting for Kafka at {KAFKA_BROKER}...") + consumer = None + while consumer is None: + try: + consumer = KafkaConsumer( + KAFKA_TOPIC, + bootstrap_servers=[KAFKA_BROKER], + auto_offset_reset='earliest', + group_id='cve-file-writer-group', + value_deserializer=lambda m: json.loads(m.decode('utf-8')), + request_timeout_ms=20000, + session_timeout_ms=10000, + ) + print("Successfully connected to Redpanda!") + except NoBrokersAvailable: + print("Kafka not ready yet... retrying in 2 seconds.") + time.sleep(2) + + print(f"Worker Active. Writing per-trace files to: {output_dir}/ (grace period: {FLUSH_GRACE_PERIOD_SEC}s)") + os.makedirs(output_dir, exist_ok=True) + traces_table = {} + flat_spans_table = {} + map_trace_id_to_job_id = {} + pending_flushes = {} + + endpoint_url = url + "/api/v1/traces" + EndpointAuthType = AuthType.BEARER + if auth_type != DEFAULT_AUTH_TYPE: + EndpointAuthType = AuthType.NONE + http_provider = HttpProvider( + url=TEMPO_URL, + auth_type=AuthType.NONE, + enable_verify=False, + ) + tempo_provider = HttpProvider( + url=TEMPO_URL, + auth_type=AuthType.NONE, + enable_verify=False, + ) + + try: + while not stop_triggered: + messages = consumer.poll(timeout_ms=1000) + for tp, records in messages.items(): + for message in records: + data = message.value + process_message( + http_provider, tempo_provider, + traces_table, flat_spans_table, + map_trace_id_to_job_id, pending_flushes, + data, output_dir, + ) + flush_pending_traces( + pending_flushes, traces_table, flat_spans_table, + map_trace_id_to_job_id, output_dir, + ) + finally: + if pending_flushes: + print(f"\nFlushing {len(pending_flushes)} pending traces on shutdown...") + flush_pending_traces( + pending_flushes, traces_table, flat_spans_table, + map_trace_id_to_job_id, output_dir, force=True, + ) + consumer.close() + print("Stopping worker...") + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Collect traces from Kafka, dispatch to endpoint, and write per-trace JSON files for analysis." + ) + parser.add_argument("--url", default=DEFAULT_URL, help="Dispatch endpoint URL") + parser.add_argument("--auth-type", default=DEFAULT_AUTH_TYPE, help="Auth type (Bearer or None)") + parser.add_argument("--token-path", default=DEFAULT_TOKEN_PATH, help="Path to auth token file") + parser.add_argument("--verify-path", default=DEFAULT_VERIFY_PATH, help="Path to TLS CA cert") + parser.add_argument("--enable-verify", action="store_true", default=DEFAULT_ENABLE_VERIFY) + parser.add_argument("--output-dir", default=TRACES_OUTPUT_DIR, + help="Directory for per-trace JSON files (default: traces_output)") + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + run_worker( + url=args.url, + auth_type=args.auth_type, + token_path=args.token_path, + verify_path=args.verify_path, + enable_verify=args.enable_verify, + output_dir=args.output_dir, + ) diff --git a/ci/scripts/config_fetcher.py b/ci/scripts/config_fetcher.py new file mode 100644 index 000000000..b4fe57f91 --- /dev/null +++ b/ci/scripts/config_fetcher.py @@ -0,0 +1,79 @@ +import os +import sys +import csv +import json +import requests +from google.oauth2 import service_account +from google.auth.transport.requests import Request + +def get_access_token(): + """Authenticates and returns a Bearer token.""" + + # 1. Get Credentials from the mounted volume + cred_path = os.getenv('GOOGLE_SERVICE_ACCOUNT_FILE') + if cred_path is None: + raise ValueError("GOOGLE_SERVICE_ACCOUNT_FILE environment variable is not set") + + scopes = ['https://www.googleapis.com/auth/spreadsheets.readonly'] + + try: + creds = service_account.Credentials.from_service_account_file( + cred_path, + scopes=scopes + ) + creds.refresh(Request()) + return creds.token + except Exception as e: + print(f"# Error loading credentials: {e}", file=sys.stderr) + return None + +def fetch_config(): + """ + Fetches configuration from a Google Sheet (CSV export) and prints + shell-compatible export statements. + """ + + # 2. Get Input Sheet ID + sheet_id = os.getenv('GSHEETS_INPUT_SHEET_ID') + if sheet_id is None: + raise ValueError("GSHEETS_INPUT_SHEET_ID environment variable is not set") + # We need the 'spreadsheets.readonly' scope to read the data + scopes = ['https://www.googleapis.com/auth/spreadsheets.readonly'] + tab_name = "LLM_Config" + # 1. Resolve Name -> GID + token = get_access_token() + # --- THE FIX: Use the 'values' API instead of 'export' URL --- + # We ask for columns A and B from the specific tab name + # URL encoded range: "TabName!A:B" + range_name = f"{tab_name}!A:B" + url = f"https://sheets.googleapis.com/v4/spreadsheets/{sheet_id}/values/{range_name}" + + headers = {'Authorization': f'Bearer {token}'} + + try: + response = requests.get(url, headers=headers, timeout=30) + response.raise_for_status() + data = response.json() + except Exception as e: + # If this fails, print the details (often helpful for debugging) + print(f"# API Request Failed: {e}", file=sys.stderr) + sys.exit(1) + + # The API returns JSON: { "values": [ ["Key", "Val"], ["Key2", "Val2"] ] } + rows = data.get('values', []) + for row in rows: + # We need at least 2 columns (Key and Value) + if len(row) >= 2: + key = row[0].strip() + value = row[1].strip() + + # 1. Skip empty keys + # 2. Skip the header row if it literally says "Key" + # 3. Skip comments (keys starting with #) + if key and key != "Key" and not key.startswith("#"): + # Escape single quotes to prevent shell syntax errors + safe_value = value.replace("'", "'\\''") + print(f"export {key}='{safe_value}'") + +if __name__ == "__main__": + fetch_config() \ No newline at end of file diff --git a/docker-compose.mlops.yml b/docker-compose.mlops.yml new file mode 100644 index 000000000..5be14e73a --- /dev/null +++ b/docker-compose.mlops.yml @@ -0,0 +1,28 @@ +services: + # Phoenix - Visualization + phoenix: + image: arizephoenix/phoenix + ports: + - "6006:6006" + + # Redpanda - Lightweight Kafka replacement + redpanda: + image: docker.redpanda.com/redpandadata/redpanda:v23.2.1 + ports: + - "9092:9092" + command: > + redpanda start --smp 1 --overprovisioned + --kafka-addr internal://0.0.0.0:29092,external://0.0.0.0:9092 + --advertise-kafka-addr internal://redpanda:29092,external://localhost:9092 + + # OTel Collector - The Traffic Controller + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + container_name: otel-collector + volumes: + - ./otel-config.yaml:/etc/otelcol-contrib/config.yaml:Z + ports: + - "4318:4318" # Agent sends data here + depends_on: + - redpanda + - phoenix \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 76f8b29ca..42d078f4f 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -68,6 +68,7 @@ services: - CODE_VDB_RETRIEVER_MODEL_NAME=${CODE_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} - DOC_VDB_RETRIEVER_MODEL_NAME=${DOC_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} - CVE_AGENT_EXECUTOR_MODEL_NAME=${CVE_AGENT_EXECUTOR_MODEL_NAME:-meta/llama-3.1-70b-instruct} + - GENERATE_CVSS_MODEL_NAME=${GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} - SUMMARIZE_MODEL_NAME=${SUMMARIZE_MODEL_NAME:-meta/llama-3.1-70b-instruct} - JUSTIFY_MODEL_NAME=${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} - EMBEDDER_MODEL_NAME=${EMBEDDER_MODEL_NAME:-nvidia/nv-embedqa-e5-v5} diff --git a/kustomize/README.md b/kustomize/README.md new file mode 100644 index 000000000..373017ef7 --- /dev/null +++ b/kustomize/README.md @@ -0,0 +1,740 @@ + + + +# Deploying Exploit Intelligence on OpenShift Container Platform + +## Prerequisites + +> [!NOTE] +> All commands in this guide assume that your current working directory is `kustomize/`. + +### Required Tools + +Install the following tools and verify that all binaries are available on your system path: + +- [`oc`](https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html) +- [`kustomize`](https://kubectl.docs.kubernetes.io/installation/kustomize/) +- `openssl` + +### Cluster Access + +Cluster-admin privileges are required. `OAuthClient` creation, anyuid SCC grants, and `ServiceMonitor` access are all cluster-scoped operations. If cluster-admin access is not available, a cluster administrator can optionally apply `deployer-rbac.yaml` to delegate the minimum required permissions. + +### API Credentials + +Obtain the following credentials before you begin. For instructions on creating each credential, refer to the main [README](../README.md#obtain-api-keys): + +- GitHub Advisory Database (GHSA) API key +- National Vulnerability Database (NVD) API key +- NVIDIA API key (required for Remote NIM; a placeholder value is sufficient for self-hosted variants) +- Red Hat registry credentials (`registry.redhat.io`) +- SerpAPI key + +--- + +## Preparing Your Deployment + +Complete the following steps before running any deployment command. + +### Step 1. Create a Project Namespace + +```shell +export YOUR_NAMESPACE_NAME= +oc new-project $YOUR_NAMESPACE_NAME +``` + +### Step 2. Create API Credentials + +Create the `base/secrets.env` file with your API keys. The following table shows which keys are required for each deployment variant: + +| Key | Self-Hosted LLM | Remote NIM | Description | +| --- | --- | --- | --- | +| `serpapi_api_key` | Required | Required | Web search for patch intelligence | +| `ghsa_api_key` | Required | Required | GitHub token for advisory lookups and repository scanning | +| `nvidia_api_key` | Placeholder | Required | A placeholder value is sufficient for self-hosted variants | +| `nvd_api_key` | Required | Required | National Vulnerability Database API key | +| `registry_redhat_username` | Required | Required | Red Hat registry credentials for container image scanning | +| `registry_redhat_password` | Required | Required | Red Hat registry credentials for container image scanning | + +```shell +cat > base/secrets.env << EOF +serpapi_api_key= +ghsa_api_key= +nvidia_api_key= +nvd_api_key= +registry_redhat_username= +registry_redhat_password= +EOF +``` + +### Step 3. Configure Image Registry Credentials + +> [!IMPORTANT] +> Product scanning requires valid registry credentials to pull product component images. If you skip this step, the deployment succeeds but authenticated image pulls fail. + +Create the `base/image-registry-credentials.env` file with credentials for the registries that product scanning uses to pull component images. The `auth` value is the base64 encoding of `:`. + +For `registry.redhat.io`, Red Hat recommends creating a registry service account at [https://access.redhat.com/terms-based-registry/](https://access.redhat.com/terms-based-registry/) + +```shell +cat > base/image-registry-credentials.env << 'EOF' +{ + "auths": { + "registry.example.io": {"auth": ""}, + "example.io": {"auth": ""}, + "example.docker.io": {"auth": ""} + } +} +EOF +``` + +### Step 4. Configure Argilla Feedback Credentials + +Argilla runs as an internal cluster service and does not connect to any external account. Use any values for the username, password, and API key. + +Create the `base/argilla/feedback_secret.env` file: + +```shell +cat > base/argilla/feedback_secret.env << EOF +argilla_username= +argilla_password= +argilla_api_key= +EOF +``` + +### Step 5. Configure OAuth Credentials + +Exploit Intelligence uses OpenShift OAuth for user authentication. The OAuth client secret must be at least 32 bytes (256 bits) because Exploit Intelligence uses it to sign internal session tokens with HS256, which requires a minimum key length of 256 bits. + +> [!IMPORTANT] +> Save the value of `$OAUTH_CLIENT_SECRET` after running the commands below. You need it after deployment to create or update the `OAuthClient` resource. + +#### First-Time Deployment + +Use this procedure only if no `OAuthClient` named `exploit-iq-client` exists on the cluster. If another Exploit Intelligence installation already uses that `OAuthClient`, you must use the [Reusing an Existing OAuthClient](#reusing-an-existing-oauthclient) procedure instead — generating a new secret overwrites the existing one and breaks authentication for all users of that installation. + +Verify that the `OAuthClient` does not exist before proceeding: + +```shell +oc get oauthclient exploit-iq-client 2>&1 | grep -q "not found" && echo "Safe to proceed" || echo "OAuthClient exists — use Reusing an Existing OAuthClient" +``` + +Generate a new client secret and write the credentials file: + +```shell +export OAUTH_CLIENT_SECRET=$(openssl rand -base64 32) +export OAUTH_SESSION_SECRET=$(openssl rand -base64 32) +export OCP_DOMAIN=$(oc whoami --show-server | sed -E 's#^https?://api\.##; s#:[0-9]+/?$##; s#/$##') + +cat > base/oauth-secrets.env << EOF +client-secret=$OAUTH_CLIENT_SECRET +session-secret=$OAUTH_SESSION_SECRET +openshift-domain=$OCP_DOMAIN +EOF +``` + +#### Reusing an Existing OAuthClient + +Retrieve the secret from the existing `OAuthClient` resource and write the credentials file: + +```shell +export OAUTH_CLIENT_SECRET=$(oc get oauthclient exploit-iq-client -o jsonpath='{..secret}') +export OAUTH_SESSION_SECRET=$(openssl rand -base64 32) +export OCP_DOMAIN=$(oc whoami --show-server | sed -E 's#^https?://api\.##; s#:[0-9]+/?$##; s#/$##') + +cat > base/oauth-secrets.env << EOF +client-secret=$OAUTH_CLIENT_SECRET +session-secret=$OAUTH_SESSION_SECRET +openshift-domain=$OCP_DOMAIN +EOF +``` + +### Step 6. Create Database Credentials + +```shell +cat > base/mongodb-credentials.env << EOF +admin-user=mongoadmin +admin-password=$(openssl rand -base64 24) +exploit-iq-user=exploit-iq-user +exploit-iq-password=$(openssl rand -base64 24) +EOF +``` + +### Step 7. Set the Application Callback URL + +```shell +export CALLBACK_URL="https://exploit-iq-client.$(oc project -q).svc:8443" +find . -type f -name 'exploit-iq-config.yml' -exec sed -i "s|CALLBACK_URL_PLACEHOLDER|$CALLBACK_URL|g" {} + +``` + +--- + +## Selecting a Deployment Variant + +Exploit Intelligence supports the following deployment variants. Run only one deployment command in the next section. + +| Variant | Overlay | LLM | Use When | +| --- | --- | --- | --- | +| Self-Hosted LLM | `self-hosted-llama3.1-70b-4bit` | Any network-accessible LLM endpoint | You run your own LLM (on-cluster or external) | +| Self-Hosted LLM + MLOps | `mlops` | Any network-accessible LLM endpoint | You also need Grafana and Tempo observability | +| Remote NIM | `remote-nim-all` | NVIDIA-hosted NIM | You use NVIDIA-hosted inference | + +--- + +## Deploying Exploit Intelligence + +### Deploy with a Self-Hosted LLM + +This overlay assumes the LLM is deployed using the [exploit-iq-models](https://github.com/RHEcosystemAppEng/exploit-iq-models) Helm chart, which creates the `llama3-1-70b-instruct-4bit` service in the `exploit-iq-models` namespace. If your LLM is deployed differently, update the upstream URLs in `overlays/self-hosted-llama3.1-70b-4bit/nginx-patch.yaml` to point to your model endpoint before deploying. + +```shell +oc kustomize overlays/self-hosted-llama3.1-70b-4bit | oc apply -f - -n $YOUR_NAMESPACE_NAME +``` + +### Deploy with a Self-Hosted LLM and MLOps Observability + +Patch the overlay with your namespace: + +```shell +sed -i "s/REPLACE_NAMESPACE/$YOUR_NAMESPACE_NAME/" overlays/mlops/grafana/kustomization.yaml +sed -i "s/REPLACE_NAMESPACE/$YOUR_NAMESPACE_NAME/" overlays/mlops/tempo/kustomization.yaml +``` + +Create the Grafana token secret. Retrieve the token value from the Bitwarden vault entry **Exploit Intelligence Grafana SA Token**: + +```shell +oc create secret generic grafana-bearer-token \ + --from-literal=token='' +``` + +Deploy: + +```shell +oc kustomize overlays/mlops | oc apply -f - -n $YOUR_NAMESPACE_NAME +``` + +> [!WARNING] +> Grafana custom resources may fail to create if the Grafana Operator has not yet reached the `Succeeded` phase. Check the operator status: +> +> ```shell +> oc get csv -n $YOUR_NAMESPACE_NAME | grep grafana-operator +> ``` +> +> When the PHASE column shows `Succeeded`, redeploy the Grafana resources: +> +> ```shell +> oc kustomize overlays/mlops \ +> | yq e 'select(.metadata.labels.managed-by == "grafana-operator")' - \ +> | oc replace -f - -n $YOUR_NAMESPACE_NAME +> ``` + +### Deploy with Remote NIM + +```shell +oc kustomize overlays/remote-nim-all | oc apply -f - -n $YOUR_NAMESPACE_NAME +``` + +--- + +## Post-Deployment Configuration + +### Configure OpenShift OAuth + +> [!WARNING] +> Complete this step before attempting to log in to the Exploit Intelligence UI. Authentication fails if the `OAuthClient` resource is not configured correctly. + +After the deployment completes and the `exploit-iq-client` route is available, configure the OpenShift OAuth client. Select the procedure that matches your situation. + +#### Create the OAuthClient Resource + +Complete this procedure if you followed Step 5, "First-Time Deployment": + +```shell +oc create -f - < +``` + +### Enable OAuth for the MCP Server + +Complete this step if you want MCP clients such as Claude Code or Cursor to authenticate with OpenShift OAuth. + +If the `OAuthClient` resource does not exist, create it: + +```shell +oc create -f - < | oc apply -f - -n $YOUR_NAMESPACE_NAME +``` + +To restrict MCP access to specific OpenShift groups, set the `EXPLOITIQ_ALLOWED_GROUPS` environment variable in `base/exploitiq_mcp_server.yaml`. The default value is `exploitiq-users`. To allow any authenticated user, remove the environment variable entirely. + +### Configure the Swagger UI Endpoint + +Complete this step to enable the Swagger UI endpoint for API testing. This configuration applies to non-production environments only. + +```shell +oc set env deployment -l component=exploit-iq-client \ + QUARKUS_SMALLRYE_OPENAPI_SERVERS=https://$(oc get route exploit-iq-client -o=jsonpath='{..spec.host}') +``` + +### Configure a Custom Git Server CA + +Complete this step if your Git server uses a certificate signed by a custom Certificate Authority (CA). + +> [!IMPORTANT] +> Complete this step if you access Red Hat internal Git repositories such as `gitlab.cee.redhat.com`. + +**1.** Create the certificate directory: + +```shell +mkdir -p base/ca-certs +``` + +**2.** Obtain your CA certificates. + +For Red Hat internal Git repositories: + +```shell +curl -o base/ca-certs/internal-root-ca.pem \ + https://certs.corp.redhat.com/certs/2022-IT-Root-CA.pem + +curl -o base/ca-certs/rhcs-intermediate-ca.crt \ + https://certs.corp.redhat.com/chains/rhcs-ca-chain-2022-self-signed.crt +``` + +For other custom CAs: + +```shell +cp /path/to/your-custom-ca.pem base/ca-certs/ +``` + +**3.** Create the CA bundle: + +```shell +cat base/ca-certs/*.{pem,crt} > base/ca-certs/ca-bundle.crt +``` + +**4.** Verify the bundle: + +```shell +openssl crl2pkcs7 -nocrl -certfile base/ca-certs/ca-bundle.crt | \ + openssl pkcs7 -print_certs -noout +``` + +Then redeploy: + +```shell +oc kustomize overlays/ | oc apply -f - -n $YOUR_NAMESPACE_NAME +``` + +### Configure the OAuth Endpoint CA + +Complete this step if you see the error `PKIX path building failed` in the `exploit-iq-client` pod logs after deployment. + +**1.** Fetch and merge CA chains from both OpenShift OIDC endpoints: + +- `oauth-openshift.apps.` (authorization/token endpoint chain) +- `api.:6443` (JWKS and user-info endpoint chain) + +The command extracts only CA certificates (skips endpoint leaf certificates, which are not trust anchors): + +```shell +export OCP_DOMAIN=$(oc whoami --show-server | sed -E 's#^https?://api\.##; s#:[0-9]+/?$##; s#/$##') +export OIDC_CA_DIR=components/oidc-ca +mkdir -p "${OIDC_CA_DIR}" + +openssl s_client -connect oauth-openshift.apps.${OCP_DOMAIN}:443 \ + -servername oauth-openshift.apps.${OCP_DOMAIN} \ + -showcerts /dev/null \ + | awk '/BEGIN CERT/,/END CERT/' \ + | awk 'BEGIN{n=0;buf=""} /BEGIN CERT/{n++;buf=""} {buf=buf $0"\n"} /END CERT/{if(n>1) printf buf}' \ + > "${OIDC_CA_DIR}/oidc-oauth-ca.crt" + +openssl s_client -connect api.${OCP_DOMAIN}:6443 \ + -servername api.${OCP_DOMAIN} \ + -showcerts /dev/null \ + | awk '/BEGIN CERT/,/END CERT/' \ + | awk 'BEGIN{n=0;buf=""} /BEGIN CERT/{n++;buf=""} {buf=buf $0"\n"} /END CERT/{if(n>1) printf buf}' \ + > "${OIDC_CA_DIR}/oidc-api-ca.crt" +``` + +**2.** Verify the bundle: + +```shell +cat "${OIDC_CA_DIR}/oidc-oauth-ca.crt" "${OIDC_CA_DIR}/oidc-api-ca.crt" > "${OIDC_CA_DIR}/ca-bundle.crt" + +openssl crl2pkcs7 -nocrl -certfile "${OIDC_CA_DIR}/ca-bundle.crt" | \ + openssl pkcs7 -print_certs -noout +``` + +**3.** Create or update the `oidc-ca-bundle` ConfigMap in your namespace (no certificate files are stored in this repository): + +```shell +oc -n $YOUR_NAMESPACE_NAME create configmap oidc-ca-bundle \ + --from-file=ca-bundle.crt="${OIDC_CA_DIR}/ca-bundle.crt" \ + --dry-run=client -o yaml | oc apply -f - +``` + +**4.** Add the `oidc-ca` component to your deployment variant and redeploy. The component patches the `exploit-iq-client` deployment to mount the CA bundle and configure the named OIDC TLS trust store: + +```shell +(cd overlays/ && kustomize edit add component ../../components/oidc-ca) +oc kustomize overlays/ | oc apply -f - -n $YOUR_NAMESPACE_NAME +``` + +> [!NOTE] +> **Dev/temporary workaround:** If you need to unblock quickly without the certificate setup, set the following environment variable on the deployment. This disables TLS verification entirely and must not be used in production environments. +> +> ```shell +> oc set env deployment/exploit-iq-client QUARKUS_TLS_TRUST_ALL=true -n $YOUR_NAMESPACE_NAME +> ``` + +--- + +## Uninstalling Exploit Intelligence + +Set your deployment variant and run one of the following commands: + +```shell +export DEPLOYMENT_VARIANT_NAME=remote-nim-all +# export DEPLOYMENT_VARIANT_NAME=self-hosted-llama3.1-70b-4bit +# export DEPLOYMENT_VARIANT_NAME=mlops +``` + +To delete all resources but preserve data in PersistentVolumeClaims: + +```shell +kustomize build overlays/$DEPLOYMENT_VARIANT_NAME/ | oc delete -l purpose!=persistent -f - +``` + +To delete all resources including persistent data: + +```shell +kustomize build overlays/$DEPLOYMENT_VARIANT_NAME/ | oc delete -f - +``` + +--- + +## Running Exploit Intelligence Locally + +You can run Exploit Intelligence on a local machine without GPU hardware, for development, debugging, and troubleshooting. + +Before you begin, install the following tools and verify that all binaries are available on your system path: + +- [uv package manager](https://docs.astral.sh/uv/getting-started/installation) +- Python 3.12 +- [Java JDK 21 or later](https://www.oracle.com/il-en/java/technologies/downloads/#java21) +- [Maven](https://maven.apache.org/install.html) +- [Go](https://go.dev/doc/install) + +### Running the Application Locally + +**1.** Navigate to the repository root and create a Python virtual environment: + +```shell +cd $(git rev-parse --show-toplevel) +rm -rf .venv || true +uv venv --python 3.12 --no-cache +source .venv/bin/activate +``` + +**2.** Install dependencies: + +```shell +uv sync --no-cache +``` + +**3.** Set the required environment variables: + +```shell +export CHECKLIST_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +export CODE_VDB_RETRIEVER_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +export CVE_AGENT_EXECUTOR_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +export DOC_VDB_RETRIEVER_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +export JUSTIFY_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +export SUMMARIZE_MODEL_NAME=hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +export NVIDIA_API_BASE=http:///v1 +export PYTHONUNBUFFERED=1 +export GOPROXY=https://proxy.golang.org,direct +export SERPAPI_API_KEY= +export REGISTRY_REDHAT_USERNAME="" +export REGISTRY_REDHAT_PASSWORD="" +``` + +**4.** Start the application: + +```shell +nat --log-level debug serve \ + --config_file=src/vuln_analysis/configs/config-http-openai.yml \ + --host 0.0.0.0 \ + --port 26466 +``` + +**5.** In a separate terminal with the virtual environment active, start the Arize Phoenix tracing service: + +```shell +phoenix serve +``` + +### Enabling Container Source Download + +Complete this step to extract RPM dependencies from Red Hat container registries for C/C++ projects. + +Before enabling this feature, verify that the following prerequisites are met: + +- `skopeo` is installed and available on your system path +- Valid Red Hat registry credentials are configured +- Network access to `registry.redhat.io` is available + +Set the following environment variable before starting the application: + +```shell +export USE_CONTAINER_SOURCES=true +``` + +--- + +## Deploying the Test Variant + +The test variant uses encrypted secret files. To decrypt them, you need the following tools and access: + +- [GnuPG](https://www.gnupg.org/download/) +- [SOPS](https://github.com/getsops/sops/releases) +- The private decryption key from the Bitwarden vault entry **Exploit Intelligence Tests Deployment Variant Private Key for Decryption** + +### Deploying the Test Overlay + +**1.** Create a project namespace: + +```shell +export PROJECT_NAME=exploit-test +oc new-project $PROJECT_NAME +``` + +**2.** Import the private decryption key: + +```shell +gpg --import /path/to/sec-decryption.key +``` + +**3.** Navigate to the test overlay directory and decrypt all secret files: + +```shell +cd $(git rev-parse --show-toplevel)/kustomize/overlays/tests +mkdir -p secrets + +sops -d exploit-iq-ips.secret > secrets/exploit-iq-ips.json +sops -d google-sheets-secrets-enc.yaml > secrets/google-sheets-secrets.yaml +sops -d integration-tests-secrets-enc.yaml > secrets/integration-tests-secrets.yaml +sops -d mongodb-credentials.env2 > secrets/mongodb-credentials.env +sops -d oauth-secrets.env2 > secrets/oauth-secrets.env +sops -d registry-app-creds-enc.yaml > secrets/registry-app-creds.yaml +sops -d secrets.env2 > secrets/secrets.env +sops -d server-model-config-enc.yaml > secrets/server-model-config.yaml +sops -d exploit-iq-client-build-ips-enc.yaml > secrets/exploit-iq-client-build-ips.yaml +sops -d exploit-iq-automation-token-enc.yaml > secrets/exploit-iq-automation-token.yaml +sops -d credential-encryption-key.env2 > secrets/credential-encryption-key.env +``` + +**4.** Override any decrypted secret values as needed, then continue to the next step. + +**5.** Configure the OAuth client secret. + +> [!WARNING] +> Authentication fails if you do not complete this step with the correct OAuth secret before deployment. + +If an `OAuthClient` named `exploit-iq-client` already exists on the cluster, retrieve its secret: + +```shell +export OAUTH_CLIENT_SECRET=$(oc get oauthclient exploit-iq-client -o jsonpath='{..secret}') +``` + +Otherwise, generate a new secret (minimum 32 characters): + +```shell +export OAUTH_CLIENT_SECRET=$(openssl rand -base64 32) +``` + +Write the credentials file: + +```shell +export OCP_DOMAIN=$(oc whoami --show-server | sed -E 's#^https?://api\.##; s#:[0-9]+/?$##; s#/$##') + +cat > secrets/oauth-secrets.env << EOF +client-secret=$OAUTH_CLIENT_SECRET +openshift-domain=$OCP_DOMAIN +EOF +``` + +**6.** Set the application callback URL: + +```shell +cd $(git rev-parse --show-toplevel)/kustomize +export CALLBACK_URL="https://exploit-iq-client.$(oc project -q).svc:8443" +find . -type f -name 'exploit-iq-config.yml' -exec sed -i "s|CALLBACK_URL_PLACEHOLDER|$CALLBACK_URL|g" {} + +cd $(git rev-parse --show-toplevel)/kustomize/overlays/tests +``` + +**7.** Deploy the test overlay: + +```shell +kustomize build . | oc apply -f - +``` + +**8.** Configure the `OAuthClient` resource. + +If the resource does not exist, create it: + +```shell +oc create -f - < \ + exploit-iq-tests ../../../exploit-iq-models/agent-morpheus-models +``` + +**11.** Remove the decrypted secret files: + +```shell +rm -rf secrets/ +``` + +### Tearing Down the Test Environment + +```shell +helm delete exploit-iq-tests +oc delete project $(oc project --short -q) +``` + +### Installing Pipelines as Code + +If you need to install the OpenShift Pipelines Operator on a new cluster, refer to the [OpenShift Pipelines installation documentation](https://docs.redhat.com/en/documentation/red_hat_openshift_pipelines/1.19/html/installing_and_configuring/installing-pipelines). + +To configure the [Exploit Intelligence PAC GitHub application](https://github.com/apps/exploit-iq-pac/) on a new cluster, follow the [PAC GitHub application configuration guide](https://pipelinesascode.com/docs/install/github_apps/#configure-pipelines-as-code-on-your-cluster-to-access-the-github-app). You need the GitHub application private key and the webhook secret from the application settings. diff --git a/kustomize/base/.gitignore b/kustomize/base/.gitignore new file mode 100644 index 000000000..5992ad117 --- /dev/null +++ b/kustomize/base/.gitignore @@ -0,0 +1,8 @@ +# Certificate files - contains sensitive internal CA certificates +ca-certs/*.pem +ca-certs/*.crt +ca-certs/ca-bundle.crt + +# Keep the directory structure +!ca-certs/.gitkeep +!ca-certs/README.md diff --git a/kustomize/base/anyuid-scc-permission.yaml b/kustomize/base/anyuid-scc-permission.yaml new file mode 100644 index 000000000..207d0f509 --- /dev/null +++ b/kustomize/base/anyuid-scc-permission.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system:openshift:scc:anyuid +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:openshift:scc:anyuid +subjects: + - kind: ServiceAccount + name: exploit-iq-sa diff --git a/kustomize/base/argilla/argilla-route.yaml b/kustomize/base/argilla/argilla-route.yaml new file mode 100644 index 000000000..be5ccb4e6 --- /dev/null +++ b/kustomize/base/argilla/argilla-route.yaml @@ -0,0 +1,15 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: argilla + annotations: + haproxy.router.openshift.io/timeout: 5m +spec: + tls: + insecureEdgeTerminationPolicy: Redirect + termination: edge + port: + targetPort: 6900 + to: + kind: Service + name: argilla \ No newline at end of file diff --git a/kustomize/base/argilla/argilla-service.yaml b/kustomize/base/argilla/argilla-service.yaml new file mode 100644 index 000000000..cc9f2840b --- /dev/null +++ b/kustomize/base/argilla/argilla-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: argilla + labels: + app: argilla +spec: + selector: + app: morpheus-feedback-api + ports: + - protocol: TCP + port: 6900 + targetPort: 6900 + type: ClusterIP diff --git a/kustomize/base/argilla/argilla-user-feedback-pvc.yaml b/kustomize/base/argilla/argilla-user-feedback-pvc.yaml new file mode 100644 index 000000000..8a730ef7c --- /dev/null +++ b/kustomize/base/argilla/argilla-user-feedback-pvc.yaml @@ -0,0 +1,13 @@ + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: argilla-user-feedback-pvc + labels: + app: morpheus-feedback-api +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi \ No newline at end of file diff --git a/kustomize/base/argilla/configmap.yaml b/kustomize/base/argilla/configmap.yaml new file mode 100644 index 000000000..199835636 --- /dev/null +++ b/kustomize/base/argilla/configmap.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +data: + ARGILLA_HOME_PATH: "/var/lib/argilla" + ARGILLA_ELASTICSEARCH: "http://localhost:9200" + ARGILLA_DATABASE_URL: "postgresql+asyncpg://postgres:postgres@localhost:5432/argilla" + ARGILLA_REDIS_URL: "redis://localhost:6379/0" +kind: ConfigMap +metadata: + name: argilla-config \ No newline at end of file diff --git a/kustomize/base/argilla/deployment.yaml b/kustomize/base/argilla/deployment.yaml new file mode 100644 index 000000000..4854af9ad --- /dev/null +++ b/kustomize/base/argilla/deployment.yaml @@ -0,0 +1,116 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: morpheus-feedback-api + labels: + app: morpheus-feedback-api +spec: + replicas: 1 + selector: + matchLabels: + app: morpheus-feedback-api + strategy: + type: Recreate + template: + metadata: + labels: + app: morpheus-feedback-api + spec: + restartPolicy: Always + serviceAccountName: argilla + securityContext: + fsGroup: 1000 + containers: + - name: flask-service + image: quay.io/ecosystem-appeng/exploit-iq-feedback-api:latest + imagePullPolicy: Always + ports: + - containerPort: 5001 + env: + - name: ARGILLA_API_KEY + valueFrom: + secretKeyRef: + key: argilla_api_key + name: argilla-feedback-secret + - name: ARGILLA_WORKSPACE + value: rh-argilla + - name: ARGILLA_API_URL + value: 'http://argilla:6900' + - name: argilla-server + image: argilla/argilla-server:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 6900 + env: + - name: USERNAME + valueFrom: + secretKeyRef: + key: argilla_username + name: argilla-feedback-secret + - name: PASSWORD + valueFrom: + secretKeyRef: + key: argilla_password + name: argilla-feedback-secret + - name: API_KEY + valueFrom: + secretKeyRef: + key: argilla_api_key + name: argilla-feedback-secret + - name: WORKSPACE + value: default + envFrom: + - configMapRef: + name: argilla-config + - name: argilla-worker + image: argilla/argilla-server:latest + imagePullPolicy: IfNotPresent + command: ["bash","-c","python -m argilla_server worker --num-workers $BACKGROUND_NUM_WORKERS"] + envFrom: + - configMapRef: + name: argilla-config + env: + - name: BACKGROUND_NUM_WORKERS + value: "2" + - name: redis + image: redis:6 + imagePullPolicy: IfNotPresent + - name: elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch:8.17.0 + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /usr/share/elasticsearch/data + name: user-feedback-storage + subPath: elasticsearch + env: + - name: node.name + value: "elasticsearch" + - name: ES_JAVA_OPTS + value: "-Xms512m -Xmx512m" + - name: discovery.type + value: "single-node" + - name: cluster.name + value: "es-argilla-local" + - name: cluster.routing.allocation.disk.threshold_enabled + value: "false" + - name: xpack.security.enabled + value: "false" + - name: postgresql + image: postgres:13 + imagePullPolicy: IfNotPresent + env: + - name: POSTGRES_USER + value: postgres + - name: POSTGRES_PASSWORD + value: postgres + - name: POSTGRES_DB + value: argilla + volumeMounts: + - name: user-feedback-storage + mountPath: /var/lib/postgresql + subPath: postgres + serviceAccount: argilla + volumes: + - name: user-feedback-storage + persistentVolumeClaim: + claimName: argilla-user-feedback-pvc \ No newline at end of file diff --git a/kustomize/base/argilla/kustomization.yaml b/kustomize/base/argilla/kustomization.yaml new file mode 100644 index 000000000..f95625431 --- /dev/null +++ b/kustomize/base/argilla/kustomization.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - argilla-service.yaml + - configmap.yaml + - deployment.yaml + - rbac.yaml + - sa.yaml + - service.yaml + - argilla-route.yaml + - argilla-user-feedback-pvc.yaml + +secretGenerator: +- name: argilla-feedback-secret + envs: + - feedback_secret.env \ No newline at end of file diff --git a/kustomize/base/argilla/rbac.yaml b/kustomize/base/argilla/rbac.yaml new file mode 100644 index 000000000..62f935309 --- /dev/null +++ b/kustomize/base/argilla/rbac.yaml @@ -0,0 +1,21 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: argilla-anyuid-scc +rules: + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["anyuid"] + verbs: ["use"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: argilla-anyuid-scc +subjects: + - kind: ServiceAccount + name: argilla +roleRef: + kind: Role + name: argilla-anyuid-scc + apiGroup: rbac.authorization.k8s.io diff --git a/kustomize/base/argilla/sa.yaml b/kustomize/base/argilla/sa.yaml new file mode 100644 index 000000000..44107321d --- /dev/null +++ b/kustomize/base/argilla/sa.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: argilla diff --git a/kustomize/base/argilla/service.yaml b/kustomize/base/argilla/service.yaml new file mode 100644 index 000000000..545316330 --- /dev/null +++ b/kustomize/base/argilla/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: morpheus-feedback-api + labels: + app: morpheus-feedback-api +spec: + selector: + app: morpheus-feedback-api + ports: + - protocol: TCP + port: 5001 + targetPort: 5001 + type: ClusterIP diff --git a/kustomize/base/ca-certs/.gitkeep b/kustomize/base/ca-certs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/kustomize/base/ca-certs/README.md b/kustomize/base/ca-certs/README.md new file mode 100644 index 000000000..6b6ea70b6 --- /dev/null +++ b/kustomize/base/ca-certs/README.md @@ -0,0 +1,22 @@ +# Git SSL CA Certificates Directory + +This directory contains Certificate Authority (CA) certificates for Git SSL verification. + +## Purpose + +If your Git server uses certificates signed by a custom CA, place the CA certificate files in this directory. The deployment process creates a ConfigMap from these certificates and mounts it to the ExploitIQ pod. + +## Instructions + +Refer to the [Configuring Git SSL Certificate Authority for Custom CAs](../../README.md#configuring-git-ssl-certificate-authority-for-custom-cas) section in the main deployment documentation for detailed instructions on downloading and configuring CA certificates. + +## Red Hat Internal Access + +For Red Hat internal Git repositories (gitlab.cee.redhat.com), download: + +- Root CA: +- Intermediate CA: + +## Security + +**Do not commit certificate files to version control.** These files are gitignored to prevent accidental exposure of internal CA certificates in public repositories. diff --git a/kustomize/base/excludes.json b/kustomize/base/excludes.json new file mode 100644 index 000000000..9876ce1de --- /dev/null +++ b/kustomize/base/excludes.json @@ -0,0 +1,70 @@ +{ + "Go": [ + "go.mod", + "go.sum", + "**/*test*.go" + ], + "Java": [ + "target/**/*", + "build/**/*", + "*.class", + ".gradle/**/*", + ".mvn/**/*", + ".gitignore", + "test/**/*", + "tests/**/*", + "src/test/**/*", + "pom.xml", + "build.gradle" + ], + "C": [ + "**/tests/**/*", + "**/test/**/*", + "**/demos/**/*", + "**/examples/**/*", + "**/samples/**/*", + "**/doc/**/*", + "**/docs/**/*" + ], + "JavaScript": [ + "dist/**/*", + "build/**/*", + "test/**/*", + "tests/**/*", + "example/**/*", + "examples/**/*", + "package.json", + "package-lock.json", + "yarn.lock" + ], + "TypeScript": [ + "node_modules/**/*", + "dist/**/*", + "build/**/*", + "test/**/*", + "tests/**/*", + "example/**/*", + "examples/**/*", + "package.json", + "package-lock.json", + "yarn.lock" + ], + "Python": [ + "tests/**/*", + "test/**/*", + "venv/**/*", + ".venv/**/*", + "env/**/*", + "build/**/*", + "dist/**/*", + ".mypy_cache/**/*", + ".pytest_cache/**/*", + "__pycache__/**/*", + "*.pyc", + "*.pyo", + "*.pyd", + "requirements.txt", + "Pipfile", + "Pipfile.lock" + ] +} \ No newline at end of file diff --git a/kustomize/base/exploit-iq-config.yml b/kustomize/base/exploit-iq-config.yml new file mode 100644 index 000000000..e5a9363bd --- /dev/null +++ b/kustomize/base/exploit-iq-config.yml @@ -0,0 +1,335 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +general: + front_end: + _type: fastapi + endpoints: + - path: /health + method: GET + description: Perform a health check. + function_name: health_check + use_uvloop: true + telemetry: + tracing: + phoenix: + _type: phoenix + endpoint: ${OTEL_TRACES_ENDPOINT:-http://localhost:6006/v1/traces} + project: cve_agent + +functions: + cve_clone_and_deps: + _type: cve_clone_and_deps + base_git_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}git + base_pickle_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}pickle + base_rpm_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}rpms + cve_segmentation: + _type: cve_segmentation + agent_name: cve_agent_executor + embedder_name: nim_embedder + base_git_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}git + base_vdb_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}vdb + base_code_index_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}code_index + base_pickle_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}pickle + ignore_code_embedding: true + cve_fetch_intel: + _type: cve_fetch_intel + rpm_user_type: ${RPM_USER_TYPE:-internal} + retry_on_client_errors: false + intel_plugin_config: + plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin + plugin_config: + source: Product Security research + endpoint: CALLBACK_URL_PLACEHOLDER/api/v1/vulnerabilities/{vuln_id}/comments + token_path: /var/run/secrets/kubernetes.io/serviceaccount/token + verify_path: /app/certs/service-ca.crt + + cve_process_sbom: + _type: cve_process_sbom + cve_verify_vuln_package: + _type: cve_verify_vuln_package + skip: false + base_git_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}git + llm_name: checklist_llm + cve_checklist: + _type: cve_checklist + llm_name: checklist_llm + Call Chain Analyzer: + _type: transitive_code_search + enable_transitive_search: true + Function Caller Finder: + _type: calling_function_name_extractor + enable_functions_usage_search: true + Function Locator: + _type: package_and_function_locator + Function Library Version Finder: + _type: calling_function_library_version_finder + Code Semantic Search: + _type: local_vdb_retriever + embedder_name: nim_embedder + llm_name: code_vdb_retriever_llm + vdb_type: code + return_source_documents: false + Docs Semantic Search: + _type: local_vdb_retriever + embedder_name: nim_embedder + llm_name: doc_vdb_retriever_llm + vdb_type: doc + return_source_documents: false + Code Keyword Search: + _type: lexical_code_search + top_k: 5 + Source Grep: + _type: source_grep + base_checker_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}checker + max_results: 50 + context_lines: 2 + CVE Web Search: + _type: serp_wrapper + max_retries: 5 + Container Analysis Data: + _type: container_image_analysis_data + Configuration Scanner: + _type: configuration_scanner + max_results: 15 + context_lines: 5 + Import Usage Analyzer: + _type: import_usage_analyzer + max_files: 20 + cve_agent_executor: + _type: cve_agent_executor + llm_name: cve_agent_executor_llm + tool_names: + - Code Semantic Search + - Docs Semantic Search + - Code Keyword Search + - CVE Web Search + - Call Chain Analyzer + - Function Caller Finder + - Function Locator + - Function Library Version Finder + - Configuration Scanner + - Import Usage Analyzer + max_concurrency: null + max_iterations: 10 + prompt_examples: false + replace_exceptions: true + replace_exceptions_value: "I do not have a definitive answer for this checklist item." + return_intermediate_steps: false +# transitive_search_tool_enabled: false + cve_web_search_enabled: true + uber_jar_file_threshold: 600 + verbose: false + cve_generate_cvss: + _type: cve_generate_cvss + skip: true + llm_name: generate_cvss_llm + tool_names: + - Code Semantic Search + - Docs Semantic Search + - Code Keyword Search + - Container Analysis Data + max_concurrency: null + max_iterations: 10 + prompt_examples: true + replace_exceptions: false + replace_exceptions_value: "Failed to generate CVSS for this analysis." + return_intermediate_steps: false + verbose: false + cve_summarize: + _type: cve_summarize + llm_name: summarize_llm + cve_justify: + _type: cve_justify + llm_name: justify_llm + cve_generate_vex: + _type: cve_generate_vex + skip: false + # vex_format: csaf + cve_http_output: + _type: cve_http_output + url: CALLBACK_URL_PLACEHOLDER + endpoint: /api/v1/reports + auth_type: bearer + token_path: /var/run/secrets/kubernetes.io/serviceaccount/token + verify_path: /app/certs/service-ca.crt + enable_mlops: ${ENABLE_MLOPS:-false} + mlops_config: + mlops_url: http://localhost:8080 + auth_type: "bearer" + token_path: "/var/run/secrets/kubernetes.io/serviceaccount/token" + verify_path: "/app/certs/service-ca.crt" + enable_verify: true + cve_calculate_intel_score: + _type: cve_calculate_intel_score + llm_name: intel_source_score_llm + generate_intel_score: true + intel_low_score: 51 + insist_analysis: false + cve_source_acquisition: + _type: cve_source_acquisition + base_git_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}git + base_pickle_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}pickle + base_rpm_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}rpms + base_checker_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}checker + rpm_user_type: ${RPM_USER_TYPE:-internal} + cve_checker_segmentation: + _type: cve_checker_segmentation + base_checker_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}checker + base_code_index_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}code_index + cve_package_code_agent: + _type: cve_package_code_agent + llm_name: cve_agent_executor_llm + base_checker_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}checker + base_code_index_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}code_index + rpm_user_type: ${RPM_USER_TYPE:-internal} + tool_names: + - Source Grep + - Code Keyword Search + cve_checker_report: + _type: cve_checker_report + llm_name: cve_agent_executor_llm + base_checker_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}checker + cve_build_agent: + _type: cve_build_agent + llm_name: cve_agent_executor_llm + base_checker_dir: ${EXPLOIT_IQ_DATA_DIR:-/exploit-iq-data/}checker + max_iterations: 10 + tool_names: + - Source Grep + - Code Keyword Search + cve_fetch_patches: + _type: cve_fetch_patches + health_check: + _type: health_check + +llms: + checklist_llm: + _type: ${LLM_TYPE_CHECKLIST:-nim} + api_key: ${LLM_API_KEY_CHECKLIST:-"EMPTY"} + base_url: ${CHECKLIST_LLM_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CHECKLIST_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + code_vdb_retriever_llm: + _type: ${LLM_TYPE_VDB_CODE_RETRIEVER:-nim} + api_key: ${LLM_API_KEY_CODE_VDB_RETRIEVER:-"EMPTY"} + base_url: ${CODE_VDB_RETRIEVER_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CODE_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + doc_vdb_retriever_llm: + _type: ${LLM_TYPE_VDB_DOC_RETRIEVER:-nim} + api_key: ${LLM_API_KEY_DOC_VDB_RETRIEVER:-"EMPTY"} + base_url: ${DOC_VDB_RETRIEVER_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${DOC_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + cve_agent_executor_llm: + _type: ${LLM_TYPE_AGENT_EXECUTOR:-nim} + api_key: ${LLM_API_KEY_AGENT_EXECUTOR:-"EMPTY"} + base_url: ${AGENT_EXECUTOR_LLM_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${AGENT_EXECUTOR_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + generate_cvss_llm: + _type: ${LLM_TYPE_GENERATE_CVSS:-nim} + api_key: ${LLM_API_KEY_GENERATE_CVSS:-"EMPTY"} + base_url: ${GENERATE_CVSS_LLM_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + summarize_llm: + _type: ${LLM_TYPE_SUMMARIZE:-nim} + api_key: ${LLM_API_KEY_SUMMARIZE:-"EMPTY"} + base_url: ${SUMMARIZE_LLM_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${SUMMARIZE_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + justify_llm: + _type: ${LLM_TYPE_JUSTIFY:-nim} + api_key: ${LLM_API_KEY_JUSTIFY:-"EMPTY"} + base_url: ${JUSTIFY_LLM_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + + intel_source_score_llm: + _type: ${LLM_TYPE_INTEL_SOURCE_SCORE:-nim} + api_key: ${LLM_API_KEY_INTEL_SOURCE_SCORE:-"EMPTY"} + base_url: ${INTEL_SOURCE_SCORE_LLM_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${INTEL_SOURCE_SCORE_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + +embedders: + nim_embedder: + _type: nim + base_url: ${NIM_EMBED_BASE_URL:-https://integrate.api.nvidia.com/v1} + model_name: ${EMBEDDER_MODEL_NAME:-nvidia/nv-embedqa-e5-v5} + truncate: END + max_batch_size: 128 + +workflow: + _type: cve_agent + cve_clone_and_deps_name: cve_clone_and_deps + cve_segmentation_name: cve_segmentation + cve_fetch_intel_name: cve_fetch_intel + cve_calculate_intel_score_name: cve_calculate_intel_score + cve_process_sbom_name: cve_process_sbom + cve_verify_vuln_package_name: cve_verify_vuln_package + cve_checklist_name: cve_checklist + cve_agent_executor_name: cve_agent_executor + cve_generate_cvss_name: cve_generate_cvss + cve_generate_vex_name: cve_generate_vex + cve_summarize_name: cve_summarize + cve_justify_name: cve_justify + cve_output_config_name: cve_http_output + cve_source_acquisition_name: cve_source_acquisition + cve_checker_segmentation_name: cve_checker_segmentation + cve_package_code_agent_name: cve_package_code_agent + cve_checker_report_name: cve_checker_report + cve_build_agent_name: cve_build_agent + cve_fetch_patches_name: cve_fetch_patches + +eval: + general: + output_dir: ./.tmp/eval/cve_agent + dataset: + _type: json + file_path: data/eval_datasets/eval_dataset.json + + profiler: + token_uniqueness_forecast: true + workflow_runtime_forecast: true + compute_llm_metrics: true + csv_exclude_io_text: true + prompt_caching_prefixes: + enable: true + min_frequency: 0.1 + bottleneck_analysis: + # Can also be simple_stack + enable_nested_stack: true + concurrency_spike_analysis: + enable: true + spike_threshold: 7 diff --git a/kustomize/base/exploit_iq_client.yaml b/kustomize/base/exploit_iq_client.yaml new file mode 100644 index 000000000..7829e0e90 --- /dev/null +++ b/kustomize/base/exploit_iq_client.yaml @@ -0,0 +1,192 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: exploit-iq-client + labels: + app: exploit-iq + component: exploit-iq-client +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app: exploit-iq + component: exploit-iq-client + template: + metadata: + labels: + app: exploit-iq + component: exploit-iq-client + spec: + imagePullSecrets: [] + serviceAccountName: exploit-iq-client-sa + containers: + - name: exploit-iq-client + args: + - ./application + - -Dquarkus.http.host=0.0.0.0 + - -Dquarkus.log.category."com.redhat.ecosystemappeng.morpheus".level=DEBUG + image: quay.io/ecosystem-appeng/agent-morpheus-client:latest + imagePullPolicy: Always + ports: + - name: http + protocol: TCP + containerPort: 8080 + env: + - name: QUARKUS_REST-CLIENT_MORPHEUS_URL + value: http://nginx-cache:8080/generate + - name: QUARKUS_MONGODB_HOSTS + value: exploit-iq-client-db:27017 + - name: QUARKUS_MONGODB_DATABASE + value: exploit-iq-client + - name: QUARKUS_MONGODB_CREDENTIALS_USERNAME + valueFrom: + secretKeyRef: + name: mongodb-credentials + key: exploit-iq-user + - name: QUARKUS_MONGODB_CREDENTIALS_PASSWORD + valueFrom: + secretKeyRef: + name: mongodb-credentials + key: exploit-iq-password + - name: QUARKUS_MONGODB_CREDENTIALS_AUTH-SOURCE + value: exploit-iq-client + - name: QUARKUS_MONGODB_APPLICATION_NAME + value: exploit-iq-client + - name: OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: oauth-client-secret + key: client-secret + - name: OPENSHIFT_DOMAIN + valueFrom: + secretKeyRef: + name: oauth-client-secret + key: openshift-domain + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: QUARKUS_HTTP_SSL_CERTIFICATE_FILES + value: /etc/tls/tls.crt + - name: QUARKUS_HTTP_SSL_CERTIFICATE_KEY-FILES + value: /etc/tls/tls.key + - name: QUARKUS_HTTP_SSL-PORT + value: "8443" + - name: QUARKUS_HTTP_INSECURE-REQUESTS + value: disabled + - name: QUARKUS_HTTP_SSL_CERTIFICATE_RELOAD-PERIOD + value: 30m + - name: MORPHEUS_UI_INCLUDES_PATH + value: /config/includes.json + - name: MORPHEUS_UI_EXCLUDES_PATH + value: /config/excludes.json + - name: DOCKER_CONFIG + value: /tmp/.docker + - name: GITHUB_TOKEN + valueFrom: + secretKeyRef: + name: exploit-iq-secret + key: ghsa_api_key + - name: CREDENTIAL_ENCRYPTION_KEY + valueFrom: + secretKeyRef: + name: credential-encryption-key + key: credential-encryption-key + + volumeMounts: + - name: tls-certs + mountPath: /etc/tls + readOnly: true + + - name: includes-excludes + mountPath: /config + readOnly: true + - name: docker-config + readOnly: true + mountPath: /tmp/.docker + + volumes: + - name: tls-certs + secret: + secretName: exploit-iq-client-tls + - name: includes-excludes + configMap: + name: includes-excludes-json + - name: docker-config + secret: + secretName: image-registry-credentials + items: + - key: .dockerconfigjson + path: config.json + defaultMode: 420 +--- +apiVersion: v1 +kind: Service +metadata: + name: exploit-iq-client + labels: + app: exploit-iq + component: exploit-iq-client + annotations: + service.beta.openshift.io/serving-cert-secret-name: exploit-iq-client-tls +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + - name: metrics + protocol: TCP + port: 9000 + targetPort: 9000 + selector: + app: exploit-iq + component: exploit-iq-client +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: exploit-iq-client + annotations: + haproxy.router.openshift.io/timeout: 5m +spec: + tls: + insecureEdgeTerminationPolicy: Redirect + termination: reencrypt + port: + targetPort: https + to: + kind: Service + name: exploit-iq-client +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: exploit-iq-client-swagger-ui + annotations: + haproxy.router.openshift.io/timeout: 5m +spec: + tls: + insecureEdgeTerminationPolicy: Redirect + port: + targetPort: metrics + to: + kind: Service + name: exploit-iq-client +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: exploit-iq +spec: + endpoints: + - interval: 30s + path: /q/metrics + port: metrics + scheme: http + selector: + matchLabels: + app: exploit-iq + component: exploit-iq-client diff --git a/kustomize/base/exploit_iq_client_db.yaml b/kustomize/base/exploit_iq_client_db.yaml new file mode 100644 index 000000000..e7aeebe42 --- /dev/null +++ b/kustomize/base/exploit_iq_client_db.yaml @@ -0,0 +1,93 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: exploit-iq-client-db + labels: + app: exploit-iq + component: exploit-iq-client-db +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app: exploit-iq + component: exploit-iq-client-db + template: + metadata: + labels: + app: exploit-iq + component: exploit-iq-client-db + spec: + containers: + - name: mongodb + image: mongodb/mongodb-community-server:8.0.3-ubi8 + imagePullPolicy: IfNotPresent + env: + - name: MONGODB_INITDB_ROOT_USERNAME + valueFrom: + secretKeyRef: + name: mongodb-credentials + key: admin-user + - name: MONGODB_INITDB_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: mongodb-credentials + key: admin-password + - name: MONGODB_APP_USERNAME + valueFrom: + secretKeyRef: + name: mongodb-credentials + key: exploit-iq-user + - name: MONGODB_APP_PASSWORD + valueFrom: + secretKeyRef: + name: mongodb-credentials + key: exploit-iq-password + ports: + - name: db + protocol: TCP + containerPort: 27017 + volumeMounts: + - name: data + mountPath: /data/db + - name: init-scripts + mountPath: /docker-entrypoint-initdb.d + readOnly: true + volumes: + - name: data + persistentVolumeClaim: + claimName: exploit-iq-client-data + - name: init-scripts + configMap: + name: mongodb-init +--- +apiVersion: v1 +kind: Service +metadata: + name: exploit-iq-client-db + labels: + app: exploit-iq + component: exploit-iq-client-db +spec: + ports: + - name: db + port: 27017 + protocol: TCP + targetPort: 27017 + selector: + app: exploit-iq + component: exploit-iq-client-db +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: exploit-iq-client-data + labels: + purpose: persistent +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi diff --git a/kustomize/base/exploit_iq_service.yaml b/kustomize/base/exploit_iq_service.yaml new file mode 100644 index 000000000..2f99c7411 --- /dev/null +++ b/kustomize/base/exploit_iq_service.yaml @@ -0,0 +1,271 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: exploit-iq + labels: + app: exploit-iq + component: exploit-iq +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app: exploit-iq + component: exploit-iq + template: + metadata: + labels: + app: exploit-iq + component: exploit-iq + spec: + securityContext: + fsGroup: 1000 + imagePullSecrets: [] + serviceAccountName: exploit-iq-sa + containers: + - name: exploit-iq-phoenix-tracing + image: quay.io/ecosystem-appeng/agent-morpheus-rh:nat + imagePullPolicy: Always + workingDir: /workspace/ + args: + - "phoenix" + - "serve" + securityContext: + runAsUser: 0 + ports: + - name: tracing + protocol: TCP + containerPort: 6006 + resources: + limits: + memory: "2Gi" + cpu: "500m" + requests: + memory: "1Gi" + cpu: "100m" + - name: exploit-iq + image: quay.io/ecosystem-appeng/agent-morpheus-rh:nat + imagePullPolicy: Always + workingDir: /workspace/ + args: + - "nat" + - "--log-level" + - "debug" + - "serve" + - "--config_file=/configs/exploit-iq-config.yml" + - "--host" + - "0.0.0.0" + - "--port" + - "8080" + + securityContext: + runAsUser: 1001 + ports: + - name: http + protocol: TCP + containerPort: 8080 + livenessProbe: + failureThreshold: 24 + timeoutSeconds: 10 + periodSeconds: 20 + httpGet: + path: /health + port: http +# If still doing problems under heavy cpu loads and heavy concurrent workloads , kindly uncomment the tcpSocket block below, +# and comment out the httpGet group as a temporary workaround. +# Upgrading the agent to python version 3.14 will solve this issue as it will +# be a real parallel multi-threading environment without GIL ( Global Interpreter Lock) restrictions. +# tcpSocket: +# port: 8080 + + resources: + limits: + memory: "8Gi" + cpu: "1000m" + requests: + memory: "1Gi" + cpu: "1000m" + env: + - name: SERPAPI_API_KEY + valueFrom: + secretKeyRef: + key: serpapi_api_key + name: exploit-iq-secret + - name: NVIDIA_API_KEY + value: EXPLOIT_IQ + - name: OPENAI_API_KEY + value: EXPLOIT_IQ + - name: NGC_API_KEY + value: EXPLOIT_IQ + - name: REGISTRY_REDHAT_PASSWORD + valueFrom: + secretKeyRef: + key: registry_redhat_password + name: exploit-iq-secret + - name: REGISTRY_REDHAT_USERNAME + valueFrom: + secretKeyRef: + key: registry_redhat_username + name: exploit-iq-secret + - name: CVE_DETAILS_BASE_URL + value: http://nginx-cache:8080/cve-details + - name: CWE_DETAILS_BASE_URL + value: http://nginx-cache:8080/cwe-details + - name: DEPSDEV_BASE_URL + value: http://nginx-cache:8080/depsdev + - name: FIRST_BASE_URL + value: http://nginx-cache:8080/first + - name: GHSA_BASE_URL + value: http://nginx-cache:8080/ghsa + - name: NGC_API_BASE + value: http://nginx-cache:8080/nemo/v1 + - name: NIM_EMBED_BASE_URL + value: http://nginx-cache:8080/nim_embed/v1 + - name: NVD_BASE_URL + value: http://nginx-cache:8080/nvd + - name: NVIDIA_API_BASE + value: http://nginx-cache:8080/nim_llm/v1 + - name: RHSA_BASE_URL + value: http://nginx-cache:8080/rhsa + - name: SERPAPI_BASE_URL + value: http://nginx-cache:8080/serpapi + - name: UBUNTU_BASE_URL + value: http://nginx-cache:8080/ubuntu + - name: ENABLE_EXTENDED_JS_PARSERS + value: "True" + - name: EXPLOIT_IQ_DATA_DIR + value: /exploit-iq-data/ + - name: RPM_USER_TYPE + value: "internal" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: GOMODCACHE + value: /exploit-iq-package-cache/go/pkg/mod + - name: ENABLE_MLOPS + value: "true" + - name: CREDENTIAL_ENCRYPTION_KEY + valueFrom: + secretKeyRef: + name: credential-encryption-key + key: credential-encryption-key + - name: CLIENT_BACKEND_URL + value: https://exploit-iq-client.$(NAMESPACE).svc.cluster.local:8443 + - name: JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH + value: /maven-config/settings.xml + volumeMounts: + - name: config + mountPath: /configs + - name: maven-settings-config + mountPath: /maven-config + - name: cache + mountPath: /exploit-iq-data + - name: package-cache + mountPath: /exploit-iq-package-cache + - name: ca-bundle + mountPath: /app/certs + readOnly: true + - name: git-ca-bundle + mountPath: /app/git-ca-bundle + readOnly: true + volumes: + - name: config + configMap: + name: exploit-iq-config + - name: maven-settings-config + configMap: + name: exploit-iq-maven-settings-config + - name: cache + persistentVolumeClaim: + claimName: exploit-iq-data + - name: package-cache + persistentVolumeClaim: + claimName: exploit-iq-package-cache + - name: ca-bundle + configMap: + name: openshift-service-ca.crt + - name: git-ca-bundle + configMap: + name: git-ca-bundle + optional: true +--- +apiVersion: v1 +kind: Service +metadata: + name: exploit-iq + labels: + app: exploit-iq + component: exploit-iq +spec: + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: exploit-iq + component: exploit-iq +--- +apiVersion: v1 +kind: Service +metadata: + name: exploit-iq-phoenix-tracing + labels: + app: exploit-iq + component: exploit-iq-tracing +spec: + ports: + - name: tracing + port: 6006 + protocol: TCP + targetPort: 6006 + selector: + app: exploit-iq + component: exploit-iq +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + app: exploit-iq + component: exploit-iq-tracing + name: exploit-iq-phoenix-tracing +spec: + port: + targetPort: tracing + to: + kind: Service + name: exploit-iq-phoenix-tracing + weight: 100 +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: exploit-iq-data + labels: + purpose: persistent +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: exploit-iq-package-cache + labels: + purpose: package-cache +spec: + # ReadWriteOncePod is used instead of ReadWriteOnce to avoid long SELinux relabeling + # when volume contains large number of files (tens of thousands or more). + # This prevents pod startup failures and excessive startup times. + # See: https://access.redhat.com/solutions/6221251 and https://access.redhat.com/solutions/7079407 + accessModes: + - ReadWriteOncePod + resources: + requests: + storage: 200Gi diff --git a/kustomize/base/exploitiq_mcp_server.yaml b/kustomize/base/exploitiq_mcp_server.yaml new file mode 100644 index 000000000..5b48f5213 --- /dev/null +++ b/kustomize/base/exploitiq_mcp_server.yaml @@ -0,0 +1,171 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: exploitiq-mcp-server-sa + labels: + app: exploit-iq + component: exploitiq-mcp-server +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: exploitiq-mcp-server-anyuid-scc + labels: + app: exploit-iq + component: exploitiq-mcp-server +rules: + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["anyuid"] + verbs: ["use"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: exploitiq-mcp-server-anyuid-scc + labels: + app: exploit-iq + component: exploitiq-mcp-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: exploitiq-mcp-server-anyuid-scc +subjects: + - kind: ServiceAccount + name: exploitiq-mcp-server-sa +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: exploitiq-mcp-server + labels: + app: exploit-iq + component: exploitiq-mcp-server +spec: + strategy: + type: RollingUpdate + replicas: 1 + selector: + matchLabels: + app: exploit-iq + component: exploitiq-mcp-server + template: + metadata: + labels: + app: exploit-iq + component: exploitiq-mcp-server + spec: + serviceAccountName: exploitiq-mcp-server-sa + imagePullSecrets: [] + containers: + - name: exploitiq-mcp-server + securityContext: + runAsNonRoot: true + image: quay.io/ecosystem-appeng/exploitiq-mcp-server:latest + imagePullPolicy: Always + ports: + - name: http + protocol: TCP + containerPort: 3000 + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: EXPLOITIQ_CLIENT_URL + value: https://exploit-iq-client.$(POD_NAMESPACE).svc:8443 + - name: EXPLOITIQ_MCP_PORT + value: "3000" + - name: NODE_EXTRA_CA_CERTS + value: /etc/tls/tls.crt + - name: EXPLOITIQ_READ_ONLY + value: "true" + - name: EXPLOITIQ_REDACT_RESPONSES + value: "true" + - name: OPENSHIFT_DOMAIN + valueFrom: + secretKeyRef: + name: oauth-client-secret + key: openshift-domain + - name: EXPLOITIQ_OAUTH_ISSUER_URL + value: https://oauth-openshift.apps.$(OPENSHIFT_DOMAIN) + - name: EXPLOITIQ_OAUTH_CLIENT_ID + value: exploitiq-mcp-server + - name: EXPLOITIQ_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: oauth-client-secret + key: client-secret + - name: EXPLOITIQ_MCP_PUBLIC_URL + value: https://exploitiq-mcp-server-$(POD_NAMESPACE).apps.$(OPENSHIFT_DOMAIN) + # Restrict MCP access to specific OpenShift groups (comma-separated). + # Checked during OAuth callback (unauthorized users see a 403 in + # the browser before a token is issued) and on every /mcp request. + # When unset, any authenticated user can access the MCP server. + - name: EXPLOITIQ_ALLOWED_GROUPS + value: "exploitiq-users" + startupProbe: + tcpSocket: + port: 3000 + initialDelaySeconds: 2 + periodSeconds: 5 + failureThreshold: 10 + readinessProbe: + httpGet: + path: /readyz + port: 3000 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: 3000 + periodSeconds: 30 + resources: + requests: + memory: 128Mi + cpu: 100m + limits: + memory: 256Mi + cpu: 500m + volumeMounts: + - name: service-ca + mountPath: /etc/tls/tls.crt + subPath: service-ca.crt + readOnly: true + volumes: + - name: service-ca + configMap: + name: openshift-service-ca.crt +--- +apiVersion: v1 +kind: Service +metadata: + name: exploitiq-mcp-server + labels: + app: exploit-iq + component: exploitiq-mcp-server +spec: + ports: + - name: http + port: 3000 + protocol: TCP + targetPort: 3000 + selector: + app: exploit-iq + component: exploitiq-mcp-server +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: exploitiq-mcp-server + annotations: + haproxy.router.openshift.io/timeout: 5m +spec: + tls: + insecureEdgeTerminationPolicy: Redirect + termination: edge + port: + targetPort: http + to: + kind: Service + name: exploitiq-mcp-server \ No newline at end of file diff --git a/kustomize/base/image-pull-secret-permission.yaml b/kustomize/base/image-pull-secret-permission.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/kustomize/base/image-registry-credentials.env b/kustomize/base/image-registry-credentials.env new file mode 100644 index 000000000..1a73678d8 --- /dev/null +++ b/kustomize/base/image-registry-credentials.env @@ -0,0 +1,3 @@ +{ + "auths": {} +} \ No newline at end of file diff --git a/kustomize/base/includes.json b/kustomize/base/includes.json new file mode 100644 index 000000000..3b96ed8b3 --- /dev/null +++ b/kustomize/base/includes.json @@ -0,0 +1,75 @@ +{ + "Go": [ + "**/*.go" + ], + "Python": [ + "**/*.py", + "pyproject.toml", + "setup.py", + "setup.cfg" + ], + "Java": [ + "**/*.java", + "settings.gradle", + "src/main/**/*" + ], + "JavaScript": [ + "**/*.js", + "**/*.mjs", + "**/*.cjs", + "webpack.config.js", + "rollup.config.js", + "babel.config.js", + ".babelrc", + ".eslintrc.js", + ".eslintrc.json", + "tsconfig.json", + "*.config.js", + "*.config.json", + "public/**/*", + "src/**/*" + ], + "TypeScript": [ + "**/*.ts", + "**/*.tsx", + "tsconfig.json", + "tsconfig.*.json", + "webpack.config.js", + "webpack.config.ts", + "rollup.config.js", + "rollup.config.ts", + "babel.config.js", + ".babelrc", + ".eslintrc.js", + ".eslintrc.json", + "*.config.js", + "*.config.ts", + "*.json", + "src/**/*", + "public/**/*", + "assets/**/*" + ], + "C":[ + "**/*.c", + "**/*.h" + ], + "Dockerfile": [ + "Dockerfile*", + "docker-compose.yml", + "*.dockerfile", + "*.dockerignore", + "docker-compose.*.yml", + "*.sh", + "scripts/**/*", + "*.env", + "*.yaml", + "*.yml", + "*.json", + "config/**/*", + "conf.d/**/*" + ], + "Docs": [ + "**/*.md", + "docs/**/*.rst" + ] +} \ No newline at end of file diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml new file mode 100644 index 000000000..26f9fc41a --- /dev/null +++ b/kustomize/base/kustomization.yaml @@ -0,0 +1,90 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - service-account.yaml + - anyuid-scc-permission.yaml + - nginx.yaml + - exploit_iq_service.yaml + - exploit_iq_client_db.yaml + - exploit_iq_client.yaml + - exploitiq_mcp_server.yaml + - mongodb_init_configmap.yaml + - mongodb_network_policy.yaml + - argilla + + +commonAnnotations: + deployment-variant: base + +generatorOptions: + disableNameSuffixHash: false + +secretGenerator: + - name: exploit-iq-secret + envs: + - secrets.env + + - name: oauth-client-secret + envs: + - oauth-secrets.env + options: + disableNameSuffixHash: true + + - name: mongodb-credentials + envs: + - mongodb-credentials.env + + - name: image-registry-credentials + files: + - .dockerconfigjson=image-registry-credentials.env + type: kubernetes.io/dockerconfigjson + options: + disableNameSuffixHash: true + + - name: credential-encryption-key + envs: + - credential-encryption-key.env + options: + disableNameSuffixHash: true + +configMapGenerator: + - name: nginx-cache-config + files: + - nginx.conf=nginx/nginx_cache.conf + - name: nginx-cache-variables + files: + - nginx/templates/variables/template-variables.conf.template + - name: nginx-cache-routes + files: + - nginx/templates/routes/intel.conf.template + - nginx/templates/routes/nemo.conf.template + - nginx/templates/routes/nim.conf.template + - nginx/templates/routes/nvidia.conf.template + - nginx/templates/routes/openai.conf.template + - name: exploit-iq-config + files: + - exploit-iq-config.yml + + - name: includes-excludes-json + files: + - excludes.json + - includes.json + - name: exploit-iq-maven-settings-config + files: + - settings.xml + + - name: git-ca-bundle + files: + - ca-certs/ca-bundle.crt + options: + disableNameSuffixHash: true +images: + - name: quay.io/ecosystem-appeng/agent-morpheus-rh + newTag: latest + + - name: quay.io/ecosystem-appeng/agent-morpheus-client + newTag: latest + + - name: quay.io/ecosystem-appeng/exploitiq-mcp-server + newTag: latest diff --git a/kustomize/base/mongodb_init_configmap.yaml b/kustomize/base/mongodb_init_configmap.yaml new file mode 100644 index 000000000..6bf5a7031 --- /dev/null +++ b/kustomize/base/mongodb_init_configmap.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: mongodb-init +data: + create-app-user.js: | + db = db.getSiblingDB('exploit-iq-client'); + + // Get exploit-iq credentials from environment + const appUser = process.env.MONGODB_APP_USERNAME; + const appPassword = process.env.MONGODB_APP_PASSWORD; + + if (!appUser || !appPassword) { + print('ERROR: MONGODB_APP_USERNAME or MONGODB_APP_PASSWORD environment variable not set'); + quit(1); + } + + db.createUser({ + user: appUser, + pwd: appPassword, + roles: [ + { role: 'readWrite', db: 'exploit-iq-client' } + ] + }); + + print(`Successfully created ${appUser} with readWrite role on exploit-iq-client database`); + diff --git a/kustomize/base/mongodb_network_policy.yaml b/kustomize/base/mongodb_network_policy.yaml new file mode 100644 index 000000000..547e465fc --- /dev/null +++ b/kustomize/base/mongodb_network_policy.yaml @@ -0,0 +1,33 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: mongodb-network-policy + labels: + app: exploit-iq + component: exploit-iq-client-db +spec: + podSelector: + matchLabels: + app: exploit-iq + component: exploit-iq-client-db + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + app: exploit-iq + component: exploit-iq-client + ports: + - protocol: TCP + port: 27017 + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: openshift-dns + ports: + - protocol: UDP + port: 5353 + diff --git a/kustomize/base/nginx.yaml b/kustomize/base/nginx.yaml new file mode 100644 index 000000000..c4bbd0521 --- /dev/null +++ b/kustomize/base/nginx.yaml @@ -0,0 +1,174 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-cache + labels: + app: exploit-iq + component: nginx-cache +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app: exploit-iq + component: nginx-cache + template: + metadata: + labels: + app: exploit-iq + component: nginx-cache + spec: + imagePullSecrets: [] + initContainers: + - name: render-nginx-config + image: registry.redhat.io/ubi9/python-312@sha256:55ddb67dcfc6c7ec072fce58753910b290c2903b10cd71d0d64c631d44ef4ab0 + command: ["python3", "-c"] + args: + - | + import os, sys, string + from pathlib import Path + + required = [ + "SERPAPI_API_KEY", "NVIDIA_API_KEY", "OPENAI_API_KEY", + "NGC_API_KEY", "NGC_ORG_ID", "NGINX_UPSTREAM_NVAI", + "NGINX_UPSTREAM_NIM_LLM", "NGINX_UPSTREAM_NIM_EMBED", + "NGINX_UPSTREAM_OPENAI", + ] + missing = [v for v in required if v not in os.environ] + if missing: + print(f"missing required variables: {', '.join(missing)}", file=sys.stderr) + sys.exit(1) + + def render_dir(src_dir, dst_dir): + src, dst = Path(src_dir), Path(dst_dir) + dst.mkdir(parents=True, exist_ok=True) + for tmpl in src.glob("*.conf.template"): + out = dst / tmpl.stem + out.write_text(string.Template(tmpl.read_text()).safe_substitute(os.environ)) + print(f"rendered: {tmpl} -> {out}") + + render_dir("/templates/routes", "/generated/routes") + render_dir("/templates/variables", "/generated/variables") + env: + - name: SERPAPI_API_KEY + valueFrom: + secretKeyRef: + key: serpapi_api_key + name: exploit-iq-secret + - name: NVIDIA_API_KEY + valueFrom: + secretKeyRef: + key: nvidia_api_key + name: exploit-iq-secret + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + key: nvidia_api_key + name: exploit-iq-secret + - name: NGC_API_KEY + value: "" + - name: NGC_ORG_ID + value: "" + - name: NGINX_UPSTREAM_NVAI + value: https://api.nvcf.nvidia.com + - name: NGINX_UPSTREAM_NIM_LLM + value: https://integrate.api.nvidia.com + - name: NGINX_UPSTREAM_NIM_EMBED + value: http://nim-embed.exploit-iq-models.svc.cluster.local:8000 + - name: NGINX_UPSTREAM_OPENAI + value: https://integrate.api.nvidia.com + volumeMounts: + - name: routes + mountPath: /templates/routes + readOnly: true + - name: variables + mountPath: /templates/variables + readOnly: true + - name: extra-config + mountPath: /generated + containers: + - name: nginx + image: registry.access.redhat.com/hi/nginx:1.30.3 + imagePullPolicy: IfNotPresent + ports: + - name: http + protocol: TCP + containerPort: 8080 + resources: + limits: + memory: "256Mi" + cpu: "250m" + requests: + memory: "100Mi" + cpu: "100m" + volumeMounts: + - name: config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + readOnly: true + - name: extra-config + mountPath: /etc/nginx/conf.d + - name: cache + mountPath: /server_cache + volumes: + - name: config + configMap: + name: nginx-cache-config + - name: extra-config + emptyDir: {} + - name: routes + configMap: + name: nginx-cache-routes + - name: variables + configMap: + name: nginx-cache-variables + - name: cache + persistentVolumeClaim: + claimName: nginx-cache-server +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-cache + labels: + app: exploit-iq + component: nginx-cache +spec: + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: exploit-iq + component: nginx-cache +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: exploit-iq + annotations: + haproxy.router.openshift.io/timeout: 5m +spec: + tls: + insecureEdgeTerminationPolicy: Redirect + termination: edge + port: + targetPort: 8080 + to: + kind: Service + name: nginx-cache +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nginx-cache-server + labels: + purpose: persistent +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Gi diff --git a/kustomize/base/nginx/nginx_cache.conf b/kustomize/base/nginx/nginx_cache.conf new file mode 100644 index 000000000..4d7385b84 --- /dev/null +++ b/kustomize/base/nginx/nginx_cache.conf @@ -0,0 +1,116 @@ +pid /tmp/nginx.pid; + +worker_processes 1; + +events { + worker_connections 1024; +} + +http { + proxy_ssl_server_name on; + + proxy_cache_path /server_cache/llm levels=1:2 keys_zone=llm_cache:10m max_size=20g inactive=14d use_temp_path=off; + + proxy_cache_path /server_cache/intel levels=1:2 keys_zone=intel_cache:10m max_size=20g inactive=14d use_temp_path=off; + + log_format upstream_time '$remote_addr - $remote_user [$time_local] ' + '"$request_method $loggable_request_uri $server_protocol" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent"' + 'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"'; + + log_format cache_log '[$time_local] traceId: $http_traceId - ($upstream_cache_status) "$request_method $loggable_request_uri $server_protocol" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present'; + + log_format no_cache_log '[$time_local] traceId: $http_traceId - (BYPASSED) "$request_method $loggable_request_uri $server_protocol" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present'; + + log_format mirror_log '[$time_local] traceId: $http_traceId - (MIRROR) "$request_method $loggable_request_uri $server_protocol" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present'; + + log_format nvai_cache_log '[$time_local] traceId: $http_traceId - ($upstream_cache_status) "$request_method $loggable_request_uri $server_protocol" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present. Final Auth: $http_authorization_present'; + + include /etc/nginx/conf.d/variables/*.conf; + + map $http_cache_control $cache_bypass { + no-cache 1; + } + + # Log to stdout + access_log /dev/stdout cache_log; + + error_log /dev/stdout info; + + client_max_body_size 1G; + + server { + listen 8080; + server_name localhost; + + proxy_http_version 1.1; + + # Headers to Add + # proxy_set_header Host $host; + proxy_set_header Connection ''; + + # Headers to Remove + proxy_ignore_headers Cache-Control; + proxy_ignore_headers "Set-Cookie"; + proxy_hide_header "Set-Cookie"; + + # Proxy Buffer Config + proxy_busy_buffers_size 1024k; + proxy_buffers 4 512k; + proxy_buffer_size 1024k; + + # Proxy validity + proxy_cache_valid 200 202 14d; + proxy_read_timeout 8m; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + proxy_cache_background_update on; + proxy_cache_lock on; + proxy_cache_bypass $cache_bypass; + + set $http_authorization_present '[NOT PROVIDED]'; # Default to '[NOT PROVIDED]' + + if ($http_authorization) { + set $http_authorization_present '[REDACTED]'; # Set to '[REDACTED]' when the Authorization header is present + } + + # Configure a resolver to use for DNS resolution with IPv4-only + # ipv6=off ensures that only IPv4 addresses are used for all external domains + # This prevents "Network is unreachable" errors in environments without IPv6 support + resolver dns-default.openshift-dns.svc.cluster.local 8.8.8.8 ipv6=off; + + # rewrite_log on; + + ################ Docker Compose Services ################# + + location /generate { + proxy_pass http://exploit-iq:8080; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + } + + # Include any additional routes from the routes directory + include /etc/nginx/conf.d/routes/*.conf; + + + ################### Redirect Handling #################### + + location @handle_redirects { + # store the current state of the world so we can reuse it in a minute + # We need to capture these values now, because as soon as we invoke + # the proxy_* directives, these will disappear + set $original_uri $uri; + set $orig_loc $upstream_http_location; + + # nginx goes to fetch the value from the upstream Location header + proxy_pass $orig_loc; + proxy_cache llm_cache; + + # But we store the result with the cache key of the original request URI + # so that future clients don't need to follow the redirect too + proxy_cache_key $original_uri; + proxy_cache_valid 200 206 14d; + } + } +} diff --git a/kustomize/base/nginx/nginx_ssl.conf b/kustomize/base/nginx/nginx_ssl.conf new file mode 100644 index 000000000..4825e0ffc --- /dev/null +++ b/kustomize/base/nginx/nginx_ssl.conf @@ -0,0 +1,24 @@ +worker_processes 1; + +events { + worker_connections 1024; +} + +http { + server { + listen 8443 ssl; + listen [::]:8443 ssl; + server_name localhost; + ssl_certificate /etc/nginx/ssl/cert.pem; + ssl_certificate_key /etc/nginx/ssl/key.pem; + + + location / { + proxy_pass http://nginx-cache:8080; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + } +} diff --git a/kustomize/base/nginx/templates/routes/intel.conf.template b/kustomize/base/nginx/templates/routes/intel.conf.template new file mode 100644 index 000000000..e6225731f --- /dev/null +++ b/kustomize/base/nginx/templates/routes/intel.conf.template @@ -0,0 +1,133 @@ +####################### Intel APIs ####################### + +location /serpapi { + set $backend "serpapi.com"; + + # Inject api_key if not provided by client + set $args "$api_key_prefix$args"; + + rewrite ^\/serpapi(\/.*)$ $1 break; + proxy_pass https://$backend; + proxy_ssl_server_name on; + proxy_cache intel_cache; + proxy_cache_methods GET; + + # Cache key without api_key (share cache across keys) + proxy_cache_key "$request_method|$request_uri_no_key"; + + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /nvd { + set $backend "services.nvd.nist.gov"; + rewrite ^\/nvd(\/.*)$ /rest$1 break; + proxy_pass https://$backend; + proxy_ssl_server_name on; + proxy_cache intel_cache; + proxy_cache_valid 200 202 365d; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /cve-details { + set $backend "www.cvedetails.com"; + rewrite ^\/cve-details(\/.*)$ $1 break; + proxy_pass https://$backend; + proxy_ssl_server_name on; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /cwe-details { + set $backend "cwe.mitre.org"; + rewrite ^\/cwe-details(\/.*)$ $1 break; + proxy_pass https://$backend; + proxy_ssl_server_name on; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; + + error_page 301 302 307 = @handle_redirects; +} + +location /first { + set $backend "api.first.org"; + rewrite ^\/first(\/.*)$ /data/v1$1 break; + proxy_pass https://$backend; + proxy_ssl_server_name on; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /recf { + set $backend "api.recordedfuture.com"; + rewrite ^\/recf(\/.*)$ /v2$1 break; + proxy_pass https://$backend; + proxy_ssl_server_name on; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /ghsa { + set $backend "api.github.com"; + rewrite ^\/ghsa(\/.*)$ $1 break; + proxy_pass https://$backend; + proxy_ssl_server_name on; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /rhsa { + set $backend "access.redhat.com"; + rewrite ^\/rhsa(\/.*)$ /hydra/rest$1 break; + proxy_pass https://$backend; + proxy_ssl_server_name on; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /ubuntu { + set $backend "ubuntu.com"; + rewrite ^\/ubuntu(\/.*)$ $1 break; + proxy_pass https://$backend; + proxy_ssl_server_name on; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /depsdev { + set $backend "api.deps.dev"; + rewrite ^ $request_uri; + rewrite ^\/depsdev\/(.*) $1 break; + return 400; + proxy_pass https://$backend/$uri; + proxy_ssl_server_name on; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} diff --git a/kustomize/base/nginx/templates/routes/nemo.conf.template b/kustomize/base/nginx/templates/routes/nemo.conf.template new file mode 100644 index 000000000..fb05d7bba --- /dev/null +++ b/kustomize/base/nginx/templates/routes/nemo.conf.template @@ -0,0 +1,26 @@ +location /nemo { + + location ~* ^\/nemo\/v1\/models(\/.+\/completions)?$ { + set $backend "api.llm.ngc.nvidia.com"; + rewrite ^\/nemo(\/.*)$ $1 break; + proxy_pass https://$backend; + proxy_ssl_server_name on; + proxy_set_header Connection ''; + proxy_set_header Authorization $nemo_http_authorization; + proxy_set_header Organization-ID $nemo_http_organization_id; + proxy_cache llm_cache; + proxy_cache_methods GET POST; + proxy_cache_key "$request_method|$request_uri|$request_body"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; + } + + location /nemo/v1 { + set $backend "api.llm.ngc.nvidia.com"; + rewrite ^\/nemo(\/.*)$ $1 break; + proxy_pass https://$backend; + proxy_ssl_server_name on; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; + } +} diff --git a/kustomize/base/nginx/templates/routes/nim.conf.template b/kustomize/base/nginx/templates/routes/nim.conf.template new file mode 100644 index 000000000..13bd6844a --- /dev/null +++ b/kustomize/base/nginx/templates/routes/nim.conf.template @@ -0,0 +1,39 @@ + +location ~* ^\/nim_llm\/v1\/(?:chat\/completions|completions|edits|moderations|answers)$ { + rewrite ^\/nim_llm(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NIM_LLM}; + proxy_set_header Connection ''; + proxy_set_header Authorization $nim_http_authorization; + proxy_cache llm_cache; + proxy_cache_methods GET POST; + proxy_cache_key "$request_method|$request_uri|$request_body"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /nim_llm/v1 { + rewrite ^\/nim_llm(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NIM_LLM}; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; +} + + +location ~* ^/nim_embed/v1/embeddings$ { + rewrite ^/nim_embed(.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NIM_EMBED}; + proxy_set_header Connection ''; + proxy_set_header Authorization $nim_http_authorization; + proxy_cache llm_cache; + proxy_cache_methods GET POST; + proxy_cache_key "$request_method|$request_uri|$request_body"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /nim_embed/v1 { + rewrite ^/nim_embed(.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NIM_EMBED}; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; +} diff --git a/kustomize/base/nginx/templates/routes/nvidia.conf.template b/kustomize/base/nginx/templates/routes/nvidia.conf.template new file mode 100644 index 000000000..0f6806629 --- /dev/null +++ b/kustomize/base/nginx/templates/routes/nvidia.conf.template @@ -0,0 +1,45 @@ + +location /nvai { + + location ~* ^\/nvai\/v2\/nvcf\/((pexec\/functions\/.+)|(functions))$ { + rewrite ^\/nvai(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NVAI}; + proxy_set_header Connection ''; + proxy_set_header Authorization $nvai_http_authorization; + proxy_cache llm_cache; + proxy_cache_methods GET POST; + proxy_cache_key "$request_method|$request_uri|$request_body"; + + # Only cache 200/202, not 404. + proxy_cache_valid 200 14d; + + add_header X-Cache-Status $upstream_cache_status always; + client_body_buffer_size 4m; + + access_log /dev/stdout nvai_cache_log; + access_log /var/log/nginx/access.log nvai_cache_log; + } + + location ~* ^\/nvai\/v2\/nvcf\/(pexec\/status\/.+)$ { + rewrite ^\/nvai(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NVAI}; + proxy_set_header Connection ''; + proxy_set_header Authorization $nvai_http_authorization; + proxy_cache llm_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + + # Only cache 200. Not 202 as it will cause a never ending loop + proxy_cache_valid 200 202 14d; + + add_header X-Cache-Status $upstream_cache_status always; + client_body_buffer_size 4m; + } + + location /nvai/v2 { + rewrite ^\/nvai(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NVAI}; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; + } +} diff --git a/kustomize/base/nginx/templates/routes/openai.conf.template b/kustomize/base/nginx/templates/routes/openai.conf.template new file mode 100644 index 000000000..8a5902687 --- /dev/null +++ b/kustomize/base/nginx/templates/routes/openai.conf.template @@ -0,0 +1,24 @@ + + +location /openai { + + location ~* ^\/openai\/v1\/((engines\/.+\/)?(?:chat\/completions|completions|edits|moderations|answers|embeddings))$ { + rewrite ^\/openai(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_OPENAI}; + proxy_set_header Connection ''; + proxy_set_header Authorization $openai_http_authorization; + proxy_cache llm_cache; + proxy_cache_methods POST; + proxy_cache_key "$request_method|$request_uri|$request_body"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; + } + + location /openai/v1 { + rewrite ^\/openai(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_OPENAI}; + proxy_set_header Authorization $openai_http_authorization; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; + } +} diff --git a/kustomize/base/nginx/templates/variables/template-variables.conf.template b/kustomize/base/nginx/templates/variables/template-variables.conf.template new file mode 100644 index 000000000..a1cf62efb --- /dev/null +++ b/kustomize/base/nginx/templates/variables/template-variables.conf.template @@ -0,0 +1,56 @@ +map $http_authorization $nemo_http_authorization { + 'Bearer EXPLOIT_IQ' 'Bearer ${NGC_API_KEY}'; + 'Bearer "EXPLOIT_IQ"' 'Bearer ${NGC_API_KEY}'; + 'Bearer CYBER_DEV_DAY' 'Bearer ${NGC_API_KEY}'; + 'Bearer "CYBER_DEV_DAY"' 'Bearer ${NGC_API_KEY}'; + default $http_authorization; +} + +map $http_organization_id $nemo_http_organization_id { + 'EXPLOIT_IQ' '${NGC_ORG_ID}'; + '"EXPLOIT_IQ"' '${NGC_ORG_ID}'; + 'CYBER_DEV_DAY' '${NGC_ORG_ID}'; + '"CYBER_DEV_DAY"' '${NGC_ORG_ID}'; + default $http_organization_id; +} + +map $http_authorization $nim_http_authorization { + 'Bearer EXPLOIT_IQ' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer "EXPLOIT_IQ"' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer CYBER_DEV_DAY' '${NVIDIA_API_KEY}'; + 'Bearer "CYBER_DEV_DAY"' '${NVIDIA_API_KEY}'; + default $http_authorization; +} + +map $http_authorization $nvai_http_authorization { + 'Bearer EXPLOIT_IQ' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer nvapi-EXPLOIT_IQ' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer "nvapi-EXPLOIT_IQ"' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer CYBER_DEV_DAY' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer nvapi-CYBER_DEV_DAY' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer "nvapi-CYBER_DEV_DAY"' 'Bearer ${NVIDIA_API_KEY}'; + default $http_authorization; +} + +map $http_authorization $openai_http_authorization { + 'Bearer EXPLOIT_IQ' 'Bearer ${OPENAI_API_KEY}'; + 'Bearer "EXPLOIT_IQ"' 'Bearer ${OPENAI_API_KEY}'; + 'Bearer CYBER_DEV_DAY' 'Bearer ${OPENAI_API_KEY}'; + 'Bearer "CYBER_DEV_DAY"' 'Bearer ${OPENAI_API_KEY}'; + default $http_authorization; +} + +map $arg_api_key $api_key_prefix { + "" "api_key=${SERPAPI_API_KEY}&"; + default ""; +} + +map $request_uri $loggable_request_uri { + default $request_uri; + "~^(?P.*[?&]api_key=)[^&]+(?P.*)$" "$prefix[REDACTED]$suffix"; +} + +map $request_uri $request_uri_no_key { + default $request_uri; + "~^(?P.*)[?&]api_key=[^&]*(?P.*)$" "$prefix$suffix"; +} \ No newline at end of file diff --git a/kustomize/base/service-account.yaml b/kustomize/base/service-account.yaml new file mode 100644 index 000000000..f86c4505e --- /dev/null +++ b/kustomize/base/service-account.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: exploit-iq-sa +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: exploit-iq-client-sa diff --git a/kustomize/base/settings.xml b/kustomize/base/settings.xml new file mode 100644 index 000000000..0f9299f34 --- /dev/null +++ b/kustomize/base/settings.xml @@ -0,0 +1,62 @@ + + + + + red-hat + + + red-hat-ga + https://maven.repository.redhat.com/ga + + + jboss-public-repository-group + JBoss Public Maven Repository Group + https://repository.jboss.org/nexus/content/groups/public/ + default + + true + never + + + true + never + + + + + + red-hat-ga + https://maven.repository.redhat.com/ga + + true + + + false + + + + + + + red-hat + + + + + + maven-default-http-blocker + Disabled default HTTP blocker + + + __no_such_repo_id__ + + + https://repo1.maven.org/maven2 + + + + \ No newline at end of file diff --git a/kustomize/components/oidc-ca/kustomization.yaml b/kustomize/components/oidc-ca/kustomization.yaml new file mode 100644 index 000000000..e0887cfe9 --- /dev/null +++ b/kustomize/components/oidc-ca/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +configMapGenerator: + - name: oidc-ca-bundle + files: + - ca-bundle.crt + options: + disableNameSuffixHash: true + +patches: + - path: oidc-ca-patch.yaml + target: + name: exploit-iq-client + kind: Deployment diff --git a/kustomize/components/oidc-ca/oidc-ca-patch.yaml b/kustomize/components/oidc-ca/oidc-ca-patch.yaml new file mode 100644 index 000000000..e61335c2f --- /dev/null +++ b/kustomize/components/oidc-ca/oidc-ca-patch.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: exploit-iq-client +spec: + template: + spec: + containers: + - name: exploit-iq-client + env: + - name: QUARKUS_TLS__OIDC__TRUST_STORE_PEM_CERTS + value: /etc/oidc-ca/ca-bundle.crt + - name: QUARKUS_OIDC_TLS_TLS_CONFIGURATION_NAME + value: oidc + volumeMounts: + - name: oidc-ca + mountPath: /etc/oidc-ca + readOnly: true + volumes: + - name: oidc-ca + configMap: + name: oidc-ca-bundle diff --git a/kustomize/config-http-openai-local.yml b/kustomize/config-http-openai-local.yml new file mode 100644 index 000000000..99cd4c809 --- /dev/null +++ b/kustomize/config-http-openai-local.yml @@ -0,0 +1,270 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +general: + use_uvloop: true + + telemetry: + tracing: + phoenix: + _type: phoenix + endpoint: ${OTEL_TRACES_ENDPOINT:-http://localhost:6006/v1/traces} + project: cve_agent +# tracing: +# langsmith: +# _type: langsmith +# project: default +# api_key: ${LANGSMITH_API_KEY} + +functions: + cve_clone_and_deps: + _type: cve_clone_and_deps + base_git_dir: .cache/am_cache/git + base_pickle_dir: .cache/am_cache/pickle + base_rpm_dir: .cache/am_cache/rpms + cve_segmentation: + _type: cve_segmentation + agent_name: cve_agent_executor + embedder_name: nim_embedder + base_git_dir: .cache/am_cache/git + base_vdb_dir: .cache/am_cache/vdb + base_code_index_dir: .cache/am_cache/code_index + base_pickle_dir: .cache/am_cache/pickle + ignore_code_embedding: true + cve_fetch_intel: + _type: cve_fetch_intel + rpm_user_type: ${RPM_USER_TYPE:-internal} + retry_on_client_errors: false + intel_plugin_config: + plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin + plugin_config: + source: Product Security research + endpoint: http://localhost:8080/api/v1/vulnerabilities/{vuln_id}/comments + cve_process_sbom: + _type: cve_process_sbom + cve_checklist: + _type: cve_checklist + llm_name: checklist_llm + Call Chain Analyzer: + _type: transitive_code_search + enable_transitive_search: true + Function Caller Finder: + _type: calling_function_name_extractor + enable_functions_usage_search: true + Function Locator: + _type: package_and_function_locator + Code Semantic Search: + _type: local_vdb_retriever + embedder_name: nim_embedder + llm_name: code_vdb_retriever_llm + vdb_type: code + return_source_documents: false + Docs Semantic Search: + _type: local_vdb_retriever + embedder_name: nim_embedder + llm_name: doc_vdb_retriever_llm + vdb_type: doc + return_source_documents: false + Code Keyword Search: + _type: lexical_code_search + top_k: 5 + CVE Web Search: + _type: serp_wrapper + max_retries: 5 + Container Analysis Data: + _type: container_image_analysis_data + Function Library Version Finder: + _type: calling_function_library_version_finder + cve_agent_executor: + _type: cve_agent_executor + llm_name: cve_agent_executor_llm + tool_names: + - Code Semantic Search + - Docs Semantic Search + - Code Keyword Search + - CVE Web Search + - Call Chain Analyzer + - Function Caller Finder + - Function Locator + - Function Library Version Finder + max_concurrency: null + max_iterations: 10 + prompt_examples: false + replace_exceptions: true + replace_exceptions_value: "I do not have a definitive answer for this checklist item." + return_intermediate_steps: false +# transitive_search_tool_enabled: false + cve_web_search_enabled: true + verbose: false + cve_generate_cvss: + _type: cve_generate_cvss + skip: false + llm_name: generate_cvss_llm + tool_names: + - Code Semantic Search + - Docs Semantic Search + - Code Keyword Search + - Container Analysis Data + max_concurrency: null + max_iterations: 10 + prompt_examples: true + replace_exceptions: false + replace_exceptions_value: "Failed to generate CVSS for this analysis." + return_intermediate_steps: false + verbose: false + cve_summarize: + _type: cve_summarize + llm_name: summarize_llm + cve_justify: + _type: cve_justify + llm_name: justify_llm + # cve_file_output: + # _type: cve_file_output + # file_path: .tmp/output.json + # markdown_dir: .tmp/vulnerability_markdown_reports + # overwrite: true + cve_http_output: + _type: cve_http_output + url: http://localhost:8080 + endpoint: /api/v1/reports + enable_mlops: ${ENABLE_MLOPS:-false} + mlops_config: + mlops_url: http://localhost:8080 + auth_type: "None" + token_path: "/var/run/secrets/kubernetes.io/serviceaccount/token" + verify_path: "/app/certs/service-ca.crt" + enable_verify: false + cve_file_output: + _type: cve_file_output + file_path: .tmp/output.json + markdown_dir: .tmp/vulnerability_markdown_reports + overwrite: true + cve_calculate_intel_score: + _type: cve_calculate_intel_score + llm_name: intel_source_score_llm + generate_intel_score: true + intel_low_score: 51 + insist_analysis: false + +llms: + checklist_llm: + _type: openai + api_key: ${OPENAI_API_KEY:-EMPTY} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CHECKLIST_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + code_vdb_retriever_llm: + _type: openai + api_key: ${OPENAI_API_KEY:-EMPTY} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CODE_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + doc_vdb_retriever_llm: + _type: openai + api_key: ${OPENAI_API_KEY:-EMPTY} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${DOC_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + cve_agent_executor_llm: + _type: openai + api_key: ${OPENAI_API_KEY:-EMPTY} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CVE_AGENT_EXECUTOR_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + generate_cvss_llm: + _type: openai + api_key: ${OPENAI_API_KEY:-EMPTY} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + summarize_llm: + _type: openai + api_key: ${OPENAI_API_KEY:-EMPTY} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${SUMMARIZE_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + justify_llm: + _type: openai + api_key: ${OPENAI_API_KEY:-EMPTY} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + + intel_source_score_llm: + _type: openai + api_key: ${OPENAI_API_KEY:-EMPTY} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + +embedders: + nim_embedder: + _type: nim + base_url: ${NIM_EMBED_BASE_URL:-https://integrate.api.nvidia.com/v1} + model_name: ${EMBEDDER_MODEL_NAME:-nvidia/nv-embedqa-e5-v5} + truncate: END + max_batch_size: 128 + +workflow: + _type: cve_agent + cve_clone_and_deps_name: cve_clone_and_deps + cve_segmentation_name: cve_segmentation + cve_fetch_intel_name: cve_fetch_intel + cve_calculate_intel_score_name: cve_calculate_intel_score + cve_process_sbom_name: cve_process_sbom + cve_checklist_name: cve_checklist + cve_agent_executor_name: cve_agent_executor + cve_generate_cvss_name: cve_generate_cvss + cve_summarize_name: cve_summarize + cve_justify_name: cve_justify + cve_output_config_name: cve_file_output + +eval: + general: + output_dir: ./.tmp/eval/cve_agent + dataset: + _type: json + file_path: data/eval_datasets/eval_dataset.json + + profiler: + token_uniqueness_forecast: true + workflow_runtime_forecast: true + compute_llm_metrics: true + csv_exclude_io_text: true + prompt_caching_prefixes: + enable: true + min_frequency: 0.1 + bottleneck_analysis: + # Can also be simple_stack + enable_nested_stack: true + concurrency_spike_analysis: + enable: true + spike_threshold: 7 diff --git a/kustomize/deployer-rbac.yaml b/kustomize/deployer-rbac.yaml new file mode 100644 index 000000000..8be695176 --- /dev/null +++ b/kustomize/deployer-rbac.yaml @@ -0,0 +1,101 @@ +# deployer-rbac.yaml +# +# Grants a non-cluster-admin user the minimum permissions required to +# deploy Exploit Intelligence on OpenShift Container Platform. +# +# Please replace the following placeholders: +# — the OpenShift username of the deployer (e.g. jdoe) +# — the target namespace (e.g. exploit-iq) +# + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: exploit-intelligence-oauthclient-deploy +rules: + # get and patch scoped to the two project OAuthClients only. + - apiGroups: + - oauth.openshift.io + resources: + - oauthclients + resourceNames: + - exploit-iq-client + - exploitiq-mcp-server + verbs: + - get + - patch + # create cannot be restricted by resourceNames (resource does not exist yet). + - apiGroups: + - oauth.openshift.io + resources: + - oauthclients + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: exploit-intelligence-oauthclient-deploy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: exploit-intelligence-oauthclient-deploy +subjects: + - kind: User + name: +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: exploit-intelligence-rbac-deploy + namespace: +rules: + - apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + verbs: + - get + - create + - update + - patch + - apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + verbs: + - get + - create + - update + - patch + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - anyuid + verbs: + - use + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - get + - create + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: exploit-intelligence-rbac-deploy + namespace: +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: exploit-intelligence-rbac-deploy +subjects: + - kind: User + name: diff --git a/kustomize/img.png b/kustomize/img.png new file mode 100644 index 000000000..62aa97cbb Binary files /dev/null and b/kustomize/img.png differ diff --git a/kustomize/network-policy.yaml b/kustomize/network-policy.yaml new file mode 100644 index 000000000..5ea2ba756 --- /dev/null +++ b/kustomize/network-policy.yaml @@ -0,0 +1,17 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + networking.knative.dev/ingress-provider: istio + name: allow-from-agent-morpheus-namespaces + namespace: morpheus-cn-models-nim +spec: + ingress: + - from: + - namespaceSelector: + matchLabels: + application: agent-morpheus + podSelector: {} + policyTypes: + - Ingress + diff --git a/kustomize/overlays/batch-processing/exploit-iq-client-batch-patch.yaml b/kustomize/overlays/batch-processing/exploit-iq-client-batch-patch.yaml new file mode 100644 index 000000000..a753910a2 --- /dev/null +++ b/kustomize/overlays/batch-processing/exploit-iq-client-batch-patch.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: exploit-iq-client + labels: + app: exploit-iq + component: exploit-iq-client +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app: exploit-iq + component: exploit-iq-client + template: + metadata: + labels: + app: exploit-iq + component: exploit-iq-client + spec: + imagePullSecrets: [] + serviceAccountName: exploit-iq-client-sa + containers: + - name: exploit-iq-client + imagePullPolicy: Always + env: + - name: MORPHEUS_QUEUE_TIMEOUT + value: 60m + - name: MORPHEUS_QUEUE_MAX_ACTIVE + value: "5" diff --git a/kustomize/overlays/batch-processing/exploit-iq-resources-patch.yaml b/kustomize/overlays/batch-processing/exploit-iq-resources-patch.yaml new file mode 100644 index 000000000..195e8792d --- /dev/null +++ b/kustomize/overlays/batch-processing/exploit-iq-resources-patch.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: exploit-iq + labels: + app: exploit-iq + component: exploit-iq +spec: + selector: + matchLabels: + app: exploit-iq + component: exploit-iq + template: + metadata: + labels: + app: exploit-iq + component: exploit-iq + spec: + containers: + - name: exploit-iq-service + imagePullPolicy: Always + workingDir: /workspace/ + # Deploy with QoS(Guaranteed) + resources: + limits: + memory: "12Gi" + cpu: "1000m" + nvidia.com/gpu: "1" + requests: + memory: "12Gi" + cpu: "1000m" + nvidia.com/gpu: "1" diff --git a/kustomize/overlays/batch-processing/kustomization.yaml b/kustomize/overlays/batch-processing/kustomization.yaml new file mode 100644 index 000000000..5d7cc1acd --- /dev/null +++ b/kustomize/overlays/batch-processing/kustomization.yaml @@ -0,0 +1,22 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../self-hosted-llama3.1-70b-4bit + +commonAnnotations: + deployment-variant: batch-processing + +patches: +- path: exploit-iq-resources-patch.yaml +- path: exploit-iq-client-batch-patch.yaml + +configMapGenerator: +- behavior: replace + name: nginx-cache-routes + files: + - nginx/templates/routes/intel.conf.template + - nginx/templates/routes/nemo.conf.template + - nginx/templates/routes/nim.conf.template + - nginx/templates/routes/nvidia.conf.template + - nginx/templates/routes/openai.conf.template diff --git a/kustomize/overlays/batch-processing/nginx/templates/routes/intel.conf.template b/kustomize/overlays/batch-processing/nginx/templates/routes/intel.conf.template new file mode 100644 index 000000000..a3389c38d --- /dev/null +++ b/kustomize/overlays/batch-processing/nginx/templates/routes/intel.conf.template @@ -0,0 +1,110 @@ +####################### Intel APIs ####################### + +location /serpapi { + rewrite ^\/serpapi(\/.*)$ $1 break; + proxy_pass https://serpapi.com; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /nvd { + rewrite ^\/nvd(\/.*)$ /rest$1 break; + proxy_pass https://services.nvd.nist.gov; + proxy_set_header apiKey $nvd_http_api_key; + proxy_cache intel_cache; + proxy_cache_valid 200 202 365d; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /cve-details { + rewrite ^\/cve-details(\/.*)$ $1 break; + proxy_pass https://www.cvedetails.com; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /cwe-details { + rewrite ^\/cwe-details(\/.*)$ $1 break; + proxy_pass https://cwe.mitre.org; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; + + error_page 301 302 307 = @handle_redirects; +} + +location /first { + rewrite ^\/first(\/.*)$ /data/v1$1 break; + proxy_pass https://api.first.org; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /recf { + rewrite ^\/recf(\/.*)$ /v2$1 break; + proxy_pass https://api.recordedfuture.com; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /ghsa { + rewrite ^\/ghsa(\/.*)$ $1 break; + proxy_pass https://api.github.com; + proxy_set_header Authorization $ghsa_http_authorization; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /rhsa { + rewrite ^\/rhsa(\/.*)$ /hydra/rest$1 break; + proxy_pass https://access.redhat.com; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /ubuntu { + rewrite ^\/ubuntu(\/.*)$ $1 break; + proxy_pass https://ubuntu.com; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /depsdev { + rewrite ^ $request_uri; + rewrite ^\/depsdev\/(.*) $1 break; + return 400; + proxy_pass https://api.deps.dev/$uri; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; + + resolver dns-default.openshift-dns.svc.cluster.local 8.8.8.8 valid=60s; +} diff --git a/kustomize/overlays/batch-processing/nginx/templates/routes/nemo.conf.template b/kustomize/overlays/batch-processing/nginx/templates/routes/nemo.conf.template new file mode 100644 index 000000000..cd671f249 --- /dev/null +++ b/kustomize/overlays/batch-processing/nginx/templates/routes/nemo.conf.template @@ -0,0 +1,19 @@ +location /nemo { + + location ~* ^\/nemo\/v1\/models(\/.+\/completions)?$ { + rewrite ^\/nemo(\/.*)$ $1 break; + proxy_pass https://api.llm.ngc.nvidia.com; + proxy_set_header Connection ''; + proxy_set_header Authorization $nemo_http_authorization; + proxy_set_header Organization-ID $nemo_http_organization_id; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; + } + + location /nemo/v1 { + rewrite ^\/nemo(\/.*)$ $1 break; + proxy_pass https://api.llm.ngc.nvidia.com; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; + } +} diff --git a/kustomize/overlays/batch-processing/nginx/templates/routes/nim.conf.template b/kustomize/overlays/batch-processing/nginx/templates/routes/nim.conf.template new file mode 100644 index 000000000..a44ccd3c6 --- /dev/null +++ b/kustomize/overlays/batch-processing/nginx/templates/routes/nim.conf.template @@ -0,0 +1,33 @@ + +location ~* ^\/nim_llm\/v1\/(?:chat\/completions|completions|edits|moderations|answers)$ { + rewrite ^\/nim_llm(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NIM_LLM}; + proxy_set_header Connection ''; + proxy_set_header Authorization $nim_http_authorization; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /nim_llm/v1 { + rewrite ^\/nim_llm(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NIM_LLM}; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; +} + + +location ~* ^/nim_embed/v1/embeddings$ { + rewrite ^/nim_embed(.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NIM_EMBED}; + proxy_set_header Connection ''; + proxy_set_header Authorization $nim_http_authorization; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /nim_embed/v1 { + rewrite ^/nim_embed(.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NIM_EMBED}; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; +} diff --git a/kustomize/overlays/batch-processing/nginx/templates/routes/nvidia.conf.template b/kustomize/overlays/batch-processing/nginx/templates/routes/nvidia.conf.template new file mode 100644 index 000000000..87a155202 --- /dev/null +++ b/kustomize/overlays/batch-processing/nginx/templates/routes/nvidia.conf.template @@ -0,0 +1,31 @@ + +location /nvai { + + location ~* ^\/nvai\/v2\/nvcf\/((pexec\/functions\/.+)|(functions))$ { + rewrite ^\/nvai(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NVAI}; + proxy_set_header Connection ''; + proxy_set_header Authorization $nvai_http_authorization; + add_header X-Cache-Status $upstream_cache_status always; + client_body_buffer_size 4m; + + access_log /dev/stdout nvai_cache_log; + access_log /var/log/nginx/access.log nvai_cache_log; + } + + location ~* ^\/nvai\/v2\/nvcf\/(pexec\/status\/.+)$ { + rewrite ^\/nvai(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NVAI}; + proxy_set_header Connection ''; + proxy_set_header Authorization $nvai_http_authorization; + add_header X-Cache-Status $upstream_cache_status always; + client_body_buffer_size 4m; + } + + location /nvai/v2 { + rewrite ^\/nvai(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_NVAI}; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; + } +} diff --git a/kustomize/overlays/batch-processing/nginx/templates/routes/openai.conf.template b/kustomize/overlays/batch-processing/nginx/templates/routes/openai.conf.template new file mode 100644 index 000000000..9ea09d9dd --- /dev/null +++ b/kustomize/overlays/batch-processing/nginx/templates/routes/openai.conf.template @@ -0,0 +1,21 @@ + + +location /openai { + + location ~* ^\/openai\/v1\/((engines\/.+\/)?(?:chat\/completions|completions|edits|moderations|answers|embeddings))$ { + rewrite ^\/openai(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_OPENAI}; + proxy_set_header Connection ''; + proxy_set_header Authorization $openai_http_authorization; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; + } + + location /openai/v1 { + rewrite ^\/openai(\/.*)$ $1 break; + proxy_pass ${NGINX_UPSTREAM_OPENAI}; + proxy_set_header Authorization $openai_http_authorization; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; + } +} diff --git a/kustomize/overlays/mlops/grafana/auth/kustomization.yaml b/kustomize/overlays/mlops/grafana/auth/kustomization.yaml new file mode 100644 index 000000000..a27ba6506 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/auth/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - serviceAccount.yaml \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/auth/serviceAccount.yaml b/kustomize/overlays/mlops/grafana/auth/serviceAccount.yaml new file mode 100644 index 000000000..17f2eb7c2 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/auth/serviceAccount.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: grafana-sa + annotations: + serviceaccounts.openshift.io/oauth-redirectreference.primary: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"REPLACE_ROUTE"}}' \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/dashboards/batch-info.yaml b/kustomize/overlays/mlops/grafana/dashboards/batch-info.yaml new file mode 100644 index 000000000..dba082859 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/dashboards/batch-info.yaml @@ -0,0 +1,169 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: batch-info +spec: + allowCrossNamespaceImport: false + folder: Batches + instanceSelector: + matchLabels: + app: exploit-iq + json: | + { + "templating": { + "list": [ + { + "definition": "Batch ID the batch was executed by", + "name": "batch_id", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Batch ID", + "root_selector": "$.batch_id", + "source": "url", + "type": "json", + "url": "/batch/all", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "Batch execution start timestamp", + "name": "execution_start", + "type": "textbox" + }, + { + "definition": "Batch language", + "name": "language", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Component", + "root_selector": "$distinct($.app_language)", + "source": "url", + "type": "json", + "url": "/batch/all", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "panels": [ + { + "datasource": { + "uid": "infinity" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "targets": [ + { + "parser": "backend", + "datasource": { + "uid": "infinity" + }, + "format": "table", + "refId": "by_batch_id", + "source": "url", + "type": "json", + "url": "/batch/${batch_id}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Batch by Batch ID", + "type": "table" + }, + { + "datasource": { + "uid": "infinity" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "targets": [ + { + "parser": "backend", + "filterExpression": "execution_start_timestamp==\"${execution_start}\"", + "datasource": { + "uid": "infinity" + }, + "format": "table", + "refId": "by_execution_start", + "source": "url", + "type": "json", + "url": "/batch/all", + "url_options": { + "method": "GET" + } + } + ], + "title": "Batches by Execution Time Start", + "type": "table" + }, + { + "datasource": { + "uid": "infinity" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "targets": [ + { + "datasource": { + "uid": "infinity" + }, + "format": "table", + "refId": "by_language", + "source": "url", + "type": "json", + "url": "/batch/all?language=${language}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Batches by Language", + "type": "table" + } + ], + "time": { + "from": "now-1y", + "to": "now" + }, + "title": "Batch info" + } \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/dashboards/batch-metrics.yaml b/kustomize/overlays/mlops/grafana/dashboards/batch-metrics.yaml new file mode 100644 index 000000000..e2b881118 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/dashboards/batch-metrics.yaml @@ -0,0 +1,195 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: batch-metrics +spec: + allowCrossNamespaceImport: false + folder: Batches + instanceSelector: + matchLabels: + app: exploit-iq + json: | + { + "panels": [ + { + "datasource": { + "uid": "infinity" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1 , + "transformations": [ + { + "id": "organize", + "options": { + "indexByName": { + "batch_id": 0, + "app_language": 1, + "confusion_matrix_accuracy": 2, + "confusion_matrix_f1_score": 3, + "confusion_matrix_precision": 4, + "confusion_matrix_recall": 5 + } + } + } + ], + "targets": [ + { + "datasource": { + "uid": "infinity" + }, + "format": "table", + "refId": "confusion_matrix", + "source": "url", + "type": "json", + "url": "/batch/all", + "url_options": { + "method": "GET" + }, + "root_selector": "$map($, function($v) { {\"batch_id\": $v.batch_id, \"language\": $v.app_language, \"accuracy\": $v.confusion_matrix_accuracy, \"precision\": $v.confusion_matrix_precision, \"f1_score\": $v.confusion_matrix_f1_score, \"recall\": $v.confusion_matrix_recall} })", + "parser": "backend" + } + ], + "title": "Confusion Matrix", + "type": "table" + }, + { + "datasource": { + "uid": "infinity" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "transformations": [ + { + "id": "organize", + "options": { + "indexByName": { + "batch_id": 0, + "number_of_regressive_jobs_ids": 1, + "regression_ratio": 2 + } + } + } + ], + "targets": [ + { + "datasource": { + "uid": "infinity" + }, + "format": "table", + "refId": "regressive_job_count_by_batch_id", + "source": "url", + "type": "json", + "url": "/batch/all", + "url_options": { + "method": "GET" + }, + "root_selector": "$map($, function($v) { {\"batch_id\": $v.batch_id, \"number_of_regressive_jobs_ids\": $v.number_of_regressive_jobs_ids, \"regression_ratio\": $v.total_number_of_executed_jobs != 0 ? $v.number_of_regressive_jobs_ids / $v.total_number_of_executed_jobs : null} })", + "parser": "backend" + } + ], + "title": "Regressive Job Count by Batch ID", + "type": "table" + }, + { + "datasource": { + "uid": "infinity" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "transformations": [ + { + "id": "organize", + "options": { + "indexByName": { + "app_language": 0, + "number_of_regressive_jobs_ids": 1, + "regression_ratio": 2 + } + } + } + ], + "targets": [ + { + "datasource": { + "uid": "infinity" + }, + "format": "table", + "refId": "regressive_job_count_by_language", + "source": "url", + "type": "json", + "url": "/batch/all", + "url_options": { + "method": "GET" + }, + "root_selector": "$map($distinct($.app_language), function($lang) { {\"app_language\": $lang, \"number_of_regressive_jobs_ids\": $sum($filter($, function($v) { $v.app_language = $lang }).number_of_regressive_jobs_ids), \"regression_ratio\": $sum($filter($, function($v) { $v.app_language = $lang }).total_number_of_executed_jobs) != 0 ? $sum($filter($, function($v) { $v.app_language = $lang }).number_of_regressive_jobs_ids) / $sum($filter($, function($v) { $v.app_language = $lang }).total_number_of_executed_jobs) : null} })", + "parser": "backend" + } + ], + "title": "Regressive Job Count by Language", + "type": "table" + }, + { + "datasource": { + "uid": "infinity" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 4, + "transformations": [ + { + "id": "organize", + "options": { + "indexByName": { + "app_language": 0, + "total_number_of_failures": 1, + "failure_ratio": 2 + } + } + } + ], + "targets": [ + { + "datasource": { + "uid": "infinity" + }, + "format": "table", + "refId": "failure_job_count_by_language", + "source": "url", + "type": "json", + "url": "/batch/all", + "url_options": { + "method": "GET" + }, + "root_selector": "$map($distinct($.app_language), function($lang) { {\"app_language\": $lang, \"total_number_of_failures\": $sum($filter($, function($v) { $v.app_language = $lang }).total_number_of_failures), \"failed_ratio\": $sum($filter($, function($v) { $v.app_language = $lang }).total_number_of_executed_jobs) != 0 ? $sum($filter($, function($v) { $v.app_language = $lang }).total_number_of_failures) / $sum($filter($, function($v) { $v.app_language = $lang }).total_number_of_executed_jobs) : null} })", + "parser": "backend" + } + ], + "title": "Failure Job Count by Language", + "type": "table" + } + ], + "time": { + "from": "now-1y", + "to": "now" + }, + "title": "Batch Metrics" + } \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/dashboards/evals.yaml b/kustomize/overlays/mlops/grafana/dashboards/evals.yaml new file mode 100644 index 000000000..06ffe6fe7 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/dashboards/evals.yaml @@ -0,0 +1,236 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: evals +spec: + allowCrossNamespaceImport: false + folder: Evals + instanceSelector: + matchLabels: + app: exploit-iq + json: | + { + "templating": { + "list": [ + { + "definition": "Job ID", + "name": "job_id", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Job ID", + "root_selector": "$.job_id", + "source": "url", + "type": "json", + "url": "/evals/all", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "Job component", + "name": "component", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Component", + "root_selector": "$distinct($.component)", + "source": "url", + "type": "json", + "url": "/evals/all", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "Job component version", + "name": "component_version", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Component Version", + "root_selector": "$distinct($.component_version)", + "source": "url", + "type": "json", + "url": "/evals/all", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "CVE identifier", + "name": "cve", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "CVE", + "root_selector": "$distinct($.cve)", + "source": "url", + "type": "json", + "url": "/evals/all", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "Metric name", + "name": "metric_name", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Metric Name", + "root_selector": "$distinct($.metric_name)", + "source": "url", + "type": "json", + "url": "/evals/all", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "panels": [ + { + "datasource": { + "uid": "infinity" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "targets": [ + { + "datasource": { + "uid": "infinity" + }, + "format": "table", + "refId": "by_job_id", + "source": "url", + "type": "json", + "url": "/evals/all?job_id=${job_id}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Job by Request ID", + "type": "table" + }, + { + "datasource": { + "uid": "infinity" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "targets": [ + { + "datasource": { + "uid": "infinity" + }, + "format": "table", + "refId": "by_cve_component", + "source": "url", + "type": "json", + "url": "/evals/${cve}?component=${component}&componentVersion=${component_version}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Jobs by CVE + Component + Version", + "type": "table" + }, + { + "datasource": { + "uid": "infinity" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "targets": [ + { + "datasource": { + "uid": "infinity" + }, + "format": "table", + "refId": "by_cve_component_metric", + "source": "url", + "type": "json", + "url": "/evals/${cve}?component=${component}&componentVersion=${component_version}&metric_name=${metric_name}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Jobs by CVE + Component + Version + Metric Name", + "type": "table" + } + ], + "time": { + "from": "now-1y", + "to": "now" + }, + "title": "evals" + } diff --git a/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml b/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml new file mode 100644 index 000000000..27461dc94 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/dashboards/jobs.yaml @@ -0,0 +1,236 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: jobs +spec: + allowCrossNamespaceImport: false + folder: Jobs + instanceSelector: + matchLabels: + app: exploit-iq + json: | + { + "templating": { + "list": [ + { + "definition": "Batch ID the job was executed by", + "name": "batch_id", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Batch ID", + "root_selector": "$distinct($.batch_id)", + "source": "url", + "type": "json", + "url": "/jobs/all", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "Job ID", + "name": "job_id", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Job ID", + "root_selector": "$.job_id", + "source": "url", + "type": "json", + "url": "/jobs/all", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "Job component", + "name": "component", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Component", + "root_selector": "$distinct($.component)", + "source": "url", + "type": "json", + "url": "/jobs/all", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "Job component version", + "name": "component_version", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Component Version", + "root_selector": "$distinct($.component_version)", + "source": "url", + "type": "json", + "url": "/jobs/all", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "CVE identifier", + "name": "cve", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "CVE", + "root_selector": "$distinct($.cve)", + "source": "url", + "type": "json", + "url": "/jobs/all", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "panels": [ + { + "datasource": { + "uid": "infinity" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "targets": [ + { + "datasource": { + "uid": "infinity" + }, + "format": "table", + "refId": "by_job_id", + "source": "url", + "type": "json", + "url": "/jobs/${job_id}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Job by Job ID", + "type": "table" + }, + { + "datasource": { + "uid": "infinity" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "targets": [ + { + "datasource": { + "uid": "infinity" + }, + "format": "table", + "refId": "by_batch_id", + "source": "url", + "type": "json", + "url": "/jobs/all?batchId=${batch_id}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Jobs by Batch ID", + "type": "table" + }, + { + "datasource": { + "uid": "infinity" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "targets": [ + { + "datasource": { + "uid": "infinity" + }, + "format": "table", + "refId": "by_cve_component", + "source": "url", + "type": "json", + "url": "/jobs/all?cveId=${cve}&component=${component:percentencode}&componentVersion=${component_version:percentencode}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Jobs by CVE + Component + Version", + "type": "table" + } + ], + "time": { + "from": "now-1y", + "to": "now" + }, + "title": "jobs" + } diff --git a/kustomize/overlays/mlops/grafana/dashboards/kustomization.yaml b/kustomize/overlays/mlops/grafana/dashboards/kustomization.yaml new file mode 100644 index 000000000..97a51c1e4 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/dashboards/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - jobs.yaml + - evals.yaml + - batch-info.yaml + - batch-metrics.yaml + - traces.yaml + +labels: + - pairs: + managed-by: grafana-operator \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/dashboards/traces.yaml b/kustomize/overlays/mlops/grafana/dashboards/traces.yaml new file mode 100644 index 000000000..3630a6d7b --- /dev/null +++ b/kustomize/overlays/mlops/grafana/dashboards/traces.yaml @@ -0,0 +1,161 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: traces +spec: + allowCrossNamespaceImport: false + folder: Traces + instanceSelector: + matchLabels: + app: exploit-iq + json: | + { + "templating": { + "list": [ + { + "definition": "Job ID", + "name": "job_id", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Job ID", + "root_selector": "$distinct($.job_id)", + "source": "url", + "type": "json", + "url": "/traces/all", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "Trace ID", + "name": "trace_id", + "query": { + "infinityQuery": { + "columns": [], + "filters": [], + "format": "table", + "parser": "backend", + "refId": "Trace ID", + "root_selector": "$.trace_id", + "source": "url", + "type": "json", + "url": "/traces/all", + "url_options": { + "method": "GET" + } + }, + "query": "", + "queryType": "infinity" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "panels": [ + { + "datasource": { + "uid": "infinity" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "targets": [ + { + "datasource": { + "uid": "infinity" + }, + "format": "table", + "refId": "by_job_id", + "source": "url", + "type": "json", + "url": "/traces/all?jobId=${job_id}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Traces by Job ID", + "type": "table" + }, + { + "datasource": { + "uid": "infinity" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "targets": [ + { + "datasource": { + "uid": "infinity" + }, + "format": "table", + "refId": "by_trace_id", + "source": "url", + "type": "json", + "url": "/traces/all?traceId=${trace_id}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Traces by Trace ID", + "type": "table" + }, + { + "datasource": { + "uid": "infinity" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "targets": [ + { + "datasource": { + "uid": "infinity" + }, + "format": "table", + "refId": "by_job_id_and_trace_id", + "source": "url", + "type": "json", + "url": "/traces/all?jobId=${job_id}&traceId=${trace_id}", + "url_options": { + "method": "GET" + } + } + ], + "title": "Traces by Job ID and Trace ID", + "type": "table" + } + ], + "time": { + "from": "now-1y", + "to": "now" + }, + "title": "traces" + } diff --git a/kustomize/overlays/mlops/grafana/datasources/infinity.yaml b/kustomize/overlays/mlops/grafana/datasources/infinity.yaml new file mode 100644 index 000000000..7af6b9ca1 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/datasources/infinity.yaml @@ -0,0 +1,40 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDatasource +metadata: + name: infinity +spec: + valuesFrom: + - targetPath: "secureJsonData.httpHeaderValue1" + valueFrom: + secretKeyRef: + name: "grafana-bearer-token" + key: "token" + - targetPath: "secureJsonData.tlsCACert" + valueFrom: + configMapKeyRef: + name: "openshift-service-ca.crt" + key: "service-ca.crt" + instanceSelector: + matchLabels: + app: exploit-iq + datasource: + access: proxy + isDefault: true + url: https://exploit-iq-client..svc:8443/api/v1 + jsonData: + httpHeaderName1: "Authorization" + tlsSkipVerify: false + tlsAuthWithCACert: true + global_queries: [] + allowedHosts: + - "https://exploit-iq-client..svc:8443" + secureJsonData: + httpHeaderValue1: "Bearer ${token}" + tlsCACert: ${service-ca.crt} + name: infinity + uid: infinity + type: yesoreyeram-infinity-datasource + plugins: + - name: yesoreyeram-infinity-datasource + version: 3.6.0 + diff --git a/kustomize/overlays/mlops/grafana/datasources/kustomization.yaml b/kustomize/overlays/mlops/grafana/datasources/kustomization.yaml new file mode 100644 index 000000000..988d839a8 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/datasources/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - infinity.yaml + - tempo.yaml + +labels: + - pairs: + managed-by: grafana-operator \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/datasources/tempo.yaml b/kustomize/overlays/mlops/grafana/datasources/tempo.yaml new file mode 100644 index 000000000..963cdbc6e --- /dev/null +++ b/kustomize/overlays/mlops/grafana/datasources/tempo.yaml @@ -0,0 +1,15 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDatasource +metadata: + name: tempo +spec: + instanceSelector: + matchLabels: + app: exploit-iq + datasource: + access: proxy + url: http://tempo:3200 + name: Tempo + uid: tempo + type: tempo + isDefault: false diff --git a/kustomize/overlays/mlops/grafana/folders/batches.yaml b/kustomize/overlays/mlops/grafana/folders/batches.yaml new file mode 100644 index 000000000..f84769e63 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/folders/batches.yaml @@ -0,0 +1,9 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaFolder +metadata: + name: batches +spec: + instanceSelector: + matchLabels: + app: exploit-iq + title: "Batches" \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/folders/evals.yaml b/kustomize/overlays/mlops/grafana/folders/evals.yaml new file mode 100644 index 000000000..494c2b285 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/folders/evals.yaml @@ -0,0 +1,9 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaFolder +metadata: + name: evals +spec: + instanceSelector: + matchLabels: + app: exploit-iq + title: "Evals" \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/folders/jobs.yaml b/kustomize/overlays/mlops/grafana/folders/jobs.yaml new file mode 100644 index 000000000..37fcf4aba --- /dev/null +++ b/kustomize/overlays/mlops/grafana/folders/jobs.yaml @@ -0,0 +1,9 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaFolder +metadata: + name: jobs +spec: + instanceSelector: + matchLabels: + app: exploit-iq + title: "Jobs" \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/folders/kustomization.yaml b/kustomize/overlays/mlops/grafana/folders/kustomization.yaml new file mode 100644 index 000000000..b390590ca --- /dev/null +++ b/kustomize/overlays/mlops/grafana/folders/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - jobs.yaml + - evals.yaml + - batches.yaml + - traces.yaml + +labels: + - pairs: + managed-by: grafana-operator \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/folders/traces.yaml b/kustomize/overlays/mlops/grafana/folders/traces.yaml new file mode 100644 index 000000000..76cb156d3 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/folders/traces.yaml @@ -0,0 +1,9 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaFolder +metadata: + name: traces +spec: + instanceSelector: + matchLabels: + app: exploit-iq + title: "Traces" \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/instances/grafana.yaml b/kustomize/overlays/mlops/grafana/instances/grafana.yaml new file mode 100644 index 000000000..cd2ff3c14 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/instances/grafana.yaml @@ -0,0 +1,76 @@ +kind: Grafana +apiVersion: grafana.integreatly.org/v1beta1 +metadata: + name: grafana +spec: + config: + auth: + disable_login_form: 'true' + auth.proxy: + auto_sign_up: 'true' + enabled: 'true' + header_name: X-Forwarded-User + header_property: username + log: + mode: console + deployment: + spec: + template: + spec: + containers: + - args: + - '-openshift-service-account=grafana-sa' + - '-provider=openshift' + - '-https-address=:9091' + - '-tls-cert=/etc/proxy/tls/tls.crt' + - '-tls-key=/etc/proxy/tls/tls.key' + - '-upstream=http://localhost:3000' + - '-cookie-secret=/etc/proxy/secrets/session-secret' + image: 'quay.io/openshift/origin-oauth-proxy:latest' + name: grafana-oauth-proxy + ports: + - containerPort: 9091 + name: https + protocol: TCP + volumeMounts: + - mountPath: /etc/proxy/secrets + name: oauth-proxy-secret + readOnly: true + - mountPath: /etc/proxy/tls + name: oauth-proxy-tls + readOnly: true + - name: grafana + readinessProbe: + failureThreshold: 1 + httpGet: + path: /api/health + port: 3000 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 20 + successThreshold: 1 + timeoutSeconds: 30 + volumes: + - name: oauth-proxy-tls + secret: + secretName: exploit-iq-grafana-tls + - name: oauth-proxy-secret + secret: + secretName: oauth-client-secret + route: + spec: + port: + targetPort: https + tls: + insecureEdgeTerminationPolicy: Redirect + termination: reencrypt + service: + metadata: + annotations: + service.beta.openshift.io/serving-cert-secret-name: exploit-iq-grafana-tls + spec: + ports: + - name: https + port: 9091 + protocol: TCP + targetPort: 9091 diff --git a/kustomize/overlays/mlops/grafana/instances/kustomization.yaml b/kustomize/overlays/mlops/grafana/instances/kustomization.yaml new file mode 100644 index 000000000..c4e4484bc --- /dev/null +++ b/kustomize/overlays/mlops/grafana/instances/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - grafana.yaml + +labels: + - pairs: + managed-by: grafana-operator \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/kustomization.yaml b/kustomize/overlays/mlops/grafana/kustomization.yaml new file mode 100644 index 000000000..867518e5f --- /dev/null +++ b/kustomize/overlays/mlops/grafana/kustomization.yaml @@ -0,0 +1,94 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: REPLACE_NAMESPACE + +resources: + - operator + + - instances + + - auth + + - datasources + + - folders + + - dashboards + +labels: + - pairs: + app: exploit-iq + component: grafana + +namePrefix: exploit-iq- + +configMapGenerator: + - name: replacement-values + literals: + - route-suffix=route"}} + +replacements: + - source: + kind: Grafana + name: grafana + fieldPath: metadata.namespace + targets: + - select: + kind: GrafanaDatasource + name: infinity + fieldPaths: + - spec.datasource.url + - spec.datasource.jsonData.allowedHosts.0 + options: + delimiter: "." + index: 1 + - source: + kind: Subscription + name: grafana-operator + fieldPath: metadata.namespace + targets: + - select: + kind: OperatorGroup + name: grafana-operators + fieldPaths: + - spec.targetNamespaces.0 + - source: + kind: ServiceAccount + name: grafana-sa + fieldPath: metadata.name + targets: + - select: + kind: Grafana + name: grafana + fieldPaths: + - spec.deployment.spec.template.spec.containers.[name=grafana-oauth-proxy].args.0 # updates the -openshift-service-account argument with the new service account name + options: + delimiter: "=" + index: 1 + - source: + kind: ServiceAccount + name: grafana-sa + fieldPath: metadata.name + targets: + - select: + kind: ServiceAccount + name: grafana-sa + fieldPaths: + - metadata.annotations.serviceaccounts\.openshift\.io/oauth-redirectreference\.primary + options: + delimiter: '"name":"' + index: 1 + - source: + kind: ConfigMap + name: replacement-values + fieldPath: data.route-suffix + targets: + - select: + kind: ServiceAccount + name: grafana-sa + fieldPaths: + - metadata.annotations.serviceaccounts\.openshift\.io/oauth-redirectreference\.primary + options: + delimiter: '-' + index: 3 \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/operator/grafana-subscription.yaml b/kustomize/overlays/mlops/grafana/operator/grafana-subscription.yaml new file mode 100644 index 000000000..98c4bca4d --- /dev/null +++ b/kustomize/overlays/mlops/grafana/operator/grafana-subscription.yaml @@ -0,0 +1,10 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: grafana-operator +spec: + channel: v5 + installPlanApproval: Automatic + name: grafana-operator + source: community-operators + sourceNamespace: openshift-marketplace \ No newline at end of file diff --git a/kustomize/overlays/mlops/grafana/operator/kustomization.yaml b/kustomize/overlays/mlops/grafana/operator/kustomization.yaml new file mode 100644 index 000000000..6ff410c89 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/operator/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - grafana-subscription.yaml + - operator-group.yaml diff --git a/kustomize/overlays/mlops/grafana/operator/operator-group.yaml b/kustomize/overlays/mlops/grafana/operator/operator-group.yaml new file mode 100644 index 000000000..a5902be80 --- /dev/null +++ b/kustomize/overlays/mlops/grafana/operator/operator-group.yaml @@ -0,0 +1,7 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: grafana-operators +spec: + targetNamespaces: + - diff --git a/kustomize/overlays/mlops/kustomization.yaml b/kustomize/overlays/mlops/kustomization.yaml new file mode 100644 index 000000000..e068e67c7 --- /dev/null +++ b/kustomize/overlays/mlops/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../self-hosted-llama3.1-70b-4bit + # Grafana monitoring resources + - grafana + # Tempo tracing (same namespace as Grafana for non-CI; CI uses grafana-tempo-poc) + - tempo + +commonAnnotations: + deployment-specialization: mlops \ No newline at end of file diff --git a/kustomize/overlays/mlops/tempo/kustomization.yaml b/kustomize/overlays/mlops/tempo/kustomization.yaml new file mode 100644 index 000000000..ed8b8dec7 --- /dev/null +++ b/kustomize/overlays/mlops/tempo/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: REPLACE_NAMESPACE + +resources: + - tempo-deployment.yaml + - tempo-scc.yaml + +labels: + - pairs: + app: exploit-iq + component: tempo diff --git a/kustomize/overlays/mlops/tempo/tempo-deployment.yaml b/kustomize/overlays/mlops/tempo/tempo-deployment.yaml new file mode 100644 index 000000000..2412a051f --- /dev/null +++ b/kustomize/overlays/mlops/tempo/tempo-deployment.yaml @@ -0,0 +1,134 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tempo-sa +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: tempo-conf +data: + tempo.yaml: | + server: + http_listen_port: 3200 + + distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + + ingester: + max_block_duration: 5m + lifecycler: + address: 0.0.0.0 + final_sleep: 0s + ring: + kvstore: + store: inmemory + replication_factor: 1 + + # Storage paths (backed by PVC so data survives restarts) + storage: + trace: + backend: local + local: + path: /var/tempo/traces + wal: + path: /var/tempo/wal + + # Metrics generator for rate() and other TraceQL metrics (Tempo 2.6+) + metrics_generator: + registry: + external_labels: + source: tempo + storage: + path: /var/tempo/generator/wal + + # Allow querying up to 7 days (fixes "Selected time range exceeds...") + query_frontend: + search: + max_duration: 168h + + # Define retention so traces have a clear "shelf life" (fixes "No data for selected query") + compactor: + compaction: + block_retention: 168h + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: tempo-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi + # Optional: uncomment and set a StorageClass name if your cluster needs it + # storageClassName: standard + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tempo +spec: + replicas: 1 + selector: + matchLabels: + app: tempo + template: + metadata: + labels: + app: tempo + spec: + serviceAccountName: tempo-sa + # So the PVC is writable by Tempo (runs as UID 10001); fixes "mkdir /var/tempo/traces: permission denied" + securityContext: + fsGroup: 10001 + containers: + - name: tempo + image: grafana/tempo:2.6.1 + args: ["-config.file=/etc/tempo/tempo.yaml"] + ports: + - name: tempo-http + containerPort: 3200 + - name: otlp-grpc + containerPort: 4317 + - name: otlp-http + containerPort: 4318 + volumeMounts: + - name: config + mountPath: /etc/tempo + - name: data + mountPath: /var/tempo + volumes: + - name: config + configMap: + name: tempo-conf + - name: data + persistentVolumeClaim: + claimName: tempo-data +--- +apiVersion: v1 +kind: Service +metadata: + name: tempo +spec: + ports: + - name: tempo-http + port: 3200 + targetPort: 3200 + - name: otlp-grpc + port: 4317 + targetPort: 4317 + - name: otlp-http + port: 4318 + targetPort: 4318 + selector: + app: tempo \ No newline at end of file diff --git a/kustomize/overlays/mlops/tempo/tempo-scc.yaml b/kustomize/overlays/mlops/tempo/tempo-scc.yaml new file mode 100644 index 000000000..eaccdbc6a --- /dev/null +++ b/kustomize/overlays/mlops/tempo/tempo-scc.yaml @@ -0,0 +1,13 @@ +# Allow Tempo pod to use fsGroup 10001 (required for Tempo's data dir on PVC). +# OpenShift: bind the built-in anyuid SCC ClusterRole to tempo-sa (same pattern as base/anyuid-scc-permission.yaml). +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: tempo-anyuid-scc +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:openshift:scc:anyuid +subjects: + - kind: ServiceAccount + name: tempo-sa diff --git a/kustomize/overlays/remote-nim-all/exploit-iq-nim-patch.yaml b/kustomize/overlays/remote-nim-all/exploit-iq-nim-patch.yaml new file mode 100644 index 000000000..15b18a3a6 --- /dev/null +++ b/kustomize/overlays/remote-nim-all/exploit-iq-nim-patch.yaml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: exploit-iq + labels: + app: exploit-iq + component: exploit-iq + annotations: + api-key: &api-key "EXPLOIT_IQ" + nim-llm-baseurl: &nim-llm-baseurl "http://nginx-cache:8080/nim_llm/v1" +spec: + selector: + matchLabels: + app: exploit-iq + component: exploit-iq + template: + metadata: + labels: + app: exploit-iq + component: exploit-iq + spec: + containers: + - name: exploit-iq + imagePullPolicy: Always + workingDir: /workspace/ + env: + - name: LLM_API_KEY_CHECKLIST + value: *api-key + - name: LLM_API_KEY_CODE_VDB_RETRIEVER + value: *api-key + - name: LLM_API_KEY_DOC_VDB_RETRIEVER + value: *api-key + - name: LLM_API_KEY_AGENT_EXECUTOR + value: *api-key + - name: LLM_API_KEY_SUMMARIZE + value: *api-key + - name: LLM_API_KEY_JUSTIFY + value: *api-key + - name: LLM_API_KEY_INTEL_SOURCE_SCORE + value: *api-key + - name: CHECKLIST_LLM_API_BASE + value: *nim-llm-baseurl + - name: CODE_VDB_RETRIEVER_API_BASE + value: *nim-llm-baseurl + - name: DOC_VDB_RETRIEVER_API_BASE + value: *nim-llm-baseurl + - name: AGENT_EXECUTOR_LLM_API_BASE + value: *nim-llm-baseurl + - name: GENERATE_CVSS_LLM_API_BASE + value: *nim-llm-baseurl + - name: SUMMARIZE_LLM_API_BASE + value: *nim-llm-baseurl + - name: JUSTIFY_LLM_API_BASE + value: *nim-llm-baseurl + - name: INTEL_SOURCE_SCORE_LLM_API_BASE + value: *nim-llm-baseurl diff --git a/kustomize/overlays/remote-nim-all/kustomization.yaml b/kustomize/overlays/remote-nim-all/kustomization.yaml new file mode 100644 index 000000000..56689c402 --- /dev/null +++ b/kustomize/overlays/remote-nim-all/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base + +commonAnnotations: + deployment-variant: remote-nim + +patches: +- path: nginx-patch.yaml +- path: exploit-iq-nim-patch.yaml diff --git a/kustomize/overlays/remote-nim-all/nginx-patch.yaml b/kustomize/overlays/remote-nim-all/nginx-patch.yaml new file mode 100644 index 000000000..e7fb543c6 --- /dev/null +++ b/kustomize/overlays/remote-nim-all/nginx-patch.yaml @@ -0,0 +1,16 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-cache +spec: + template: + spec: + initContainers: + - name: render-nginx-config + env: + - name: NGINX_UPSTREAM_NIM_LLM + value: https://integrate.api.nvidia.com + - name: NGINX_UPSTREAM_NIM_EMBED + value: https://integrate.api.nvidia.com + - name: NGINX_UPSTREAM_OPENAI + value: https://integrate.api.nvidia.com diff --git a/kustomize/overlays/self-hosted-llama3.1-70b-4bit/exploit-iq-llm-patch.yaml b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/exploit-iq-llm-patch.yaml new file mode 100644 index 000000000..1aad91928 --- /dev/null +++ b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/exploit-iq-llm-patch.yaml @@ -0,0 +1,88 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: exploit-iq + labels: + app: exploit-iq + component: exploit-iq + annotations: + llm-type: &llm-type openai + api-key: &api-key "EMPTY" + openai-llm-baseurl: &openai-llm-baseurl http://nginx-cache:8080/openai/v1 + model-name: &model-name hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4 +spec: + selector: + matchLabels: + app: exploit-iq + component: exploit-iq + template: + metadata: + labels: + app: exploit-iq + component: exploit-iq + spec: + containers: + - name: exploit-iq + imagePullPolicy: Always + workingDir: /workspace/ + env: + - name: LLM_TYPE_CHECKLIST + value: *llm-type + - name: LLM_TYPE_VDB_CODE_RETRIEVER + value: *llm-type + - name: LLM_TYPE_VDB_DOC_RETRIEVER + value: *llm-type + - name: LLM_TYPE_AGENT_EXECUTOR + value: *llm-type + - name: LLM_TYPE_SUMMARIZE + value: *llm-type + - name: LLM_TYPE_JUSTIFY + value: *llm-type + - name: LLM_TYPE_INTEL_SOURCE_SCORE + value: *llm-type + - name: LLM_API_KEY_CHECKLIST + value: *api-key + - name: LLM_API_KEY_CODE_VDB_RETRIEVER + value: *api-key + - name: LLM_API_KEY_DOC_VDB_RETRIEVER + value: *api-key + - name: LLM_API_KEY_AGENT_EXECUTOR + value: *api-key + - name: LLM_API_KEY_SUMMARIZE + value: *api-key + - name: LLM_API_KEY_JUSTIFY + value: *api-key + - name: LLM_API_KEY_INTEL_SOURCE_SCORE + value: *api-key + - name: CHECKLIST_LLM_API_BASE + value: *openai-llm-baseurl + - name: CODE_VDB_RETRIEVER_API_BASE + value: *openai-llm-baseurl + - name: DOC_VDB_RETRIEVER_API_BASE + value: *openai-llm-baseurl + - name: AGENT_EXECUTOR_LLM_API_BASE + value: *openai-llm-baseurl + - name: GENERATE_CVSS_LLM_API_BASE + value: *openai-llm-baseurl + - name: SUMMARIZE_LLM_API_BASE + value: *openai-llm-baseurl + - name: JUSTIFY_LLM_API_BASE + value: *openai-llm-baseurl + - name: INTEL_SOURCE_SCORE_LLM_API_BASE + value: *openai-llm-baseurl + - name: CHECKLIST_MODEL_NAME + value: *model-name + - name: CODE_VDB_RETRIEVER_MODEL_NAME + value: *model-name + - name: DOC_VDB_RETRIEVER_MODEL_NAME + value: *model-name + - name: AGENT_EXECUTOR_MODEL_NAME + value: *model-name + - name: SUMMARIZE_MODEL_NAME + value: *model-name + - name: JUSTIFY_MODEL_NAME + value: *model-name + - name: INTEL_SOURCE_SCORE_MODEL_NAME + value: *model-name + - name: GENERATE_CVSS_MODEL_NAME + value: *model-name diff --git a/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml new file mode 100644 index 000000000..c0948da4c --- /dev/null +++ b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base + +commonAnnotations: + deployment-variant: local-llama3.1-70b-4bit + +patches: +- path: nginx-patch.yaml +- path: exploit-iq-llm-patch.yaml diff --git a/kustomize/overlays/self-hosted-llama3.1-70b-4bit/nginx-patch.yaml b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/nginx-patch.yaml new file mode 100644 index 000000000..8e3fb5e70 --- /dev/null +++ b/kustomize/overlays/self-hosted-llama3.1-70b-4bit/nginx-patch.yaml @@ -0,0 +1,16 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-cache +spec: + template: + spec: + initContainers: + - name: render-nginx-config + env: + - name: NGINX_UPSTREAM_NIM_LLM + value: http://llama3-1-70b-instruct-4bit.exploit-iq-models.svc.cluster.local:8000 + - name: NGINX_UPSTREAM_NIM_EMBED + value: http://llama3-1-70b-instruct-4bit.exploit-iq-models.svc.cluster.local:8000 + - name: NGINX_UPSTREAM_OPENAI + value: http://llama3-1-70b-instruct-4bit.exploit-iq-models.svc.cluster.local:8000 diff --git a/kustomize/overlays/tests/buildah-task.yaml b/kustomize/overlays/tests/buildah-task.yaml new file mode 100644 index 000000000..953dceaff --- /dev/null +++ b/kustomize/overlays/tests/buildah-task.yaml @@ -0,0 +1,167 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: buildah-pvc + +spec: + description: | + Buildah task builds source into a container image and + then pushes it to a container registry. + params: + - description: | + Fully qualified container image name to be built by buildah. + name: IMAGE + type: string + - default: ./Dockerfile + description: | + Path to the `Dockerfile` (or `Containerfile`) relative to the `source` workspace. + name: DOCKERFILE + type: string + - default: + - '' + description: | + Dockerfile build arguments, array of key=value + name: BUILD_ARGS + type: array + - default: . + description: | + Path to the directory to use as context. + name: CONTEXT + type: string + - default: overlay + description: | + Set buildah storage driver to reflect the currrent cluster node's + settings. + name: STORAGE_DRIVER + type: string + - default: oci + description: 'The format of the built container, oci or docker' + name: FORMAT + type: string + - default: '' + description: | + Extra parameters passed for the build command when building images. + name: BUILD_EXTRA_ARGS + type: string + - default: '' + description: | + Extra parameters passed for the push command when pushing images. + name: PUSH_EXTRA_ARGS + type: string + - default: 'false' + description: | + Skip pushing the image to the container registry. + name: SKIP_PUSH + type: string + - default: 'true' + description: | + Sets the TLS verification flag, `true` is recommended. + name: TLS_VERIFY + type: string + - default: 'false' + description: | + Turns on verbose logging, all commands executed will be printed out. + name: VERBOSE + type: string + results: + - description: | + Fully qualified image name. + name: IMAGE_URL + type: string + - description: | + Digest of the image just built. + name: IMAGE_DIGEST + type: string + stepTemplate: + computeResources: + requests: + memory: 5Gi + cpu: 1 + limits: + memory: 10Gi + cpu: 2 + env: + - name: PARAMS_IMAGE + value: $(params.IMAGE) + - name: PARAMS_CONTEXT + value: $(params.CONTEXT) + - name: PARAMS_DOCKERFILE + value: $(params.DOCKERFILE) + - name: PARAMS_FORMAT + value: $(params.FORMAT) + - name: PARAMS_STORAGE_DRIVER + value: $(params.STORAGE_DRIVER) + - name: PARAMS_BUILD_EXTRA_ARGS + value: $(params.BUILD_EXTRA_ARGS) + - name: PARAMS_PUSH_EXTRA_ARGS + value: $(params.PUSH_EXTRA_ARGS) + - name: PARAMS_SKIP_PUSH + value: $(params.SKIP_PUSH) + - name: PARAMS_TLS_VERIFY + value: $(params.TLS_VERIFY) + - name: PARAMS_VERBOSE + value: $(params.VERBOSE) + - name: WORKSPACES_SOURCE_BOUND + value: $(workspaces.source.bound) + - name: WORKSPACES_SOURCE_PATH + value: $(workspaces.source.path) + - name: WORKSPACES_DOCKERCONFIG_BOUND + value: $(workspaces.dockerconfig.bound) + - name: WORKSPACES_DOCKERCONFIG_PATH + value: $(workspaces.dockerconfig.path) + - name: WORKSPACES_RHEL_ENTITLEMENT_BOUND + value: $(workspaces.rhel-entitlement.bound) + - name: WORKSPACES_RHEL_ENTITLEMENT_PATH + value: $(workspaces.rhel-entitlement.path) + - name: RESULTS_IMAGE_URL_PATH + value: $(results.IMAGE_URL.path) + - name: RESULTS_IMAGE_DIGEST_PATH + value: $(results.IMAGE_DIGEST.path) + steps: + - args: + - '$(params.BUILD_ARGS[*])' + computeResources: {} + image: 'registry.redhat.io/rhel8/buildah@sha256:6d2dcb651ba680cf4ec74331f8349dec43d071d420625a1703370acc8d984e9e' + name: build + script: | + set -e + printf '%s' "IyEvdXNyL2Jpbi9lbnYgYmFzaAojCiMgV3JhcHBlciBhcm91bmQgImJ1aWxkYWggYnVkIiB0byBidWlsZCBhbmQgcHVzaCBhIGNvbnRhaW5lciBpbWFnZSBiYXNlZCBvbiBhIERvY2tlcmZpbGUuCiMKCnNob3B0IC1zIGluaGVyaXRfZXJyZXhpdApzZXQgLWV1IC1vIHBpcGVmYWlsCgpzb3VyY2UgIiQoZGlybmFtZSAke0JBU0hfU09VUkNFWzBdfSkvY29tbW9uLnNoIgpzb3VyY2UgIiQoZGlybmFtZSAke0JBU0hfU09VUkNFWzBdfSkvYnVpbGRhaC1jb21tb24uc2giCgpmdW5jdGlvbiBfYnVpbGRhaCgpIHsKICAgIGJ1aWxkYWggXAogICAgICAgIC0tc3RvcmFnZS1kcml2ZXI9IiR7UEFSQU1TX1NUT1JBR0VfRFJJVkVSfSIgXAogICAgICAgIC0tdGxzLXZlcmlmeT0iJHtQQVJBTVNfVExTX1ZFUklGWX0iIFwKICAgICAgICAiJEAiCn0KCiMKIyBQcmVwYXJlCiMKCiMgbWFraW5nIHN1cmUgdGhlIHJlcXVpcmVkIHdvcmtzcGFjZSAic291cmNlIiBpcyBib3VuZGVkLCB3aGljaCBtZWFucyBpdHMgdm9sdW1lIGlzIGN1cnJlbnRseSBtb3VudGVkCiMgYW5kIHJlYWR5IHRvIHVzZQpwaGFzZSAiSW5zcGVjdGluZyBzb3VyY2Ugd29ya3NwYWNlICcke1dPUktTUEFDRVNfU09VUkNFX1BBVEh9JyAoUFdEPScke1BXRH0nKSIKW1sgIiR7V09SS1NQQUNFU19TT1VSQ0VfQk9VTkR9IiAhPSAidHJ1ZSIgXV0gJiYKICAgIGZhaWwgIldvcmtzcGFjZSAnc291cmNlJyBpcyBub3QgYm91bmRlZCIKCnBoYXNlICJBc3NlcnRpbmcgdGhlIGRvY2tlcmZpbGUvY29udGFpbmVyZmlsZSAnJHtET0NLRVJGSUxFX0ZVTEx9JyBleGlzdHMiCltbICEgLWYgIiR7RE9DS0VSRklMRV9GVUxMfSIgXV0gJiYKICAgIGZhaWwgIkRvY2tlcmZpbGUgbm90IGZvdW5kIGF0OiAnJHtET0NLRVJGSUxFX0ZVTEx9JyIKCnBoYXNlICJJbnNwZWN0aW5nIGNvbnRleHQgJyR7UEFSQU1TX0NPTlRFWFR9JyIKW1sgISAtZCAiJHtQQVJBTVNfQ09OVEVYVH0iIF1dICYmCiAgICBmYWlsICJDT05URVhUIHBhcmFtIGlzIG5vdCBmb3VuZCBhdCAnJHtQQVJBTVNfQ09OVEVYVH0nLCBvbiBzb3VyY2Ugd29ya3NwYWNlIgoKcGhhc2UgIkJ1aWxkaW5nIGJ1aWxkIGFyZ3MiCkJVSUxEX0FSR1M9KCkKZm9yIGJ1aWxkYXJnIGluICIkQCI7IGRvCiAgICBCVUlMRF9BUkdTKz0oIi0tYnVpbGQtYXJnPSRidWlsZGFyZyIpCmRvbmUKCiMgSGFuZGxlIG9wdGlvbmFsIGRvY2tlcmNvbmZpZyBzZWNyZXQKaWYgW1sgIiR7V09SS1NQQUNFU19ET0NLRVJDT05GSUdfQk9VTkR9IiA9PSAidHJ1ZSIgXV07IHRoZW4KCiAgICAjIGlmIGNvbmZpZy5qc29uIGV4aXN0cyBhdCB3b3Jrc3BhY2Ugcm9vdCwgd2UgdXNlIHRoYXQKICAgIGlmIHRlc3QgLWYgIiR7V09SS1NQQUNFU19ET0NLRVJDT05GSUdfUEFUSH0vY29uZmlnLmpzb24iOyB0aGVuCiAgICAgICAgZXhwb3J0IERPQ0tFUl9DT05GSUc9IiR7V09SS1NQQUNFU19ET0NLRVJDT05GSUdfUEFUSH0iCgogICAgICAgICMgZWxzZSB3ZSBsb29rIGZvciAuZG9ja2VyY29uZmlnanNvbiBhdCB0aGUgcm9vdAogICAgZWxpZiB0ZXN0IC1mICIke1dPUktTUEFDRVNfRE9DS0VSQ09ORklHX1BBVEh9Ly5kb2NrZXJjb25maWdqc29uIjsgdGhlbgogICAgICAgICMgZW5zdXJlIC5kb2NrZXIgZXhpc3QgYmVmb3JlIHRoZSBjb3B5aW5nIHRoZSBjb250ZW50CiAgICAgICAgaWYgWyAhIC1kICIkSE9NRS8uZG9ja2VyIiBdOyB0aGVuCiAgICAgICAgICAgbWtkaXIgLXAgIiRIT01FLy5kb2NrZXIiCiAgICAgICAgZmkKICAgICAgICBjcCAiJHtXT1JLU1BBQ0VTX0RPQ0tFUkNPTkZJR19QQVRIfS8uZG9ja2VyY29uZmlnanNvbiIgIiRIT01FLy5kb2NrZXIvY29uZmlnLmpzb24iCiAgICAgICAgZXhwb3J0IERPQ0tFUl9DT05GSUc9IiRIT01FLy5kb2NrZXIiCgogICAgICAgICMgbmVlZCB0byBlcnJvciBvdXQgaWYgbmVpdGhlciBmaWxlcyBhcmUgcHJlc2VudAogICAgZWxzZQogICAgICAgIGVjaG8gIm5laXRoZXIgJ2NvbmZpZy5qc29uJyBub3IgJy5kb2NrZXJjb25maWdqc29uJyBmb3VuZCBhdCB3b3Jrc3BhY2Ugcm9vdCIKICAgICAgICBleGl0IDEKICAgIGZpCmZpCgpFTlRJVExFTUVOVF9WT0xVTUU9IiIKaWYgW1sgIiR7V09SS1NQQUNFU19SSEVMX0VOVElUTEVNRU5UX0JPVU5EfSIgPT0gInRydWUiIF1dOyB0aGVuCiAgICBFTlRJVExFTUVOVF9WT0xVTUU9Ii0tdm9sdW1lICR7V09SS1NQQUNFU19SSEVMX0VOVElUTEVNRU5UX1BBVEh9Oi9ldGMvcGtpL2VudGl0bGVtZW50OnJvIgpmaQoKIwojIEJ1aWxkCiMKCnBoYXNlICJCdWlsZGluZyAnJHtQQVJBTVNfSU1BR0V9JyBiYXNlZCBvbiAnJHtET0NLRVJGSUxFX0ZVTEx9JyIKCltbIC1uICIke1BBUkFNU19CVUlMRF9FWFRSQV9BUkdTfSIgXV0gJiYKICAgIHBoYXNlICJFeHRyYSAnYnVpbGRhaCBidWQnIGFyZ3VtZW50cyBpbmZvcm1lZDogJyR7UEFSQU1TX0JVSUxEX0VYVFJBX0FSR1N9JyIKCiMgUHJvY2VzcyBCVUlMRF9FWFRSQV9BUkdTCmJ1aWxkX2V4dHJhX2FyZ3NfdG1wPSQoZWNobyAiJHtQQVJBTVNfQlVJTERfRVhUUkFfQVJHUzotfSIgfCB4YXJncyAtbjEpCmlmIFtbIC1uICIkYnVpbGRfZXh0cmFfYXJnc190bXAiIF1dOyB0aGVuCiAgICByZWFkYXJyYXkgLXQgYnVpbGRfZXh0cmFfYXJncyA8PDwgIiRidWlsZF9leHRyYV9hcmdzX3RtcCIKZWxzZQogICAgYnVpbGRfZXh0cmFfYXJncz0oKSAjIEVtcHR5IGFycmF5IGlmIG5vIGV4dHJhIGFyZ3MKZmkKCl9idWlsZGFoIGJ1ZCAiJHtidWlsZF9leHRyYV9hcmdzW0BdfSIgXAogICAgJEVOVElUTEVNRU5UX1ZPTFVNRSBcCiAgICAiJHtCVUlMRF9BUkdTW0BdfSIgXAogICAgLS1maWxlPSIke0RPQ0tFUkZJTEVfRlVMTH0iIFwKICAgIC0tdGFnPSIke1BBUkFNU19JTUFHRX0iIFwKICAgICIke1BBUkFNU19DT05URVhUfSIKCmlmIFtbICIke1BBUkFNU19TS0lQX1BVU0h9IiA9PSAidHJ1ZSIgXV07IHRoZW4KICAgIHBoYXNlICJTa2lwcGluZyBwdXNoaW5nICcke1BBUkFNU19JTUFHRX0nIHRvIHRoZSBjb250YWluZXIgcmVnaXN0cnkhIgogICAgZXhpdCAwCmZpCgojCiMgUHVzaAojCgpwaGFzZSAiUHVzaGluZyAnJHtQQVJBTVNfSU1BR0V9JyB0byB0aGUgY29udGFpbmVyIHJlZ2lzdHJ5IgoKW1sgLW4gIiR7UEFSQU1TX1BVU0hfRVhUUkFfQVJHU30iIF1dICYmCiAgICBwaGFzZSAiRXh0cmEgJ2J1aWxkYWggcHVzaCcgYXJndW1lbnRzIGluZm9ybWVkOiAnJHtQQVJBTVNfUFVTSF9FWFRSQV9BUkdTfSciCgojIHRlbXBvcmFyeSBmaWxlIHRvIHN0b3JlIHRoZSBpbWFnZSBkaWdlc3QsIGluZm9ybWF0aW9uIG9ubHkgb2J0YWluZWQgYWZ0ZXIgcHVzaGluZyB0aGUgaW1hZ2UgdG8gdGhlCiMgY29udGFpbmVyIHJlZ2lzdHJ5CmRlY2xhcmUgLXIgZGlnZXN0X2ZpbGU9Ii90bXAvYnVpbGRhaC1kaWdlc3QudHh0IgoKIyBQcm9jZXNzIFBVU0hfRVhUUkFfQVJHUwpwdXNoX2V4dHJhX2FyZ3NfdG1wPSQoZWNobyAiJHtQQVJBTVNfUFVTSF9FWFRSQV9BUkdTOi19IiB8IHhhcmdzIC1uMSkKaWYgW1sgLW4gIiRwdXNoX2V4dHJhX2FyZ3NfdG1wIiBdXTsgdGhlbgogICAgcmVhZGFycmF5IC10IHB1c2hfZXh0cmFfYXJncyA8PDwgIiRwdXNoX2V4dHJhX2FyZ3NfdG1wIgplbHNlCiAgICBwdXNoX2V4dHJhX2FyZ3M9KCkgIyBFbXB0eSBhcnJheSBpZiBubyBleHRyYSBhcmdzCmZpCgpfYnVpbGRhaCBwdXNoICIke3B1c2hfZXh0cmFfYXJnc1tAXX0iIFwKICAgIC0tZGlnZXN0ZmlsZT0iJHtkaWdlc3RfZmlsZX0iIFwKICAgICIke1BBUkFNU19JTUFHRX0iIFwKICAgICJkb2NrZXI6Ly8ke1BBUkFNU19JTUFHRX0iCgojCiMgUmVzdWx0cwojCgpwaGFzZSAiSW5zcGVjdGluZyBkaWdlc3QgcmVwb3J0ICgnJHtkaWdlc3RfZmlsZX0nKSIKCltbICEgLXIgIiR7ZGlnZXN0X2ZpbGV9IiBdXSAmJgogICAgZmFpbCAiVW5hYmxlIHRvIGZpbmQgZGlnZXN0LWZpbGUgYXQgJyR7ZGlnZXN0X2ZpbGV9JyIKCmRlY2xhcmUgLXIgZGlnZXN0X3N1bT0iJChjYXQgJHtkaWdlc3RfZmlsZX0pIgoKW1sgLXogIiR7ZGlnZXN0X3N1bX0iIF1dICYmCiAgICBmYWlsICJEaWdlc3QgZmlsZSAnJHtkaWdlc3RfZmlsZX0nIGlzIGVtcHR5ISIKCnBoYXNlICJTdWNjZXNzZnVseSBidWlsdCBjb250YWluZXIgaW1hZ2UgJyR7UEFSQU1TX0lNQUdFfScgKCcke2RpZ2VzdF9zdW19JykiCmVjaG8gLW4gIiR7UEFSQU1TX0lNQUdFfSIgfCB0ZWUgJHtSRVNVTFRTX0lNQUdFX1VSTF9QQVRIfQplY2hvIC1uICIke2RpZ2VzdF9zdW19IiB8IHRlZSAke1JFU1VMVFNfSU1BR0VfRElHRVNUX1BBVEh9Cg==" |base64 -d >"/scripts/buildah-bud.sh" + printf '%s' "IyEvdXNyL2Jpbi9lbnYgYmFzaAoKZGVjbGFyZSAtcnggUEFSQU1TX0lNQUdFPSIke1BBUkFNU19JTUFHRTotfSIKZGVjbGFyZSAtcnggUEFSQU1TX0RPQ0tFUkZJTEU9IiR7UEFSQU1TX0RPQ0tFUkZJTEU6LX0iCmRlY2xhcmUgLXggUEFSQU1TX0NPTlRFWFQ9IiR7UEFSQU1TX0NPTlRFWFQ6LX0iCmRlY2xhcmUgLXJ4IFBBUkFNU19TVE9SQUdFX0RSSVZFUj0iJHtQQVJBTVNfU1RPUkFHRV9EUklWRVI6LX0iCmRlY2xhcmUgLXJ4IFBBUkFNU19CVUlMRF9FWFRSQV9BUkdTPSIke1BBUkFNU19CVUlMRF9FWFRSQV9BUkdTOi19IgpkZWNsYXJlIC1yeCBQQVJBTVNfUFVTSF9FWFRSQV9BUkdTPSIke1BBUkFNU19QVVNIX0VYVFJBX0FSR1M6LX0iCmRlY2xhcmUgLXJ4IFBBUkFNU19TS0lQX1BVU0g9IiR7UEFSQU1TX1NLSVBfUFVTSDotfSIKZGVjbGFyZSAtcnggUEFSQU1TX1RMU19WRVJJRlk9IiR7UEFSQU1TX1RMU19WRVJJRlk6LX0iCmRlY2xhcmUgLXJ4IFBBUkFNU19WRVJCT1NFPSIke1BBUkFNU19WRVJCT1NFOi19IgoKZGVjbGFyZSAtcnggV09SS1NQQUNFU19TT1VSQ0VfUEFUSD0iJHtXT1JLU1BBQ0VTX1NPVVJDRV9QQVRIOi19IgpkZWNsYXJlIC1yeCBXT1JLU1BBQ0VTX1NPVVJDRV9CT1VORD0iJHtXT1JLU1BBQ0VTX1NPVVJDRV9CT1VORDotfSIKZGVjbGFyZSAtcnggV09SS1NQQUNFU19ET0NLRVJDT05GSUdfUEFUSD0iJHtXT1JLU1BBQ0VTX0RPQ0tFUkNPTkZJR19QQVRIOi19IgpkZWNsYXJlIC1yeCBXT1JLU1BBQ0VTX0RPQ0tFUkNPTkZJR19CT1VORD0iJHtXT1JLU1BBQ0VTX0RPQ0tFUkNPTkZJR19CT1VORDotfSIKZGVjbGFyZSAtcnggV09SS1NQQUNFU19SSEVMX0VOVElUTEVNRU5UX1BBVEg9IiR7V09SS1NQQUNFU19SSEVMX0VOVElUTEVNRU5UX1BBVEg6LX0iCmRlY2xhcmUgLXJ4IFdPUktTUEFDRVNfUkhFTF9FTlRJVExFTUVOVF9CT1VORD0iJHtXT1JLU1BBQ0VTX1JIRUxfRU5USVRMRU1FTlRfQk9VTkQ6LX0iCgpkZWNsYXJlIC1yeCBSRVNVTFRTX0lNQUdFX0RJR0VTVF9QQVRIPSIke1JFU1VMVFNfSU1BR0VfRElHRVNUX1BBVEg6LX0iCmRlY2xhcmUgLXJ4IFJFU1VMVFNfSU1BR0VfVVJMX1BBVEg9IiR7UkVTVUxUU19JTUFHRV9VUkxfUEFUSDotfSIKCiMKIyBEb2NrZXJmaWxlCiMKCiMgZXhwb3NpbmcgdGhlIGZ1bGwgcGF0aCB0byB0aGUgY29udGFpbmVyIGZpbGUsIHdoaWNoIGJ5IGRlZmF1bHQgc2hvdWxkIGJlIHJlbGF0aXZlIHRvIHRoZSBwcmltYXJ5CiMgd29ya3NwYWNlLCB0byByZWNlaXZlIGEgZGlmZmVyZW50IGNvbnRhaW5lci1maWxlIGxvY2F0aW9uCmRlY2xhcmUgLXIgZG9ja2VyZmlsZV9vbl93cz0iJHtXT1JLU1BBQ0VTX1NPVVJDRV9QQVRIfS8ke1BBUkFNU19ET0NLRVJGSUxFfSIKZGVjbGFyZSAteCBET0NLRVJGSUxFX0ZVTEw9IiR7RE9DS0VSRklMRV9GVUxMOi0ke2RvY2tlcmZpbGVfb25fd3N9fSIKCiMKIyBBc3NlcnRpbmcgRW52aXJvbm1lbnQKIwoKW1sgLXogIiR7RE9DS0VSRklMRV9GVUxMfSIgXV0gJiYKICAgIGZhaWwgInVuYWJsZSB0byBmaW5kIHRoZSBEb2NrZXJmaWxlLCBET0NLRVJGSUxFIG1heSBoYXZlIGFuIGluY29ycmVjdCBsb2NhdGlvbiIKCmV4cG9ydGVkX29yX2ZhaWwgXAogICAgV09SS1NQQUNFU19TT1VSQ0VfUEFUSCBcCiAgICBQQVJBTVNfSU1BR0UKCiMKIyBWZXJib3NlIE91dHB1dAojCgppZiBbWyAiJHtQQVJBTVNfVkVSQk9TRX0iID09ICJ0cnVlIiBdXTsgdGhlbgogICAgc2V0IC14CmZpCg==" |base64 -d >"/scripts/buildah-common.sh" + printf '%s' "IyEvdXNyL2Jpbi9lbnYgYmFzaAoKIyB0ZWt0b24ncyBob21lIGRpcmVjdG9yeQpkZWNsYXJlIC1yeCBURUtUT05fSE9NRT0iJHtURUtUT05fSE9NRTotL3Rla3Rvbi9ob21lfSIKCiMKIyBGdW5jdGlvbnMKIwoKZnVuY3Rpb24gZmFpbCgpIHsKICAgIGVjaG8gIkVSUk9SOiAkeyp9IiAyPiYxCiAgICBleGl0IDEKfQoKZnVuY3Rpb24gcGhhc2UoKSB7CiAgICBlY2hvICItLS0+IFBoYXNlOiAkeyp9Li4uIgp9CgojIGFzc2VydCBsb2NhbCB2YXJpYWJsZXMgYXJlIGV4cG9ydGVkIG9uIHRoZSBlbnZpcm9ubWVudApmdW5jdGlvbiBleHBvcnRlZF9vcl9mYWlsKCkgewogICAgZGVjbGFyZSAtYSBfcmVxdWlyZWRfdmFycz0iJHtAfSIKCiAgICBmb3IgdiBpbiAke19yZXF1aXJlZF92YXJzW0BdfTsgZG8KICAgICAgICBbWyAteiAiJHshdn0iIF1dICYmCiAgICAgICAgICAgIGZhaWwgIicke3Z9JyBlbnZpcm9ubWVudCB2YXJpYWJsZSBpcyBub3Qgc2V0ISIKICAgIGRvbmUKCiAgICByZXR1cm4gMAp9Cg==" |base64 -d >"/scripts/common.sh" + ls /scripts/buildah-*.sh; + chmod +x /scripts/buildah-*.sh;echo "Running Script /scripts/buildah-bud.sh"; + /scripts/buildah-bud.sh; + securityContext: + capabilities: + add: + - SETFCAP + volumeMounts: + - mountPath: /scripts + name: scripts-dir + workingDir: $(workspaces.source.path) + volumes: + - emptyDir: {} + name: scripts-dir + workspaces: + - description: | + Container build context, like for instnace a application source code + followed by a `Dockerfile`. + name: source + - description: | + Storage root dir for the spaces required by buildah when building images. + mountPath: /var/lib/containers/storage + name: buildah-storage-dir + + - description: | + Storage for temp spaces required by buildah cache when building images. + mountPath: /var/tmp + name: buildah-temp-cache + + + - description: An optional workspace that allows providing a .docker/config.json file for Buildah to access the container registry. The file should be placed at the root of the Workspace with name config.json or .dockerconfigjson. + name: dockerconfig + optional: true + - description: An optional workspace that allows providing the entitlement keys for Buildah to access subscription. The mounted workspace contains entitlement.pem and entitlement-key.pem. + mountPath: /tmp/entitlement + name: rhel-entitlement + optional: true diff --git a/kustomize/overlays/tests/credential-encryption-key.env2 b/kustomize/overlays/tests/credential-encryption-key.env2 new file mode 100644 index 000000000..d426793bb --- /dev/null +++ b/kustomize/overlays/tests/credential-encryption-key.env2 @@ -0,0 +1,15 @@ +{ + "data": "ENC[AES256_GCM,data:vS2Mq9UvWw2fie8Ot9bi2p4lat4fF4+nIRAuB1OiynhbgjTguPaCQGcg3TTX+eHY5DtP65xwTbGkN6ymZOyiRatXp3zBUe0vm9Jk4VXb23IgedBGAK9CKjAf,iv:MJ5sFOfJf/iPj8DwtSJu3SRBd8YThpd7BRJmfWgE8NY=,tag:lgj/KzRa4ut6y4G8zWzyIg==,type:str]", + "sops": { + "lastmodified": "2026-02-25T10:27:54Z", + "mac": "ENC[AES256_GCM,data:ylpBZXqneUsWoiLpQkcIt3MR/fIqtP2nq2HBNQQfqEnFusUh9t6fKPcF8amIFMES5qhci21wWi04r6m4Y6BVpHWdsy4qnpgKQUU915VpxNPPUGL6FJbpFufAxD9W8c7U4X8YDzXA4q/GzEVLjApS4JoCckLPPNYYaKgZP7sBk0M=,iv:Qa7xr7IYV+BIMBDwDinKhrngWwoxJS5WJuXtJZUpnuk=,tag:ppY96hhacci3YS+Qq3DusA==,type:str]", + "pgp": [ + { + "created_at": "2026-02-25T10:27:54Z", + "enc": "-----BEGIN PGP MESSAGE-----\n\nhF4Dy77zzNMwU0sSAQdA7pg0i+MN8TiEACfaErvgVmx3TISeSdYbrm75nf/9H0sw\nZ920uz6tCV3A+zbuwVxmh/QuU9DyL4sjiDOeFl7PPeUp9qQOVXbQGHXREhFI4q0g\n0l4BQ4hMMszexwVTakLL5aLO5F4Slx3p9mhlINktZnf4AqUIFMW/6bcVgmYPnHQg\nunWOiJVBzBi41DFzljAwfd4MnUSFtfQtSGj56vy8j4YXBb0gbdNWB2f8VxGkf5da\n=fdFQ\n-----END PGP MESSAGE-----", + "fp": "8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7" + } + ], + "version": "3.11.0" + } +} diff --git a/kustomize/overlays/tests/exploit-iq-automation-token-enc.yaml b/kustomize/overlays/tests/exploit-iq-automation-token-enc.yaml new file mode 100644 index 000000000..d8df648f8 --- /dev/null +++ b/kustomize/overlays/tests/exploit-iq-automation-token-enc.yaml @@ -0,0 +1,24 @@ +apiVersion: ENC[AES256_GCM,data:Uns=,iv:t7ZWH0eiE63kyMW42wFsfsKN01OkC+brLLXaJUEClQs=,tag:pxTCqrwC1cUO/EMTfFrPzA==,type:str] +data: + gh-token: ENC[AES256_GCM,data:GgFKPuhztlsqsZ7PjXGH6V04uNu6ElKs5iC8uUW/UCL7vjmx5VR0suUaDA1DISb+Bc9W83bMnkW70H8CUwMX50OI536T/hDTokLAYTsFD+05HsUWKSxBqq7G25fv3mzPYSBAVKs0EL7K+f2Le6k2yeYltkOkfIRFtabFcg==,iv:cl5jl1oKbR2AtJYCoH4Je8SSyP4Jc+gPInIRfOflTSw=,tag:CrnZspbYfUjz0wgjzwfQhg==,type:str] +kind: ENC[AES256_GCM,data:j8rCiStt,iv:CaoFjqicLryq3MS+mvgVO5ffBbIX9vUQVQ5uy/NyNnM=,tag:r4LeUOBMkvXK3hd7faj/wA==,type:str] +metadata: + name: ENC[AES256_GCM,data:X7/gU5Es3+keADmoDJQWfF27KBKVjoCUcLRe,iv:zYPeX1H91bcUut1/wbVi6UdyRAwcT2QKRu//GS8KAY0=,tag:hPQtbwjqeIazh/hrnmzbjA==,type:str] +type: ENC[AES256_GCM,data:waF5cqrS,iv:yh5grziLkmXblL+zoo/DbsFI8GJdBICWq5xbGrjqrjM=,tag:W1odWNRYm9RpU4XBAXDvpA==,type:str] +sops: + lastmodified: "2026-01-27T15:12:24Z" + mac: ENC[AES256_GCM,data:AupZMMI0ycLlxF3/s2fLy97AC75/QF7itpaldF+I6Cuoj6FdmBd+2HwiDuz+505ZAm0/pP7Ez7p+zA7STyxY0vzGBu2XoOhKOcj0seJHrHZh60PXlV2BgQarBxVTtSg3BWFLrKYNRRHevHezfRKxFichlQZzj5Pc6TmheNs2c7U=,iv:TXYL4TE8GlAeuTjaw9GfdP+mmXtWpz++sHp0InHk04c=,tag:0I+c3wRyLh2vL+9N7K5i+w==,type:str] + pgp: + - created_at: "2026-01-27T15:12:24Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4Dy77zzNMwU0sSAQdA/biMrPzXXQ6FZhsAqOpLTXUzegYmekUNov4ZxfhQyWsw + GLEYcoYdGLjZo/BSUP3t6+8XJ/LY6ytRvMvsxWWJKGBspxIyE7JwTCpIdOWA4p0v + 0lwBlyu0o8Jc31ct4J1V+mPowF8L1znKEgqVBugA+l3N5JRizwecTdcb8k0OXHqf + +hXfcCYVK5FYJbdtGsGEEZmS6vGjdAViNiTyuS4NS+Lh4sEFXA0Z4CT7YILArA== + =5nCL + -----END PGP MESSAGE----- + fp: 8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7 + unencrypted_suffix: _unencrypted + version: 3.11.0 diff --git a/kustomize/overlays/tests/exploit-iq-client-build-ips-enc.yaml b/kustomize/overlays/tests/exploit-iq-client-build-ips-enc.yaml new file mode 100644 index 000000000..1e385c0b2 --- /dev/null +++ b/kustomize/overlays/tests/exploit-iq-client-build-ips-enc.yaml @@ -0,0 +1,24 @@ +apiVersion: ENC[AES256_GCM,data:V0g=,iv:aku+6VNcpHX/VwyWNuHMU9p1UJn9QgI9az5pl57Up0c=,tag:9cHqcamNq/n459qQRu6bKg==,type:str] +data: + .dockerconfigjson: ENC[AES256_GCM,data:PSc/c7Dw4D0LycEKR0DwFGucqdHms9QombznJc7BtJEfDBd1/mTFOobGw4LHkUtUc3I7eplOd+u41N0PvERRXbsf26iWQkupb578MdoTXXtzpzwKZcOt1pmROrIs/I8yIgCmOYL0feqhPwSP0aYESYwTCQXK9nMvkrCNPpdsyeBMaQ17K2G+Lg0zk2/Crh8xQwbbVuU+ZxVCQ2mlORGQVFTQzjCCvMll28szpvqSXS6fcxJ1Ctma3I9Dhzv4fDP0QVlmxM9IjCZ2gQLqqMulXzmIViqxUn5G7Nmm3GtS7DjdKTpIFr6hIowzi/ikywmOX9rvqQKpZsq1VSkdenR7Nkq2BJQERmjxrGRp460ikA9MxI076q5HdjY6fibKSFROom9fltTOdniDZjnzd7rp/WT0Q+rrI4C9hVOFZ291KEOysLfmovSXWtDOKlX1EsnrwZVll7QWjLbqElqmGluOAHKJNFVUeBMxveRPcFVT01+4ddtcOqKgvx8vVihavEDEYCHTUny+VqUhnAYbXyH27TqfcfH7xs7KOcc71xnpXtcyAkdPBXku7T1xzl9090O4ZuDEK0hADGIL3RJuO6kCVIJes8vEfAY1HAiCBO3ykcrizkV76E9FPvNZlX7vbsmWQ07NbyFHBv6lKsuPrLBf80eulUxLnBz5+jfaHi7QcNjY2ADaPcdxyKvYipWxnjmOE5UisIQ3mmH6TvMg2ZuyBfuLwgDeLRyReXjfutYz1XFUNc5dstNJ0BRQ086zfppvKn+EniP6mmpflmFRGp1jULg3SgZiWXQ7SvLWhnDGIE2JS0j+dLr2ZyGrwrXNRSb0ypLuB2fxS3Tn1ZjVGsToB0gWv574v8IE68uI1J63FnL73DpYGIBLrnpN3QsAtRXSQ8jgPU6OLn3F8CNGM0NY9EsaT5sR2KA7TOo/dWzHWPfgphs3P1JwvAiT4shvFB1weu0jAKRAE2+vWPyazK29IYyqFg18KP5b/ibU4v/cmPJo69uy7tBOhS8YGKZ68IGcq4QJO9JKgS6LRd66mqZd9YdJSqwsVSi0fh+CtesxvpHWn1bz0Q1RMSczw4/8/4SY2mDIDOf/Z1ViiO+CgH1z5gR7UNeUmWHXUlDRf3y1YRFu8BhRCw+VH4yu+pg/m4r8E4bZ8fWZTMZpm/Q9Vf5NwJoKpwgOXorImJmSb0/UUq2VkMjDTiv0W7DZ9Ks6hz1yVCsALQB0FgC73zUAFm+f5PfpB3yhloxFeUKPWW4vNfHGq1M5GIc6eW87f39/U44n7WKYz0cdqebejVUNO6bIn8uF6+KqGc8dMKOpFheZQdIciOFkjd9L73//+NiG3kPXiwtsJKGLd3QybB3yEVF/DvFI7P1Kn0SaKB4Z0+lSElAxj7FF6P/oJHA9nNIYF+FR9EISstJOoyo2nXqQO3iMPeqjVdIBo1NJHjn33uvu6hVcnA15pXJshWpUPlSWq2Vw2R6YJcxzYnSQSxZYN+I9YQ5FIAOElA/8HJv9/4vDd4fkK2+P++UzgBOiZtLyNeNc6fuJkBisqqlse2MPtAttrvpjFmSAWgBzN7MUtd6bIrsTCP1i+tdNsyzIkDQtgTQVvjGT8knc0OHsgzvNZuIXfzqNbVwok6X8Lsal1I91en/o6BSGsOSLAKKN1trTb6UKFTqzkpKbKTQpthuN1sWlZuQGyfuNKVbx2KGciUjB9RtBUtNrj71moZ+90gQ9/chvAl3bRpMEwKhdtkrgX5k2rQGLz3Rtll96tEmJqvsPaF4geydJMTS6WIdZXGaBbsDA7barMi78ntiHSgHswQIw7FdMh/E+kmyITAi/rePle3nfSXiqP281XzdSWkzVgQDuVMQ7iq4+mD+r2hVaxA/DpVfi71qryHLY1GG9R57ba7wi1LMOLxcJ0SGVVnf4Y8qV7HdBtatig64JMrMFsfwT68RSd0XjNa8o8tsPKxsWgWfLYor1R+X0DvDTjXfEai/TgExmt+d9HKT/fiHpGpu21RmmJ89Q2HxxKaBG6ItwoyQfYmV77Wxl34UkPeG3NWuEmyzZ+MS/skkbVDdpMEc0R8h9xk3Uot8yT+1KBH/cyXvpqa7Q7kfRYFaeuKUDpHMZfSa4hfriRZsTrgDGF16kHU7HLpCQN6gPe16j6GfVSFHQpjEDcx5XxDp06ZFjhl06xsNt18ytPdykOBux26f4KWyN/TTgjDU0lVqWZ6P2I9tgDyJf6lQmqQ==,iv:GiAFMGtzev6bnTtbKrhaUmJipZfYEriVDR0hIIcbuvk=,tag:p87FISG6OFI5n2suKECcKg==,type:str] +kind: ENC[AES256_GCM,data:oDXj1wmI,iv:zRQA0DPLBhe7skXIqE4vLQKlQgzsOW7PhmSQBTUhVcw=,tag:NeNhMXL3nrFDXVGiISQqvw==,type:str] +metadata: + name: ENC[AES256_GCM,data:YWKVdWVd7Vkby+mGaoqCCdFbZnk=,iv:xNy7/XNWt0Ng7A45WEZ+K8RQD6inqy9TEhVFqnWzxm0=,tag:X+ar+MFIhG4Nsz5gwopLJg==,type:str] +type: ENC[AES256_GCM,data:tC3kqUKpNt1xPF8COd35itVzvDwEXxmOvnc5iYOT,iv:4KHXCxzXmH2LSX3/WmvWjeol2kG9FtcB9K1vczpoS2k=,tag:EkcbbOGnUM5ICiwV9fyYog==,type:str] +sops: + lastmodified: "2026-01-27T14:58:25Z" + mac: ENC[AES256_GCM,data:4QX9acqwVT0yIblv0/On8BZVK3d47GcLccXhWWyrdRYW+4BgR8zc1v+k1YRxJEBlojlXNqvaFs9tEEdNWQ0QHBUsR8jMQ0T2t5EqHb9LPE4WOYuBVcTyeqDmyc01EKHDWdutYxo15kotCMG1FO5OgZ11Rcwdv3f1QVdgskN4Pks=,iv:kiq2siF75QwGOK84k013AD8ex5N/kPMc6FrUOp531v0=,tag:Qq2Y1COIOw46WyU/qUH+ZQ==,type:str] + pgp: + - created_at: "2026-01-27T14:58:25Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4Dy77zzNMwU0sSAQdAeDQZY4S70FaOHXTn3u7lIZfKvrpRyBtH5osQEepI+E4w + NnlXnnxgtOHBU8nHJZK2vmNge1hVUaGBcZ+mvsnanMEs0zpFkn2LD1hoQhVR+kYo + 0l4BIJTzMKoU0+NlCheUC8dD4lPhgVszP+Lis2ftddN2+q3rAwkcpBZC3ADkw4lp + GFuWjTCeNbfrWV5VGPj0rUfgXrqBS49df/aBBlkwCuEI+iwKFaw6UE21TrOT7p6L + =MreB + -----END PGP MESSAGE----- + fp: 8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7 + unencrypted_suffix: _unencrypted + version: 3.11.0 diff --git a/kustomize/overlays/tests/exploit-iq-ips.secret b/kustomize/overlays/tests/exploit-iq-ips.secret new file mode 100644 index 000000000..975a0461b --- /dev/null +++ b/kustomize/overlays/tests/exploit-iq-ips.secret @@ -0,0 +1,15 @@ +{ + "data": "ENC[AES256_GCM,data:7Mhg0suh1pr4gW2ZSVDfDVBxR9M+WeOS/ndzCa6HynWbnbVi6o/bXSBrUwukb9NLEPyXG+i0Ja265AmV2Ix//+0YiHxRMTNWuAWeik9C1FhPyMefg/QJ7/TqVA19011U1oaqZwttcfxtiC69lKbIG6vZnxuLtWhjoWfi1SJrqPZ+EsKSD/st2DoWkhvlGd+ea8RyboXt2knL2jy7smo1wRWSUl98SqDr6TLqNg==,iv:oup5Ep55EXokJe+jRlOBXIxGoP88ZqV6aHzQtDrAGok=,tag:UuwWcFrQ8OJf5weq0HNBrg==,type:str]", + "sops": { + "lastmodified": "2026-01-25T12:11:15Z", + "mac": "ENC[AES256_GCM,data:o+/YRK8wHs6hlEJqkwDtzV/3plYxOTRs7QGfqLGv0TaGZMIYFnbav1M4AFDYY7pMxEnRCyRJWr6G6L3mN8Uwdcr9FyMls1yQAXnu8a6iP7xLSvShvk4sXmdRKCV+ZoI6uWOGpT5um2ovpqce2GENStvk7PhFhS0R+Dc2IqBYbyA=,iv:rWe1RZ8QEIeOTrIVYTroLrqm9QaugtTVw41dtxJRk5Y=,tag:CRh9MgwYzbLV+fqkIjpcxA==,type:str]", + "pgp": [ + { + "created_at": "2026-01-25T12:11:15Z", + "enc": "-----BEGIN PGP MESSAGE-----\n\nhF4Dy77zzNMwU0sSAQdAenSUkHYnXpk59IsDKXVzwzXcmJYgwOC/mjNFPxrPUQkw\nIroNi7SaoYcdQ5bNd/IygS+LSJbqxWpMvPLgxw+Z/BUS0lWppfzAYgMeHGjH5Y+u\n0lwBGxusz5C9WM+oOHNOhrg8DZZU3iLfDgWpICqJ6OtRlcSlJlr2gXPFZngunkxz\nX5fFnLDgs2j6OV5CQEAkjC3j73t9RSE61ILuRLqZGMFjCm/xtL7KieKhstFxqw==\n=mhUv\n-----END PGP MESSAGE-----", + "fp": "8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7" + } + ], + "version": "3.11.0" + } +} diff --git a/kustomize/overlays/tests/google-sheets-secrets-enc.yaml b/kustomize/overlays/tests/google-sheets-secrets-enc.yaml new file mode 100644 index 000000000..9f9b7cd73 --- /dev/null +++ b/kustomize/overlays/tests/google-sheets-secrets-enc.yaml @@ -0,0 +1,26 @@ +apiVersion: ENC[AES256_GCM,data:FCE=,iv:mX1MhglqJCtmZ2+lAxeUdRweyKUjU0eEcxBBOyEfhQ0=,tag:wVimkqWtiSk2L6IcokRRHg==,type:str] +data: + credentials.json: ENC[AES256_GCM,data:eLGkTsE6lGTJOawS4Z4nNhbKwBLJVkSaFF1Tji8vpcH0M0A7LvRY53W6zH3TqSbvQ+AWssXjkuz7OIyHSg0WW60tkYixfToK/Rgy4TAbLTA++LcVqoe9tVspDtifEJLwTwHEiirX61SVgdK11Rm+035ICXmPlbV4VyQBfC1hHf0LDeX533raYq4QDyz/X/htT9eO/x3NyyYlPykcCT5Uyva1BV2Gi9iq8AJdpMBC/V+20DrPfzSsHgNlGUMOTmMK4qzs6nyOj3LeGc07ycKfGRK+hCCwTxoFyBJ25hgpNGB2FLRfJApqUjnKnh7VuoeilYEUdeWKVnzzsNsJc26CzeixxNXTaNPG2eo6LtNXndtMgXyEOa9zu0owJsrLl/zHHqqmtOkJvEKYkED6QhVMzfF5ciCnoyh3kwyVERVXhVMRGB19Gdn3dv9pdw/OPGOvvV3AGuzGR6SMvfKUn3lqoui7EWnOhsVd03RHXY8CXuBX7u9Y3aBEFj0DPaIBfH6CkG6oA6XhCt4kH57M/S43MbnlUC+napRqTJiJzRc49Qc3TpxG9VFSkMNOCnnm+8pxnbhGzaYA1izm1YHUTk1v6yjTaOCyTWQFEfum73TaafoF+QGypyHaiidRaa6VQ5XGkDxZ/yONWpt4i9NUp9+xoFersBYk9XBLWJdza4jVe2qOuLkM64OdNrnwKnxVqXFUNk9kyvXfMoCcweV9jCRN8Ky214MfnU5ALgVizPbW7TkZD/Tt9T4She4F17SgiR/+NTn8DArPVUuhQlCX4oR54CMFv/iQBYs1NMKKLZJ+aFyo62shAj7cV4grhtLNgezh+i5u9ISWS6ruU7GPxlOgxrdoDIPavCMi2GmcoCsxF2q/CtOghYyKh1fg02gnrkrJW8k3z9ZCmti5wlaPy8Td58xCLzsVxGSOwfexFdtR45EaPaHmlQotyIivfPKsFS24vqNdSjPYp6SygRv61G6aHg67EmzsT21hLb1PhtMLGpSdYYASBfdHq/6y2KsUQg2YZcDAURKEIc+1QWkRwsDBdr7RPrcmUFZVq7V0QsmFRiJqt1nDgn9GRfF02QXGLFoAehH64COjpERpymENoRKzqwVzL71VqoTdu9A1YXYhJdSzcWcSZ61pkLxgh6B6eM5PKLxqiQAIxgektIYf8yvSg3o+l3X36YHqb1pJXTgffe7ilehTPtkR8hbU4CpBR5+A6Se8/JBD74ym8Z0+A2UjUd/IsY6QOOza3Luko9UEDO2UMF4CgRRE1zdxW79i6HkNEEQaEB3uLfI72LuBI4bGdnAsPLzMk1/1ExtuvuzThJq3E6SXh3RSuFn2benLfvfn7GSVS4tJwdfkWzhTX13rqSByhey8kNiKYGMAj8I+rlr5th70VeZGbRD22OZ/99sK67xGA0M5GgodKvkFR3LQFYImD4rYt9k5DZ3PtI/r+dBQWZPecUuzHj3Mkn92G5wSnhJuEPUsfJJPOFyV+bBVjTh+FSqMUgmd2lzAn/Z3MqcfW2bRsNOFXHW/+vIf+xK6JsjTEipqZCLMl4cO0WL2gcL8KGetlfO91mgnpNcfZ8fkGOfRsEueA8CiQaJN4tGcKPW/TBJdqi7sCHHtwX8BlD/tcGX4OahaARR+jouKvYkKCtYsg8vq4up3BTYeHaATXxETUDzw1GYhnSzVdkZOq6ZsefvutDzR1PYgcOYHZ4mMOBghvwks6UNZNNQlFlfkD35ThBqCPzmPKuIFvUotjtxpGr6sfHBAqA6iPqe02iWs1+36ZyYzjioXyb3Mu2BVu6RnNYABzmJdAhiDxKXIjYLlIQJ3gocy5GpPkAn16qzuA3/RN9/FTVHKck60uqSsvrQ0TCG54hCN0O8lPZ2jGyH2ok4LMhyqHV1F8O1GGTDxoFZn/z00vXhh2yTv1km6ktlXdHKQB/T10YszwkczcseQw+O6BbGZEPzB4OiYX1kLthgqzkI8JIwnMZNdz1O+va2o1TAaFL9pCzGWkQqz1X6Nup0S1rJ0UsNPPAj8fZdjKrsvACbcd+oZLyUmexrXb9k8Jq0wPLKXZ51pX0JGB57BbDpBqeeA13QdFCUPNcr1nYSFyBKkkJu1zSeCluceeIHMHhIdie7HLCDL1PR909gwd0Y59c3QGfsuqHgblTbovXwx6s6ezvwlZpKXePj2Orknm/Kgmgou4ECtr/Yw1fmkCea1j5IkMCX9MJZ/TI1kVKpMdlwN+fkjADX8FJoe+EnNfjCkQxp1xFg189iWPm3VEIw989jJygHGwfu/IV6ZPvv9LCtTrV8dR8jPuliS1k0SUHB42iZB6J5XRKc3fbuHpcoLX715XNDW9zzlD+22bIHgCekOgvrlSeoXKHbbY0V9DML3ooF8mafFscMva/qExPnKTvNx3FIuTCRMgTMr57DlnwknrYR3eHM5BfWjsb7T/ZBEMTQf3jafmCoDC+V83qtNHJxGP2ObjpW878Ljtj1NF6Sqkne22aypmyvrY1kI2xeVl8UzGM9c5Wj7S3IxiROWGiubEfVgH0MZluir2jLLDUThNTTH2PlPjQ05VklTg9PobvJqna2xrUYhI6jvBwCq+SD6yTErNqLEozmZqkBkTkV7RAkY6a8S1FBkLXQDJVJUP6Thac50Ty3TDpJ044D1xwpFbrchosgVc2a4vkunlVx1tc+Uot4IYxAlCBzTQbeTq5wbgNEZSljVnAT5V1hgdJCt4wwBh+rUvLm01I5w/l6j3r9wNY2d1MJjvz2SGh4WcxkwOmsIoac9dky8imubt5Rc1kaJ2QDgeUpWvfyPaGn/vQRPtMTp+zF7aezf4Ci/FXBUUOQPZfUN70qH92Sg1cuETRCPNMGgBVlsIY+a7f75P+Av7pRheLZ8Z5EQ8Vt93pRp0o0a9Fyia4Go9V6FeTV+jgbltY03VhEFo1SEh0HU1cV1UKu++90okOjfKNTOH81zuESXvjI+V0fRIen6lbDaF3EToZ7XWL5MPobfOsuf++3HYAdoXPjI/khji0QJ3OlaTMkCrKMlOO2IXvm0rlsMvM4yLPuNeB5vQemv9hgaEwxH2RokKG2iaBEmMQec5m58b03kFhu9SnXJvKo71k3T1Y39iiTxcjzv7Ob/BqD463ndX0POhMJneM+OqyYA9mZ9ZEit+JtOquQ18wn8gRyP6oHRBkGkR880519rnzhCNhYZQyvYw6/3QUBy4FNDugtn7ANZXXkSfNPsqUsR91USMbtSAShczLcIyI0ybDiVVUbGD3zZ97pxPKq1cMeZ+iLKrgGXuDzPBgmtcluOrctWdxFoc4WPtem6mc/V2DtjNmTPGxVbA02wGHxYPLSTebm9xqfIk/QY2E4V3vQJybfgS/Ni1y9RErbI0/k0p3kdrnYXKfey1x2LMxACmIALkpMzOsms11tLMGpEONKHlUnKbVWR4GncBDraYCNGuIrQVls5O/1+eydu+2EAgbs2GG3/30zphqgQ+RA1LOwR4dFt+WswDyMiyYikl3OBUVQZ7gJNwkMEsM4Khfk90FcyEvtToqFMYKXHODEs6XcRyZFIJn/z6p2y5M2RniDc9/nM93mHJWtor/qK2eEt8ixjKDU9475HqPZefQFtYmtPq0J/Q3PL9N5jk7uodMOho2vSka86gGSDpYFWi0pw3wUfCgmAP/4zIiPOzM3Ut5YMOGWCvj4izOlwf8dpn3NJLEMhFukxxcMevzIF6uwWWPPcA6eLByWJWI1pM4TWtCY73yAuvPz1aH5wljV1ikozh3sZxeSx6kiMri0mg3Qg2g4nDMqMu8UozlCdn4uIvFgp92MBo1uMp8cnjZ82lgftI1PG2kpdeUjJRzNW7Kp9MUeG3O18IWE0KRDLU9mbd7RXLsTM534kI48NYYMJhiWu4ckr+mPg+mRkrp387serB/paD/oftcquKvvwRQKSqsIblxIrOqzx9c9smoZtEaN0Wjo1Rp/XFxAsXa9/UPGLswA4gheAnrWMldTBmacBHKz3O5FJk2KBgssqD07Gf+HALTLtdQxcCAdMjaSP/Anw17ooEZpxgbnegZi3wJXOKZujGHlZtpnMBeIip/DwzqPFPihF3/gtAvpWwvZTfPX88tShKQfg4jetcS9EhPAFcGleQqe86F6sN/WX9DVGX6vKr8TEDV2QXti3VFo+i6/lP+yFipRlxkmhsSObIW2TCDKcprBd2snmaa/xkRRFVrseaV4xuYdzVj9A+hlaJmq7gDxapCUyprIAD58p6NG1w6Ec8EbO4PJezQ==,iv:5/2vAV0mR91A76D1eknZllR6G2CcgS1CjFzH/vgC7yM=,tag:P4+B4QMpgh+v8G67QVSBUA==,type:str] + input_sheet_id: ENC[AES256_GCM,data:QGcHsxrPb4ocTIWd8CMvtetiwmeKJ2zaJ1pixjCzvB9i7IznNiTWSfm6rOmf1nd+1oCRsTFiSjlrM+73,iv:QpcPMsNz5xghxrjYw6kkdPJvL9zBLfVtyfE/Hyo2qMM=,tag:pz4UWY5aZYDLFCDjKcyqxw==,type:str] + output_sheet_id: ENC[AES256_GCM,data:XfLR39Saza3v6xX9NRxm1Lgg++0jTUVJU7T2DG/U8wpnfDJkHBlwzyQIypIWfapR2q2JMHh5CsfR6IY2,iv:MriqJEbO4qHyRuLqmYLCGghqcLXa79W0OueQRFzdQik=,tag:pIelynMOFR+qDWNQBZb7kw==,type:str] +kind: ENC[AES256_GCM,data:YfSIF+rj,iv:GSZfu8MjRgrLoZiDd3tYMC22XSBX8hN7rx4Qc+q1Sy8=,tag:pz5hprvCEnw2Js6o40s+VQ==,type:str] +metadata: + name: ENC[AES256_GCM,data:pFg7/Zxxt1Tc9WQJVivlI8hgN5TL,iv:i/nNCn/CMrNBuRXaR7OgvADNV4vSwk0SkSwMrvzW0Xg=,tag:LBmpT+Aw2vziK13JDVOcFw==,type:str] +type: ENC[AES256_GCM,data:tcJ0TrMy,iv:bFgWgWktWvbfbQO8svO90b/6izbSTHKm4e3KdalerbU=,tag:nb7GgK5m+IjokvJAqUcDeg==,type:str] +sops: + lastmodified: "2026-01-25T22:22:08Z" + mac: ENC[AES256_GCM,data:FcISvk+RreKCwSHfvvobTXNbNbnRuvh7dPmjgTSzsW9jJGSaT9pVI1KnZO7pSwZyx9e1SxFxltM+l5lKRwEx644g8sXn4vgm5iS7hydlCaUSSG1S2Vm7QTkrc+Xd4anWt3/V9NTBgC99MbADwKUxL7SKj4auzXi2rbiJgrN2To8=,iv:bHBRrT6lFX2d9eOvS3henEqMusk3X9RbqPtqzIE1sA0=,tag:SXS5CgV6HSr+Ma0jADU/8g==,type:str] + pgp: + - created_at: "2026-01-25T22:22:08Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4Dy77zzNMwU0sSAQdAtKMi6CN5tPD/ZRK0uiEZ1zZzS6XVXgaun1QQn3OufDcw + fVPefvN/Fw+6DuIzyRsBOCRT10BrD+2Cb08JE6GUMOLO7bihuAitxbwbzOivPPgK + 0l4Bj8S91shsRwhqFWDBWFHxiKJIuXLVBJd2AvijI3ErEL2hrxf3BAzaFQxtuxz7 + BPb4egF5zsUIjzkwW4vzUbqTiFzZPTh6uBOq1R5C1Ux3YFxDUWtKf4/0dOjQeiYB + =w0Fd + -----END PGP MESSAGE----- + fp: 8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7 + unencrypted_suffix: _unencrypted + version: 3.11.0 diff --git a/kustomize/overlays/tests/integration-tests-secrets-enc.yaml b/kustomize/overlays/tests/integration-tests-secrets-enc.yaml new file mode 100644 index 000000000..35e012136 --- /dev/null +++ b/kustomize/overlays/tests/integration-tests-secrets-enc.yaml @@ -0,0 +1,27 @@ +apiVersion: ENC[AES256_GCM,data:1iQ=,iv:HLBDch8hrZ1GpuB1+poHHWUNAlcKxHayVJhEvlbxq9M=,tag:wykCE87aZrCQbIdC3YO3QA==,type:str] +data: + GHSA_API_KEY: ENC[AES256_GCM,data:U28HKEJBc0nRNzS2ok6mmGU5uragLIBDkJ+8FzkNpsimf1YhXbT0/SbYgWs3TukQTBZ4nlpx1Gw=,iv:+EUrOoUJmQ6ldIsAuxcqIGd9OkfOTs11sfmj13wH9RE=,tag:rpCbWvXRi5G1x8XF5vXR3g==,type:str] + NVD_API_KEY: ENC[AES256_GCM,data:xbcl0ZI8ul0YPfI0ALb07Jx9lwXVyk+GpiE1r18lIMcC3mmFYH+v6b27RyUfa5zO,iv:RzZjdioWQmpWtQImXOM9vo6aPOXifWALgi9gmlsRS1U=,tag:aKUXAOsbfOvhhogYx6tAVw==,type:str] + NVIDIA_API_KEY: ENC[AES256_GCM,data:LV/REkQyjf9/r5pmF6isMLIrWNy7yPOILu1a3s5g2tgs6/GRuyDQjbWQEVeIS7Ia5xHLJYLsFFhv5Bp2GsCjDONejEG0iS8E/FHvvwIWm7XvwICh3/3kAO2v5zYhqYbO,iv:6ofERGT1m2SX44keQq6pGTQ2nZMsPL2GG1+Pqebp470=,tag:O/+3KL1tivQX5qoZVrq+8A==,type:str] + SERPAPI_API_KEY: ENC[AES256_GCM,data:qp8vpoTBsQUf16xrO0SwP6NC0YXYIat17sIf2Ceqg+0ywm+t456vAPwA3BzO4Q2ShoZacI325IKS73ikSbfUt38gM9kRry7S8vc7yWOapTAxDVMepNO65w==,iv:a802YttrC+iE6KIwGnRXS0Gz0TiJWHOdLJGD+RDtON4=,tag:MzyHmEPXb7FNqX6CB2kN/g==,type:str] +kind: ENC[AES256_GCM,data:GW1La2Ea,iv:Z+yqoV/+I6ZQ3EEddouc3NhYvUg/fZPSErB16foM51E=,tag:5QDiiu/FJeE9cFTmaTC1Cg==,type:str] +metadata: + name: ENC[AES256_GCM,data:F1vRnNiwgEv3Ym5uyw6HadY=,iv:mDA/W7a2YrlMVrE3rLH14FqdmZVrp9P+csIVB+wHff4=,tag:FY7zr/T+XB6Kl94Ir95/Kw==,type:str] +type: ENC[AES256_GCM,data:3czhKxVb,iv:2YRDLhXyiAalT7BJp006tjOvY/VKhb1u1wRGHpJFRHk=,tag:NHLatir8R+Zee/+7SnIkTg==,type:str] +sops: + lastmodified: "2026-01-25T22:25:21Z" + mac: ENC[AES256_GCM,data:YQEOFODZyLaVbnZbXToqv78KH3riikik97h3PSv1Ay0xhcjdBtBRv1JTIfVWCyAuMX7jA7WdUy9VL3xw7luK6TFbQRLI9W2vwovZ5ljA0qthTBIXuU98eLEsz/FNpg1GukNPhGKw54iTokGyEbRA5+Vo2zXygi5q3Ui/F5FDLyI=,iv:1ap3lpJvQuaFqD+l7jc7MHnVaCXCNBWRl0gue80anmg=,tag:yO6vcbIrM38ufAncapYcJQ==,type:str] + pgp: + - created_at: "2026-01-25T22:25:21Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4Dy77zzNMwU0sSAQdAmOC1w11IvoBbzsDvMXNQXp7YzVqNEGt6xdxpgHExxjgw + h7GhiURhkbRGTA1Bd9V9JetB1ibDMv3Z2TSI0A+BS1GoNoG8BeM0t+efur3hMGNx + 0l4B/gnyReoDLZA6aaZDOMBLZe/GDaJ+FTegb5+VCTzAAsS1RgYrvPftBbCbyXbx + PdZo1yS+IMxuCQf0c3Q69FKF8Q8qy930UFjRxxHgiYdOQijJEhSyBpwiYhZK27tM + =geg9 + -----END PGP MESSAGE----- + fp: 8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7 + unencrypted_suffix: _unencrypted + version: 3.11.0 diff --git a/kustomize/overlays/tests/kustomization.yaml b/kustomize/overlays/tests/kustomization.yaml new file mode 100644 index 000000000..74ea831cc --- /dev/null +++ b/kustomize/overlays/tests/kustomization.yaml @@ -0,0 +1,82 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../self-hosted-llama3.1-70b-4bit +- sc-llm-pvc.yaml +- buildah-task.yaml +- tekton-config.yaml +- secrets/google-sheets-secrets.yaml +- secrets/integration-tests-secrets.yaml +- secrets/registry-app-creds.yaml +- secrets/server-model-config.yaml +- secrets/exploit-iq-client-build-ips.yaml +- secrets/exploit-iq-automation-token.yaml + +secretGenerator: + - name: ecosystem-appeng-morpheus-quay + files: + - .dockerconfigjson=secrets/exploit-iq-ips.json + type: kubernetes.io/dockerconfigjson + + - name: exploit-iq-secret + behavior: replace + envs: + - secrets/secrets.env + + + + + - name: oauth-client-secret + behavior: replace + envs: + - secrets/oauth-secrets.env + + - name: mongodb-credentials + behavior: replace + envs: + - secrets/mongodb-credentials.env + - name: credential-encryption-key + behavior: replace + envs: + - secrets/credential-encryption-key.env + +commonAnnotations: + deployment-variant: tests + +patches: +- path: nginx-patch.yaml + +# Add agent instance to the tests overlay, with fine grained permissions. +#- patch: |- +# apiVersion: apps/v1 +# kind: Deployment +# metadata: +# name: exploit-iq +# $patch: delete +#- patch: |- +# apiVersion: v1 +# kind: Service +# metadata: +# name: exploit-iq +# $patch: delete +#- patch: |- +# apiVersion: v1 +# kind: Service +# metadata: +# name: exploit-iq-phoenix-tracing +# $patch: delete +#- patch: |- +# apiVersion: route.openshift.io/v1 +# kind: Route +# metadata: +# name: exploit-iq-phoenix-tracing +# $patch: delete +- target: + version: v1 + kind: PersistentVolumeClaim + name: exploit-iq-data # Use the original name to match + patch: |- + - op: replace + path: /metadata/name + value: unit-test-shared-cache diff --git a/kustomize/overlays/tests/mongodb-credentials.env2 b/kustomize/overlays/tests/mongodb-credentials.env2 new file mode 100644 index 000000000..780f26360 --- /dev/null +++ b/kustomize/overlays/tests/mongodb-credentials.env2 @@ -0,0 +1,15 @@ +{ + "data": "ENC[AES256_GCM,data:cTEW4bqf52nVEz7nWKXJAx/5NNoG6PH+sPu6BYuhj3M/4z4GpXaqSPvWL+V2LQwQCtmP1D2/BMtxbwgak1x699wYMfPdptx1I9iFnOM2P+CrYZFbOllYrEUHA6g7Cv4RItSadLPsYpNizMDxTP8udaWbYFq/2r6rnlK/V81nZaJZdxeyxC7T1CVqnxCGS3anI1ZVPDGY4F7t,iv:nkEQJuCDVI3wKzDsjh6tEHfob9kXJEegl9MhaaDClgE=,tag:kRirr2RpmIHJHQVlmbGbSA==,type:str]", + "sops": { + "lastmodified": "2026-01-25T11:54:45Z", + "mac": "ENC[AES256_GCM,data:hICbk3dfXtva1J8jqG7uGLis+pJuwGve7tdQSeu7x3M/AjXukgZqxQoOrzE7H7WEh6R4XM8z7OfFqmIegjDtJ3RGy3VIbo2uXpHrhkAGUI9204ehhe0GG3erZKWAYxdryFA27UOddYhMEqyeezrdmXEzbMhy3eOXTQlyENb9gH8=,iv:rMqa6c70Zim0SB4oba55MYcjS332/znRVU2YX9UBhsU=,tag:Gt7FgLOjKvqPc5v0zIZYBw==,type:str]", + "pgp": [ + { + "created_at": "2026-01-25T11:54:45Z", + "enc": "-----BEGIN PGP MESSAGE-----\n\nhF4Dy77zzNMwU0sSAQdACeUMbw2HeFxqxL2d3+bLpJ5O8nnaixniZfOyLFNbDxUw\nudBo0dYJ5p9eSAr7/8xQBVeloOvLfO5DXBYlwZSkM4s1MPnTYM3vL6Je3qImRKzL\n0l4BPooAU/OY5y8idxBsi5gOKw/utLJ7150AhkLOCfvBVB5WP2zLLkWMx4rYMGTM\n8/hKqGkHi+8D2XfEByknn+95q92XnkGNcxZacCARXxFeDy0m+ZEeJa/KTO+5XU96\n=srjn\n-----END PGP MESSAGE-----", + "fp": "8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7" + } + ], + "version": "3.11.0" + } +} diff --git a/kustomize/overlays/tests/nginx-patch.yaml b/kustomize/overlays/tests/nginx-patch.yaml new file mode 100644 index 000000000..c4aae48ee --- /dev/null +++ b/kustomize/overlays/tests/nginx-patch.yaml @@ -0,0 +1,12 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-cache +spec: + template: + spec: + initContainers: + - name: render-nginx-config + env: + - name: NGINX_UPSTREAM_NIM_EMBED + value: http://localhost:8000 diff --git a/kustomize/overlays/tests/oauth-secrets.env2 b/kustomize/overlays/tests/oauth-secrets.env2 new file mode 100644 index 000000000..ae7f97eb3 --- /dev/null +++ b/kustomize/overlays/tests/oauth-secrets.env2 @@ -0,0 +1,15 @@ +{ + "data": "ENC[AES256_GCM,data:JM3q6XFUVQ8XvsAWyaHAXUcxGPgB3ixEwIL6NGXTLfznS8v7FRRouw1iG7zDn+IFCk2kYhHW+wJ2fDBj23Fdh5dMPIDzMvxn2yyLLqyvYsnzM9JNVnIM1XALph/U0J0GYmzMLIJgqKnk7UexkypD8VM=,iv:6YKdj5UDd2TdPNwG4u5U1CZ3Mbi0MV82kgvNc+R4MUU=,tag:Rm3WSsiGmnmIn1NrQdZlpg==,type:str]", + "sops": { + "lastmodified": "2026-01-25T11:54:51Z", + "mac": "ENC[AES256_GCM,data:UiHjGVbr6MnTwXjpfQAba15b2dW0EeRmZ0LCqvwMfMoJlRQOgjmGDldeXBRO+jyYKvMsxfRLsGjIIDAnWE4+68NCHhH4IbDPH3gvwLijzNrUWX5wwFtQP7uXhtB5gQjt0G5e1Mwtz2oPPPRQ1BTVycHrJN8E81UANBpTN93kQFU=,iv:tnwkRwG9ChLQFudPafIGotkyjUXTedcRhT4QXiI0t80=,tag:ZEGjTCCkmRTKTboqE/GK6w==,type:str]", + "pgp": [ + { + "created_at": "2026-01-25T11:54:51Z", + "enc": "-----BEGIN PGP MESSAGE-----\n\nhF4Dy77zzNMwU0sSAQdABXzNXw0NPdMDlcCmVeio3d9jVFxFAr/IQkfrGkzimFMw\nt8tPnMaZfSe0ZNTtDMUhtXtYqu8Mqk+IGNANQyQXw9/pK6tL3sdctQpwES+h5SOK\n0l4B7x69zvOIlmK3yB8EoPE1Vn/0qoRjjyOk3/CWoeDGjL4hwwmOLTtAM0un/VCb\nhiEX4nEgWIaz4DWyQLZkqhJbPlRLTOvwXZI95UY1f5nQRt/zjbeKtRbXmiH+qlXy\n=X3t6\n-----END PGP MESSAGE-----", + "fp": "8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7" + } + ], + "version": "3.11.0" + } +} diff --git a/kustomize/overlays/tests/registry-app-creds-enc.yaml b/kustomize/overlays/tests/registry-app-creds-enc.yaml new file mode 100644 index 000000000..57af3b132 --- /dev/null +++ b/kustomize/overlays/tests/registry-app-creds-enc.yaml @@ -0,0 +1,25 @@ +apiVersion: ENC[AES256_GCM,data:IZw=,iv:xXXhE071qT3QN3TglSdLUY9cFsjaVyJb194zVl1Bso8=,tag:lgtA2jrDSfp5lMw6M9dMKg==,type:str] +data: + REGISTRY_REDHAT_PASSWORD: ENC[AES256_GCM,data:GquthNXj1O8624wEORBAvwpCGkG8Xv/BLAXuzTQX7RKwo3e9xdsxMbZ6QajCrwGeApCF+aQaKSkBXn4Weg7xJnKmrwN2X1fbdNeQFF9cvLaIY18VEJDtl5twsCYZlae62oHw4j1UNV124dQA6DPCtHND8GHw3rORYgMyGTnQxsCikbjcu94KVcFuiyXKPx21JUs0DT95YvzUtgkpPoKQJiexghfBH4NnunytiR/zamAfHLCgRdAcGC0qsbLEKDN3x+v4AF342oFnwT6OGfbcBaJrL27Kq6S+SO6ZlDAXvNawsiN60qZ3Ekj0DiUHSDKu9nuHOgAyyYQ8wfoZdwe3cpM8UHL0zLqQ3xvtOa1wxc5U5hiDaoC6kLzmvlGZEbBYbe4tHmaHyP0PUi4/qLou31x0xLrQbvnAsXbgn/2YEDhddSnW2B2uFpAxps7J1S4SIFrDIocaT7hdxu1k4vKDrVK3iIxNdzsbxJthUnbffsydXCHYAyorGpMicCwqY2WTHDlFu7whEoGzxx+z97c8eZcxgML3DHZ1tRexh3e/iuic8w8X8bwSDmYb3/i649ZnDSYPp8TbLbNPorneWMc4mch5ThOMyL4n434siHo8zsr64mSud+uWx2CBqcuVpR2eNGZYXJL4cimdLrugb58LAUBisPzLKnUtFxiiGsz2DfP8KgUEIlYaTq2a6Ns0LDIpxxvygMqcjLfEXblDV8zptI79zaOt75ghfUNP5CpkeOZmfkkp7KSPuyx1TLRXdW94pDiSE0rZp0VgzTkHFA/AF5S1QhINWuuAggWpB5zGqwM+uQb8+q+Ixwu/pCNCZd4xA+5hgyDhX3NZ+c+awD81WlwOLilneoT1O3VhCcBobxLdL3C//gtNOyxPFy68rzYgZD6MjHczw4aWYT3kGcr7MeVKKhqJPSRrESSythHX6AjVvg2ePKqTYScJo7qILgDMSKq3NhRu4vj3fSqQ13530mwAqrbV58N8t5XAkeciMI9GKFaxDiSSEUwbngOWd7aAgyePqFmLubfmM4jqtkKLJwcOzcE8F86cUbrKwAntm6x83+Jpt2qDZUCEOWbo/73WJAVv+l5Z91Qj7eWff70BOvg6jLNJoL0+r80x932VJPOwgrCaESBbFSs+olwfgyxkzQxrJ6tBtwX87fgzKhN0rwBSGukZsAZxzM2u9WS8trLIBOsEXFwofVP1nAcwRsexcni40TEw2DxjPrWW2aU8HOx+fVHy9xlglgNezu/ppcelDEdOcfwX0E1AflV7acIJSyU/k/R66LtUGUi5IUAEsDIWKgIcnAd3vIPTNxtjyJonStuLQ1JlqAMzRVyq7qch8uO5PS5Erxc=,iv:/zxAcdPQbS6r4iBGwUUlxShsfhuNjcCfo349uuvlhR8=,tag:y7+qHLSL0lSUcXgnH2tD8g==,type:str] + REGISTRY_REDHAT_USERNAME: ENC[AES256_GCM,data:+BCMkat0CX6ZoZbXcFVVEoBXFiPfyhxE,iv:KHaat2fMj/cDEPjbBC6enL9kCiCgWyjFw3ptCW2AuDk=,tag:ufvyd0TvLdKvCcaxz3ir/Q==,type:str] +kind: ENC[AES256_GCM,data:6E22NgYo,iv:Wi/gwNNrx0dL2/aacRKqJgKiSf9UF93Tixgc7E2pHIc=,tag:ZUbcU6fbNIgNXOQvGCeifw==,type:str] +metadata: + name: ENC[AES256_GCM,data:Myf/HsxEcMl/n0W/5vAtqA==,iv:qebrYwM4DLfYvUR/Sxin6GIDha8w4px/ybK8CGsnGAA=,tag:k7vi+9O9TBYHt7R8l7AcaA==,type:str] +type: ENC[AES256_GCM,data:X2uOhC+V,iv:dhqBwROQkhzoJM0PmjdmeOcNGj72k2Q3g5y0uPdJthI=,tag:irVym2z88svdyvJD5B1hfA==,type:str] +sops: + lastmodified: "2026-01-27T15:04:49Z" + mac: ENC[AES256_GCM,data:98VTQKDDdusGPrUqQE0l+i/pCcD8AQG9iWjYJofYAXkebsx3ABXlHuyuDS+fg7fGsLTSR2CoxyLyqjnanL8PpdXS1AFCSFUCpD9VoJ1WEadCJ4c9lgKJpb84f2TZTggsE6bH4HpHZuEDZR1/U4gbX0bE/4MXpMpkWxWc0+tOKDw=,iv:eyFLHWaY5vHRNyjp5I8tmQQyZDrc4wvDKowreOBvDJs=,tag:TfGnJOewwjbE/K1Uayrmog==,type:str] + pgp: + - created_at: "2026-01-27T15:04:49Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4Dy77zzNMwU0sSAQdAtMvASO+L3ZWyF7hGBPz2KsVM2iH/GASnzUZthLd0tR0w + aAtePRC34cc6hzeMsXGyPkU93Aq1kimzGUuzfIFaMHBEBwsJetzR78p0bedNTBhv + 0l4B0MFdxa2LKX57vMhBnjSPqnndz9c4EzT+4faM3sb16Ly3of2kYdtrsf8LdyIs + NHhoxWAiuyqlKZ3vHEL950R9NS7OjhxOYqVTPVXOaOSpC7SL+rO2vIH1GdJk51iE + =lqdO + -----END PGP MESSAGE----- + fp: 8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7 + unencrypted_suffix: _unencrypted + version: 3.11.0 diff --git a/kustomize/overlays/tests/sc-llm-pvc.yaml b/kustomize/overlays/tests/sc-llm-pvc.yaml new file mode 100644 index 000000000..fe5caa307 --- /dev/null +++ b/kustomize/overlays/tests/sc-llm-pvc.yaml @@ -0,0 +1,13 @@ +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: gp3-csi-throughput-2000 +parameters: + encrypted: "true" + throughput: "2000" + iops: "64000" + type: gp3 +provisioner: ebs.csi.aws.com +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer diff --git a/kustomize/overlays/tests/secrets.env2 b/kustomize/overlays/tests/secrets.env2 new file mode 100644 index 000000000..df5a50484 --- /dev/null +++ b/kustomize/overlays/tests/secrets.env2 @@ -0,0 +1,15 @@ +{ + "data": "ENC[AES256_GCM,data:qJ79PMbpPyOLezRJhCB2mRe5C+LVDHvDnv8RwbYYX2H4c2P8Ugyhcs+3uAyARQnSdzKOu2Vf57lo3eB6pyU6FZwP7rn2nNbcviPQdQZA4Z4e3S/ZPodhqzkwCU/8KlAYJUDWRt7rbtz6yRSjF+9uwu1dJXIGoFrpCo7KE8pZgaAAHvADnBT7RT6MAyQkQZQpqpGyBiF1vZ/MPnY7w1ke5/IpzY30kIepcwZ0fCdgViweYthkKu/CrBwl7h/S6FxPp/+EpjEMQimnvSaMybjoKYovQ7aNK+sE5CFHxv09Wsy2oVx7C0DDkQGJJTmDXgbNwUmwNq9OK8vteh91YvH2r3yPJT5Q2UoRSvVpOnqyQGenrYme712O5ZL1MW9qLpukUtEGQjcK7gaOANOqKFAly0yDtBg7mvHmvtqJl7mqGjqE+m931h9bNQgIdP/kxt10RxfwRatMuQUfhmpqEAQCDsEW8WUCOjE3whBCYsrY429dRRmDS4sffOnN8SQkrqNh6/+7n7WQFw71Yjy2Nak7WRdWNsPuSUAGvq2kOVjQzWUbH6LSzcuDmV5chCunDQYXvbSO4K448Zq7zU8tnL/Kq0BC51iLztUBAQgZ2mSRFimx2IIJcZs3iKHN9Rad6w03oD1vyCfUMMLWCpqN+v/o9Gdtuh54lV75ocVk/uC6WBtfkRd31KDi9rERB1lO+zufillhOcsb6ihI1xg0HLbX4N0ds5Kl142BuJ57QuTC48M3rJRhAVCHNerHlPZFIaHNCJ6XeUEPbr+Bq8GTdKLIZTT3HWzPQRid5nrvKAgOH1k+N87QM95ofmnrd/RshwtyjcvCVLY0Ua2/93JxqlhaTMR4J9u8WoN3zNQ5oPGnU1/Gf7z7HzWicNne36XBv44Apby4V0IBrnpTi7PNzqKDm6vOUyGKJvbHXlHuEBEAMcdApdGkGCMwf5vHfdk1naI/7rXWcvJtlJgAdDACDtKhOdWX7o6FvsS2qyKTWeeDV/KuUpt2IJ2v0SoYCbClSydJseKy044Yr+Q/OlNTLfoVYP6Y+qbtJtjxUqJq6aq2kg6lP7bnEpLqyo5Od6GO2pI/E/N1OifZRNvaIwildhd+BSv4cM55Syu+S4aGbyb9o3+NeVhtC6uYIJZxIxFJfxbnAG89I1Yb1rmoD5YrXOAIv/jD95B0mU/JntsblJYlKyJ8TyiOL3w5ftoWTiRKefzwnvZcoUcxQ/iO9smcWaUABFC6nBFZJblYQ1E7d/b9PAsD+KwOJBJnhMPl/3qYip3SOL8NSX7+VtfRwMgd/MQInjmnhQx96xV6UWHe1EBX+Bx/75ZCye6f+EY/4tABMHyCgfZw2rofjsANCnZqp4MxUFKBUw/gm0JVAwMPR3u+uAWWnkiekjvGeqbMGW0Y+cteKZA/b/kaw3fLASSbzAMju0Od9rEV8lJd3T4j6PJ+uXuxieUU+mD1GRn5l5ZngiL6ntSQF6UD8bZ4Gss3PgvHEmqdaY1ta5k3t1OsJF/edgwzPW2Vlzc7wSHWpkGMIUTqsMiNIIM57otjt8ox2VfoJakt0Fj1AJ6MAtUJYLgGlBzsvoQ5h+yXT1dqNOUzqra1Zl5XsdWx+gnaFN+PUG+v0pmkXl8NJInYDcpFzacEMzVT+F1nCpwJ57bLAbmJVeqQ,iv:MNFqnAFl1Ugx9oebDR66ckvZx8jksY+HeDsNFehFrk4=,tag:f6N8qxAy07EJHHwu2N5K3g==,type:str]", + "sops": { + "lastmodified": "2026-01-25T12:03:28Z", + "mac": "ENC[AES256_GCM,data:S8a8LiimwRsM2oqj6kMnG1umAXxb42GcdGLpDtcXLcbA3EewgRVCl+UIkDvFkagK8WUl2jr715wNgS/OlIWWdTBc/jAfU0iV3AHOCCz/Z1t16wQ96Pf3sEc0++l4EFlPfBPmhhCdXoAVh8TrPSpQRRKEXNY9JKcps2nZ5UDJ5tM=,iv:djGFNCf5O36+iyb+ENVdnThB749rc7PvpA8JAFJFKCg=,tag:fl7sRerdVAAXBkIATUl7zw==,type:str]", + "pgp": [ + { + "created_at": "2026-01-25T12:03:28Z", + "enc": "-----BEGIN PGP MESSAGE-----\n\nhF4Dy77zzNMwU0sSAQdAZGmK8TFHDd7hi+u6iK85h+tsLOTnv+cd5N5R2hmhizgw\nYGsqyjGgzxKosWdsGQw9n6RQwLookPSjLtO74CInBmSSpbL+k4NhAKs+G2vCojdN\n0lwB7TegWbha7ElzY6+MjOv+1p5EBhHFIWut+6+iV7HzrbralcDvo1Lo7b7e/4sv\n3jEf4HcelocgSLURYvL/5Cl48AVXrhkxdha+6xw4py4YRFIarIMU8ZRnL5gAqw==\n=caPs\n-----END PGP MESSAGE-----", + "fp": "8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7" + } + ], + "version": "3.11.0" + } +} diff --git a/kustomize/overlays/tests/server-model-config-enc.yaml b/kustomize/overlays/tests/server-model-config-enc.yaml new file mode 100644 index 000000000..55777b61b --- /dev/null +++ b/kustomize/overlays/tests/server-model-config-enc.yaml @@ -0,0 +1,33 @@ +apiVersion: ENC[AES256_GCM,data:UPY=,iv:qbLj3gBhjSfcnwa0ataze8hm5lctPL+5xAiqWe7zIx8=,tag:mkhVfM8yQAav1BLWlaSShQ==,type:str] +data: + CHECKLIST_MODEL_NAME: ENC[AES256_GCM,data:K1KWmLWRU3XK7Yy7+F56LCMoxGDZz8WgKJF0M8vXe5z+ScnRpTOJvbpe2fJy0x3g7zXr,iv:nDaOKFCkmm2uoD4+xZRFIWpy8nfcijEuONUvtleq+KI=,tag:rlUb3+DqRwpc+wvHQTrEfQ==,type:str] + CODE_VDB_RETRIEVER_MODEL_NAME: ENC[AES256_GCM,data:cnt51hBxh7tIId6GJYBwcyaoX5J5WUaPV0o999Sb9Z6mGIrfNnC2Vf/5bxXxs4l9I3Bb,iv:PJ4AKPDF2xXGNHneBZkgaRVZiiSet1o4QshSCB1comM=,tag:0KKYFR54GpWz9IGbsKRRxA==,type:str] + CVE_AGENT_EXECUTOR_MODEL_NAME: ENC[AES256_GCM,data:qxaF7Pd5OrCGzRhKleVbbwp3IawXBPRSxNMmeq9yOh4c8PfE2qcg2mB8Kh9kMQJCUjx9,iv:zEQZkdsu7+JBP/r1VO+nCsEZkloAsK7N80c4N+E4A60=,tag:T7dRB9NUnsNrEiRzYbgvow==,type:str] + DOC_VDB_RETRIEVER_MODEL_NAME: ENC[AES256_GCM,data:4NJlINBhf27A+nVu8Fx9ugIjnpDX4/V4kOyk7lWIpomi6UJkiaIDSo5raW2NpZfeOapC,iv:3N5YEc2+Gc/fwR+ecHm/XX/xwHu3RMMbziNibVLLXtA=,tag:SwYHQQg4fx/l/QEp3YJ2lA==,type:str] + GENERATE_CVSS_MODEL_NAME: ENC[AES256_GCM,data:Umwds/0JtexEGjeIliWL9RVbN0K2+gT1FimqFt+tPrEfe1D3wGaPYEAkQAacI/8XCzao,iv:HzvcW4+fvkzL/+Gid37X0tbrq9dP/QKvKwZ2mjOLsyA=,tag:IlTJed0cVSF2G7FMShHn9g==,type:str] + JUSTIFY_MODEL_NAME: ENC[AES256_GCM,data:CIhBKCjWHZbvbYslDyqP6NAEbuouDHxHPx9BAer52CnUUzaq7Lg4ko7F4YYv5o7TPl64,iv:y893vaZq0CTEZFUSFKbb+TEh4xKgGqarVGYosMG1/qw=,tag:IMMugbUBzLUukcw6KSqe0w==,type:str] + NVIDIA_API_BASE: ENC[AES256_GCM,data:G35BJ2UOsB3dLjndxSBNybQbYyeUjIDVU/EcJJLbSFqrClWXOiFiy/MhQ+PJRzmoHdntv9gCWHfUPKhRfBgsyB64wD8DrGDohhJzKQ==,iv:KUDMIRSpb/LQoXBMKTU/Kd59r/ukjdJ5zh6Kh4GLfKQ=,tag:gnWiEqpWHICAmI+PgzSFDA==,type:str] + NIM_EMBED_BASE_URL: ENC[AES256_GCM,data:8ryNEDwuzcwJNXFaUrXCIrUk/oBvDhFH/9yLtn5BUNocbAcJyeQUv5I+YwUXOVQk0fW+2JykfvBn+qQ=,iv:w5e9UNdh3wBbzd4Qc2v6DogAA627+5Fmw3fVsfjzsUs=,tag:D7fa7a1ZSwXSOXPQiE0zqA==,type:str] + PYTHON_VERSION: ENC[AES256_GCM,data:UiXJIQ==,iv:ToEOGsQBAYLE5BDd3zQPoqnH6do+WAggEFjzYMl1qB0=,tag:24jMH7zQoSFImxdy5Ocfnw==,type:str] + SUMMARIZE_MODEL_NAME: ENC[AES256_GCM,data:Nj12r4hMkvCWTTaEYg/EGBNnCulWf3zDTPDmSk3Eqn6ZvrTljNUT1bDDS5t69Y2IfdAK,iv:jIa5PFqd1O/HQ5vN15gOVduEpxs4+b1f4Z7W7cgRYl4=,tag:jycJcuUXjBt2Z2zcOWVi1w==,type:str] +immutable: ENC[AES256_GCM,data:orL33S8=,iv:/VLfTG9b1F0ftvqZcUxGWTH2cfn6dm39+1GUL6AGkJc=,tag:15FjfRmbLTFxxzogbjQL+g==,type:bool] +kind: ENC[AES256_GCM,data:+7INBDEI2vJK,iv:SkJaalGfQKME3FoazUvgnA8FQkjHUxPdOxdMHlhfiYE=,tag:2fn2v9XgCny1WUcNl++mpg==,type:str] +metadata: + name: ENC[AES256_GCM,data:qPvsckwg4xUthetbcj9+tyqkVA==,iv:8Mdy+qqHwY71YUeyUTKORupwiBTgxk2alimz7vJk4aY=,tag:Y/nOmTT1OPrcqKFxNlKk+g==,type:str] +sops: + lastmodified: "2026-05-12T06:36:55Z" + mac: ENC[AES256_GCM,data:Usc6qyfTA99yewHcWarGo1B2KuKMH0nD/ybcTHgjdkHRJ7ic6Me5RmFAdqu3VJ5hpnSN3fqZiZNaRDW38lnpoovhYn9pWc93CLbcRuvh9hIMe5G+cZMcrLfhqXkt/zRag/EV7MTKiIxyNU8Lxv1YbbB4xh3U/WnhmrHA3ZhKXAo=,iv:aYY7t0Zss2TSwuIrzyukPoYvKZva6qZn4JJvSyvOols=,tag:xkaiBNutE6+115Zsrlyrrw==,type:str] + pgp: + - created_at: "2026-05-12T06:36:55Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4Dy77zzNMwU0sSAQdA3lqmunI4yr3ONiEDe0MCZBYScWoagdmILOjylvclZQIw + Ki9ZcnxRh5s2K90HDGpwMX7/PQoDH0jTfRMSH7YmW+pADy6yJxVWmx3tVQFgsKuG + 0lwB9iyP1HoEhAEW3C+6gjZ0QKUQm0gsEWLu7LvP+YHBfGXK9NeiK22eJ/iu39xb + 0qTLHzdR9vkQObEgBkh2l/GBSO9w1ceDKYgL5DxD1R0rHncbk6cNkqvA+QJEOQ== + =b+dX + -----END PGP MESSAGE----- + fp: 8DEE2D0E1357B78C782691234A2D3B6C7E35AEF7 + unencrypted_suffix: _unencrypted + version: 3.11.0 diff --git a/kustomize/overlays/tests/tekton-config.yaml b/kustomize/overlays/tests/tekton-config.yaml new file mode 100644 index 000000000..8183bbcf6 --- /dev/null +++ b/kustomize/overlays/tests/tekton-config.yaml @@ -0,0 +1,139 @@ +apiVersion: operator.tekton.dev/v1alpha1 +kind: TektonConfig +metadata: + finalizers: + - tektonconfigs.operator.tekton.dev + labels: + openshift-pipelines.tekton.dev/sa-created: "true" + operator.tekton.dev/release-version: 1.21.0 + name: config +spec: + addon: + params: + - name: communityResolverTasks + value: "true" + - name: pipelineTemplates + value: "true" + - name: resolverTasks + value: "true" + - name: resolverStepActions + value: "true" + chain: + artifacts.oci.format: simplesigning + artifacts.oci.storage: oci + artifacts.pipelinerun.format: in-toto + artifacts.pipelinerun.storage: oci + artifacts.taskrun.format: in-toto + artifacts.taskrun.storage: oci + disabled: false + options: {} + performance: + disable-ha: false + config: {} + dashboard: + options: {} + readonly: false + hub: + options: {} + pipeline: + await-sidecar-readiness: true + coschedule: pipelineruns + default-service-account: pipeline + disable-affinity-assistant: true + disable-creds-init: false + enable-api-fields: beta + enable-bundles-resolver: true + enable-cel-in-whenexpression: false + enable-cluster-resolver: true + enable-custom-tasks: true + enable-git-resolver: true + enable-hub-resolver: true + enable-param-enum: false + enable-provenance-in-status: true + enable-step-actions: true + enforce-nonfalsifiability: none + keep-pod-on-cancel: false + max-result-size: 4096 + metrics.count.enable-reason: false + metrics.pipelinerun.duration-type: histogram + metrics.pipelinerun.level: pipeline + metrics.taskrun.duration-type: histogram + metrics.taskrun.level: task + options: {} + params: + - name: enableMetrics + value: "true" + performance: + disable-ha: false + require-git-ssh-secret-known-hosts: false + results-from: termination-message + running-in-environment-with-injected-sidecars: true + send-cloudevents-for-runs: false + set-security-context: false + trusted-resources-verification-no-match-policy: ignore + platforms: + openshift: + pipelinesAsCode: + enable: true + options: {} + settings: + application-name: Pipelines as Code CI + auto-configure-new-github-repo: "false" + auto-configure-repo-namespace-template: "" + auto-configure-repo-repository-template: "" + bitbucket-cloud-additional-source-ip: "" + bitbucket-cloud-check-source-ip: "true" + custom-console-name: "" + custom-console-url: "" + custom-console-url-namespace: "" + custom-console-url-pr-details: "" + custom-console-url-pr-tasklog: "" + default-max-keep-runs: "0" + enable-cancel-in-progress-on-pull-requests: "false" + enable-cancel-in-progress-on-push: "false" + error-detection-from-container-logs: "true" + error-detection-max-number-of-lines: "50" + error-detection-simple-regexp: ^(?P[^:]*):(?P[0-9]+):(?P[0-9]+)?([ + ]*)?(?P.*) + error-log-snippet: "true" + error-log-snippet-number-of-lines: "3" + hub-catalog-type: artifacthub + hub-url: https://artifacthub.io/api/v1 + max-keep-run-upper-limit: "0" + remember-ok-to-test: "false" + remote-tasks: "true" + require-ok-to-test-sha: "false" + secret-auto-create: "true" + secret-github-app-scope-extra-repos: "" + secret-github-app-token-scoped: "true" + skip-push-event-for-pr-commits: "true" + tekton-dashboard-url: "" + scc: + default: pipelines-scc + profile: all + pruner: + disabled: false + keep: 100 + resources: + - pipelinerun + schedule: 0 8 * * * + result: + disabled: false + is_external_db: false + options: {} + performance: + disable-ha: false + route_enabled: true + route_tls_termination: edge + targetNamespace: openshift-pipelines + tektonpruner: + disabled: true + global-config: + enforcedConfigLevel: global + historyLimit: 100 + options: {} + trigger: + default-service-account: pipeline + disabled: false + enable-api-fields: stable + options: {} diff --git a/otel-config.yaml b/otel-config.yaml new file mode 100644 index 000000000..a7c89cc8e --- /dev/null +++ b/otel-config.yaml @@ -0,0 +1,29 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + +exporters: + # 1. Export to Phoenix (Phoenix expects OTLP over HTTP) + otlphttp/phoenix: + endpoint: "http://phoenix:6006" + encoding: proto + + # 2. Export to Redpanda (Kafka) + kafka: + brokers: ["redpanda:29092"] # Use the internal Docker port + traces: + topic: "otel-traces" + encoding: otlp_json # This makes it MUCH easier for your Python script to read + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + # Add BOTH here to enable the fan-out + exporters: [otlphttp/phoenix, kafka] \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 462da4d35..7a72008a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,18 +3,22 @@ build-backend = "setuptools.build_meta" requires = ["setuptools >= 64", "setuptools-scm>=8"] [tool.setuptools_scm] -# intentionally empty, the section is required for setuptools_scm to work but we don't need to set anything +# Only match version tags (v0.0.1, v1.2.3, etc.), ignore PR tags like pr-162-4c9bb19 +git_describe_command = ["git", "describe", "--tags", "--match", "v*"] [project] name = "vuln_analysis" dynamic = ["version"] dependencies = [ + "exploit-iq-commons", "aiohttp-client-cache==0.11", + "beautifulsoup4>=4.12.0", "aioresponses==0.7.6", - "aiqtoolkit[langchain,profiling,telemetry]>=1.2.0a2,<1.3.0", + "nvidia-nat[langchain,profiling,telemetry]>=1.2.0rc8,<1.3.0", "aiofiles", "brotli", "esprima", + "cvss==3.6", "faiss-cpu==1.9.0", "gitpython", "google-search-results==2.4", @@ -23,12 +27,18 @@ dependencies = [ "nemollm", "openinference-instrumentation-langchain~=0.1.31", "ordered_set", + "packageurl-python", "pydpkg==1.9.4", "rank_bm25==0.2.2", "tantivy==0.22.2", "tree-sitter==0.21.3", "tree-sitter-languages==1.10.2", - "univers==30.12" + "univers==30.12", + "litellm<=1.75.8", + "csaf-tool==0.3.2", + "jsonschema>=4.0.0,<5.0.0", + "koji", + "unidiff>=0.7.5", ] requires-python = ">=3.11,<3.13" description = "NVIDIA AI Blueprint: Vulnerability Analysis for Container Security" @@ -49,6 +59,7 @@ dev = [ "pytest-asyncio==0.24.*", "pytest-cov~=6.1", "pytest-pretty~=1.2.0", + "pytest-asyncio~=0.24.0", "pytest~=8.3", "setuptools >= 64", "setuptools_scm>=8", @@ -779,3 +790,6 @@ reportInvalidTypeForm = "warning" [tool.uv] managed = true + +[tool.uv.sources] +exploit-iq-commons = { path = "src/exploit_iq_commons", editable = true } diff --git a/quick_start/quick_start_guide.ipynb b/quick_start/quick_start_guide.ipynb index 4aae39109..50efc88cb 100644 --- a/quick_start/quick_start_guide.ipynb +++ b/quick_start/quick_start_guide.ipynb @@ -25,7 +25,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "tags": [] }, @@ -48,7 +48,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "tags": [] }, @@ -81,7 +81,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "tags": [] }, @@ -101,7 +101,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": { "tags": [] }, @@ -128,162 +128,12 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": { + "scrolled": true, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "general:\n", - " use_uvloop: true\n", - "functions:\n", - " cve_generate_vdbs:\n", - " _type: cve_generate_vdbs\n", - " agent_name: cve_agent_executor\n", - " embedder_name: nim_embedder\n", - " base_git_dir: .cache/am_cache/git\n", - " base_vdb_dir: .cache/am_cache/vdb\n", - " base_code_index_dir: .cache/am_cache/code_index\n", - " cve_fetch_intel:\n", - " _type: cve_fetch_intel\n", - " cve_process_sbom:\n", - " _type: cve_process_sbom\n", - " cve_check_vuln_deps:\n", - " _type: cve_check_vuln_deps\n", - " cve_checklist:\n", - " _type: cve_checklist\n", - " llm_name: checklist_llm\n", - " Container Image Code QA System:\n", - " _type: local_vdb_retriever\n", - " embedder_name: nim_embedder\n", - " llm_name: code_vdb_retriever_llm\n", - " vdb_type: code\n", - " return_source_documents: false\n", - " Container Image Developer Guide QA System:\n", - " _type: local_vdb_retriever\n", - " embedder_name: nim_embedder\n", - " llm_name: doc_vdb_retriever_llm\n", - " vdb_type: doc\n", - " return_source_documents: false\n", - " Lexical Search Container Image Code QA System:\n", - " _type: lexical_code_search\n", - " top_k: 5\n", - " Internet Search:\n", - " _type: serp_wrapper\n", - " max_retries: 5\n", - " cve_agent_executor:\n", - " _type: cve_agent_executor\n", - " llm_name: cve_agent_executor_llm\n", - " tool_names:\n", - " - Container Image Code QA System\n", - " - Container Image Developer Guide QA System\n", - " - Internet Search\n", - " max_concurrency: null\n", - " max_iterations: 10\n", - " prompt_examples: false\n", - " replace_exceptions: true\n", - " replace_exceptions_value: I do not have a definitive answer for this checklist\n", - " item.\n", - " return_intermediate_steps: false\n", - " verbose: false\n", - " cve_summarize:\n", - " _type: cve_summarize\n", - " llm_name: summarize_llm\n", - " cve_justify:\n", - " _type: cve_justify\n", - " llm_name: justify_llm\n", - " cve_file_output:\n", - " _type: cve_file_output\n", - " file_path: .tmp/output.json\n", - " markdown_dir: .tmp/vulnerability_markdown_reports\n", - " overwrite: true\n", - "llms:\n", - " checklist_llm:\n", - " _type: nim\n", - " base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}\n", - " model_name: ${CHECKLIST_MODEL_NAME:-meta/llama-3.1-70b-instruct}\n", - " temperature: 0.0\n", - " max_tokens: 2000\n", - " top_p: 0.01\n", - " code_vdb_retriever_llm:\n", - " _type: nim\n", - " base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}\n", - " model_name: ${CODE_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct}\n", - " temperature: 0.0\n", - " max_tokens: 2000\n", - " top_p: 0.01\n", - " doc_vdb_retriever_llm:\n", - " _type: nim\n", - " base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}\n", - " model_name: ${DOC_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct}\n", - " temperature: 0.0\n", - " max_tokens: 2000\n", - " top_p: 0.01\n", - " cve_agent_executor_llm:\n", - " _type: nim\n", - " base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}\n", - " model_name: ${CVE_AGENT_EXECUTOR_MODEL_NAME:-meta/llama-3.1-70b-instruct}\n", - " temperature: 0.0\n", - " max_tokens: 2000\n", - " top_p: 0.01\n", - " summarize_llm:\n", - " _type: nim\n", - " base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}\n", - " model_name: ${SUMMARIZE_MODEL_NAME:-meta/llama-3.1-70b-instruct}\n", - " temperature: 0.0\n", - " max_tokens: 1024\n", - " top_p: 0.01\n", - " justify_llm:\n", - " _type: nim\n", - " base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1}\n", - " model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct}\n", - " temperature: 0.0\n", - " max_tokens: 1024\n", - " top_p: 0.01\n", - "embedders:\n", - " nim_embedder:\n", - " _type: nim\n", - " base_url: ${NIM_EMBED_BASE_URL:-https://integrate.api.nvidia.com/v1}\n", - " model_name: ${EMBEDDER_MODEL_NAME:-nvidia/nv-embedqa-e5-v5}\n", - " truncate: END\n", - " max_batch_size: 128\n", - "workflow:\n", - " _type: cve_agent\n", - " cve_generate_vdbs_name: cve_generate_vdbs\n", - " cve_fetch_intel_name: cve_fetch_intel\n", - " cve_process_sbom_name: cve_process_sbom\n", - " cve_check_vuln_deps_name: cve_check_vuln_deps\n", - " cve_checklist_name: cve_checklist\n", - " cve_agent_executor_name: cve_agent_executor\n", - " cve_summarize_name: cve_summarize\n", - " cve_justify_name: cve_justify\n", - " cve_output_config_name: cve_file_output\n", - "eval:\n", - " general:\n", - " output_dir: ./.tmp/eval/cve_agent\n", - " dataset:\n", - " _type: json\n", - " file_path: data/eval_datasets/eval_dataset.json\n", - " profiler:\n", - " token_uniqueness_forecast: true\n", - " workflow_runtime_forecast: true\n", - " compute_llm_metrics: true\n", - " csv_exclude_io_text: true\n", - " prompt_caching_prefixes:\n", - " enable: true\n", - " min_frequency: 0.1\n", - " bottleneck_analysis:\n", - " enable_nested_stack: true\n", - " concurrency_spike_analysis:\n", - " enable: true\n", - " spike_threshold: 7\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "with open('configs/config.yml', 'r') as config_in:\n", " workflow_config = yaml.safe_load(config_in)\n", @@ -299,7 +149,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": { "tags": [] }, @@ -370,82 +220,12 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": { + "scrolled": true, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"image\": {\n", - " \"name\": \"nvcr.io/nvidia/morpheus/morpheus\",\n", - " \"tag\": \"23.11-runtime\",\n", - " \"source_info\": [\n", - " {\n", - " \"type\": \"code\",\n", - " \"git_repo\": \"https://github.com/nv-morpheus/Morpheus.git\",\n", - " \"ref\": \"v23.11.01\",\n", - " \"include\": [\n", - " \"**/*.cpp\",\n", - " \"**/*.cu\",\n", - " \"**/*.cuh\",\n", - " \"**/*.h\",\n", - " \"**/*.hpp\",\n", - " \"**/*.ipynb\",\n", - " \"**/*.py\",\n", - " \"**/*Dockerfile\"\n", - " ],\n", - " \"exclude\": [\n", - " \"tests/**/*\"\n", - " ]\n", - " },\n", - " {\n", - " \"type\": \"doc\",\n", - " \"git_repo\": \"https://github.com/nv-morpheus/Morpheus.git\",\n", - " \"ref\": \"v23.11.01\",\n", - " \"include\": [\n", - " \"**/*.md\",\n", - " \"docs/**/*.rst\"\n", - " ]\n", - " }\n", - " ],\n", - " \"sbom_info\": {\n", - " \"_type\": \"file\",\n", - " \"file_path\": \"data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom\"\n", - " }\n", - " },\n", - " \"scan\": {\n", - " \"vulns\": [\n", - " {\n", - " \"vuln_id\": \"GHSA-3f63-hfp8-52jq\"\n", - " },\n", - " {\n", - " \"vuln_id\": \"CVE-2023-50782\"\n", - " },\n", - " {\n", - " \"vuln_id\": \"CVE-2023-36632\"\n", - " },\n", - " {\n", - " \"vuln_id\": \"CVE-2023-43804\"\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-cxfr-5q3r-2rc2\"\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-554w-xh4j-8w64\"\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-3ww4-gg4f-jr7f\"\n", - " }\n", - " ]\n", - " }\n", - "}\n" - ] - } - ], + "outputs": [], "source": [ "with open('data/input_messages/morpheus:23.11-runtime.json', 'r') as input_message_in:\n", " input_message = json.load(input_message_in)\n", @@ -461,7 +241,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": { "tags": [] }, @@ -495,7 +275,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": { "tags": [] }, @@ -514,70 +294,12 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": { + "scrolled": true, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"image\": {\n", - " \"name\": \"nvcr.io/nvidia/morpheus/morpheus\",\n", - " \"tag\": \"23.11-runtime\",\n", - " \"source_info\": [\n", - " {\n", - " \"type\": \"code\",\n", - " \"git_repo\": \"https://github.com/nv-morpheus/Morpheus.git\",\n", - " \"ref\": \"v23.11.01\",\n", - " \"include\": [\n", - " \"**/*.cpp\",\n", - " \"**/*.cu\",\n", - " \"**/*.cuh\",\n", - " \"**/*.h\",\n", - " \"**/*.hpp\",\n", - " \"**/*.ipynb\",\n", - " \"**/*.py\",\n", - " \"**/*Dockerfile\"\n", - " ],\n", - " \"exclude\": [\n", - " \"tests/**/*\"\n", - " ]\n", - " },\n", - " {\n", - " \"type\": \"doc\",\n", - " \"git_repo\": \"https://github.com/nv-morpheus/Morpheus.git\",\n", - " \"ref\": \"v23.11.01\",\n", - " \"include\": [\n", - " \"**/*.md\",\n", - " \"docs/**/*.rst\"\n", - " ]\n", - " }\n", - " ],\n", - " \"sbom_info\": {\n", - " \"_type\": \"file\",\n", - " \"file_path\": \"data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom\"\n", - " }\n", - " },\n", - " \"scan\": {\n", - " \"vulns\": [\n", - " {\n", - " \"vuln_id\": \"CVE-2024-21762\"\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-hh8p-p8mp-gqhm\"\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-3f63-hfp8-52jq\"\n", - " }\n", - " ]\n", - " }\n", - "}\n" - ] - } - ], + "outputs": [], "source": [ "print(json.dumps(input_message, indent=2))" ] @@ -591,22 +313,11 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": { "tags": [] }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "requests.post(f\"http://localhost:26466/generate\", json=input_message)" ] @@ -697,7 +408,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": { "tags": [] }, @@ -713,3961 +424,11 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"input\": {\n", - " \"scan\": {\n", - " \"id\": \"4fff5361-9706-41ad-8f0a-15730cc1d300\",\n", - " \"type\": null,\n", - " \"started_at\": \"2025-06-07T19:37:22.118544\",\n", - " \"completed_at\": \"2025-06-07T19:37:27.757216\",\n", - " \"vulns\": [\n", - " {\n", - " \"vuln_id\": \"CVE-2024-21762\",\n", - " \"description\": null,\n", - " \"score\": null,\n", - " \"severity\": null,\n", - " \"published_date\": null,\n", - " \"last_modified_date\": null,\n", - " \"url\": null,\n", - " \"feed_group\": null,\n", - " \"package\": null,\n", - " \"package_version\": null,\n", - " \"package_name\": null,\n", - " \"package_type\": null\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-hh8p-p8mp-gqhm\",\n", - " \"description\": null,\n", - " \"score\": null,\n", - " \"severity\": null,\n", - " \"published_date\": null,\n", - " \"last_modified_date\": null,\n", - " \"url\": null,\n", - " \"feed_group\": null,\n", - " \"package\": null,\n", - " \"package_version\": null,\n", - " \"package_name\": null,\n", - " \"package_type\": null\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-3f63-hfp8-52jq\",\n", - " \"description\": null,\n", - " \"score\": null,\n", - " \"severity\": null,\n", - " \"published_date\": null,\n", - " \"last_modified_date\": null,\n", - " \"url\": null,\n", - " \"feed_group\": null,\n", - " \"package\": null,\n", - " \"package_version\": null,\n", - " \"package_name\": null,\n", - " \"package_type\": null\n", - " }\n", - " ]\n", - " },\n", - " \"image\": {\n", - " \"name\": \"nvcr.io/nvidia/morpheus/morpheus\",\n", - " \"tag\": \"23.11-runtime\",\n", - " \"digest\": null,\n", - " \"platform\": null,\n", - " \"feed_group\": null,\n", - " \"source_info\": [\n", - " {\n", - " \"type\": \"code\",\n", - " \"git_repo\": \"https://github.com/nv-morpheus/Morpheus.git\",\n", - " \"ref\": \"eeebf55da604c16e6de3b060cbae6ad3172fdd99\",\n", - " \"include\": [\n", - " \"**/*.cpp\",\n", - " \"**/*.cu\",\n", - " \"**/*.cuh\",\n", - " \"**/*.h\",\n", - " \"**/*.hpp\",\n", - " \"**/*.ipynb\",\n", - " \"**/*.py\",\n", - " \"**/*Dockerfile\"\n", - " ],\n", - " \"exclude\": [\n", - " \"tests/**/*\"\n", - " ]\n", - " },\n", - " {\n", - " \"type\": \"doc\",\n", - " \"git_repo\": \"https://github.com/nv-morpheus/Morpheus.git\",\n", - " \"ref\": \"eeebf55da604c16e6de3b060cbae6ad3172fdd99\",\n", - " \"include\": [\n", - " \"**/*.md\",\n", - " \"docs/**/*.rst\"\n", - " ],\n", - " \"exclude\": []\n", - " }\n", - " ],\n", - " \"sbom_info\": {\n", - " \"_type\": \"file\",\n", - " \"file_path\": \"data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom\"\n", - " }\n", - " }\n", - " },\n", - " \"info\": {\n", - " \"vdb\": {\n", - " \"code_vdb_path\": \".cache/am_cache/vdb/code/ccf658f72228c497\",\n", - " \"doc_vdb_path\": \".cache/am_cache/vdb/doc/c89fe6d707036daa\",\n", - " \"code_index_path\": null\n", - " },\n", - " \"intel\": [\n", - " {\n", - " \"vuln_id\": \"CVE-2024-21762\",\n", - " \"ghsa\": {\n", - " \"ghsa_id\": \"GHSA-v4hq-m4wr-6pmj\",\n", - " \"cve_id\": \"CVE-2024-21762\",\n", - " \"summary\": \"A out-of-bounds write in Fortinet FortiOS versions 7.4.0 through 7.4.2, 7.2.0 through 7.2.6, 7.0...\",\n", - " \"description\": \"A out-of-bounds write in Fortinet FortiOS versions 7.4.0 through 7.4.2, 7.2.0 through 7.2.6, 7.0.0 through 7.0.13, 6.4.0 through 6.4.14, 6.2.0 through 6.2.15, 6.0.0 through 6.0.17, FortiProxy versions 7.4.0 through 7.4.2, 7.2.0 through 7.2.8, 7.0.0 through 7.0.14, 2.0.0 through 2.0.13, 1.2.0 through 1.2.13, 1.1.0 through 1.1.6, 1.0.0 through 1.0.7 allows attacker to execute unauthorized code or commands via specifically crafted requests\",\n", - " \"severity\": \"critical\",\n", - " \"vulnerabilities\": [],\n", - " \"cvss\": {\n", - " \"score\": 9.8,\n", - " \"vector_string\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\"\n", - " },\n", - " \"cwes\": [\n", - " {\n", - " \"cwe_id\": \"CWE-787\",\n", - " \"name\": \"Out-of-bounds Write\"\n", - " }\n", - " ],\n", - " \"published_at\": \"2024-02-09T09:31:31Z\",\n", - " \"updated_at\": \"2024-02-09T09:31:40Z\",\n", - " \"url\": \"https://api.github.com/advisories/GHSA-v4hq-m4wr-6pmj\",\n", - " \"html_url\": \"https://github.com/advisories/GHSA-v4hq-m4wr-6pmj\",\n", - " \"type\": \"unreviewed\",\n", - " \"repository_advisory_url\": null,\n", - " \"source_code_location\": \"\",\n", - " \"identifiers\": [\n", - " {\n", - " \"value\": \"GHSA-v4hq-m4wr-6pmj\",\n", - " \"type\": \"GHSA\"\n", - " },\n", - " {\n", - " \"value\": \"CVE-2024-21762\",\n", - " \"type\": \"CVE\"\n", - " }\n", - " ],\n", - " \"references\": [\n", - " \"https://nvd.nist.gov/vuln/detail/CVE-2024-21762\",\n", - " \"https://fortiguard.com/psirt/FG-IR-24-015\",\n", - " \"https://github.com/advisories/GHSA-v4hq-m4wr-6pmj\"\n", - " ],\n", - " \"github_reviewed_at\": null,\n", - " \"nvd_published_at\": \"2024-02-09T09:15:08Z\",\n", - " \"withdrawn_at\": null,\n", - " \"cvss_severities\": {\n", - " \"cvss_v3\": {\n", - " \"vector_string\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", - " \"score\": 9.8\n", - " },\n", - " \"cvss_v4\": {\n", - " \"vector_string\": null,\n", - " \"score\": 0.0\n", - " }\n", - " },\n", - " \"credits\": [],\n", - " \"epss\": {\n", - " \"percentage\": 0.92448,\n", - " \"percentile\": 0.99718\n", - " }\n", - " },\n", - " \"nvd\": {\n", - " \"cve_id\": \"CVE-2024-21762\",\n", - " \"cve_description\": \"A out-of-bounds write in Fortinet FortiOS versions 7.4.0 through 7.4.2, 7.2.0 through 7.2.6, 7.0.0 through 7.0.13, 6.4.0 through 6.4.14, 6.2.0 through 6.2.15, 6.0.0 through 6.0.17, FortiProxy versions 7.4.0 through 7.4.2, 7.2.0 through 7.2.8, 7.0.0 through 7.0.14, 2.0.0 through 2.0.13, 1.2.0 through 1.2.13, 1.1.0 through 1.1.6, 1.0.0 through 1.0.7 allows attacker to execute unauthorized code or commands via specifically crafted requests\",\n", - " \"cvss_vector\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", - " \"cvss_base_score\": 9.8,\n", - " \"cvss_severity\": \"CRITICAL\",\n", - " \"cwe_name\": \"CWE-787: Out-of-bounds Write (4.17)\",\n", - " \"cwe_description\": \"The product writes data past the end, or before the beginning, of the intended buffer.\",\n", - " \"cwe_extended_description\": null,\n", - " \"configurations\": [\n", - " {\n", - " \"package\": \"fortiproxy\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"2.0.14\",\n", - " \"versionStartIncluding\": \"1.0.0\",\n", - " \"versionEndIncluding\": null\n", - " },\n", - " {\n", - " \"package\": \"fortiproxy\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"7.0.15\",\n", - " \"versionStartIncluding\": \"7.0.0\",\n", - " \"versionEndIncluding\": null\n", - " },\n", - " {\n", - " \"package\": \"fortiproxy\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"7.2.9\",\n", - " \"versionStartIncluding\": \"7.2.0\",\n", - " \"versionEndIncluding\": null\n", - " },\n", - " {\n", - " \"package\": \"fortiproxy\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"7.4.3\",\n", - " \"versionStartIncluding\": \"7.4.0\",\n", - " \"versionEndIncluding\": null\n", - " },\n", - " {\n", - " \"package\": \"fortios\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"6.0.18\",\n", - " \"versionStartIncluding\": \"6.0.0\",\n", - " \"versionEndIncluding\": null\n", - " },\n", - " {\n", - " \"package\": \"fortios\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"6.2.16\",\n", - " \"versionStartIncluding\": \"6.2.0\",\n", - " \"versionEndIncluding\": null\n", - " },\n", - " {\n", - " \"package\": \"fortios\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"6.4.15\",\n", - " \"versionStartIncluding\": \"6.4.0\",\n", - " \"versionEndIncluding\": null\n", - " },\n", - " {\n", - " \"package\": \"fortios\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"7.0.14\",\n", - " \"versionStartIncluding\": \"7.0.0\",\n", - " \"versionEndIncluding\": null\n", - " },\n", - " {\n", - " \"package\": \"fortios\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"7.2.7\",\n", - " \"versionStartIncluding\": \"7.2.0\",\n", - " \"versionEndIncluding\": null\n", - " },\n", - " {\n", - " \"package\": \"fortios\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"7.4.3\",\n", - " \"versionStartIncluding\": \"7.4.0\",\n", - " \"versionEndIncluding\": null\n", - " }\n", - " ],\n", - " \"vendor_names\": [\n", - " \"Fortinet\"\n", - " ],\n", - " \"references\": [\n", - " \"https://fortiguard.com/psirt/FG-IR-24-015\",\n", - " \"https://fortiguard.com/psirt/FG-IR-24-015\"\n", - " ],\n", - " \"disputed\": false,\n", - " \"published_at\": \"2024-02-09T09:15:08.087\",\n", - " \"updated_at\": \"2024-11-29T15:23:32.167\"\n", - " },\n", - " \"rhsa\": {\n", - " \"bugzilla\": {\n", - " \"description\": null,\n", - " \"id\": null,\n", - " \"url\": null\n", - " },\n", - " \"details\": null,\n", - " \"statement\": null,\n", - " \"package_state\": null,\n", - " \"upstream_fix\": null,\n", - " \"cvss3\": null\n", - " },\n", - " \"ubuntu\": {\n", - " \"description\": null,\n", - " \"notes\": null,\n", - " \"notices\": null,\n", - " \"priority\": null,\n", - " \"ubuntu_description\": null,\n", - " \"impact\": null\n", - " },\n", - " \"epss\": {\n", - " \"epss\": 0.92448,\n", - " \"percentile\": 0.99719,\n", - " \"date\": \"2025-06-07\",\n", - " \"cve\": \"CVE-2024-21762\"\n", - " },\n", - " \"has_sufficient_intel_for_agent\": true\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-hh8p-p8mp-gqhm\",\n", - " \"ghsa\": {\n", - " \"ghsa_id\": \"GHSA-hh8p-p8mp-gqhm\",\n", - " \"cve_id\": \"CVE-2023-6975\",\n", - " \"summary\": \"MLFlow Path Traversal Vulnerability\",\n", - " \"description\": \"A malicious user could use this issue to get command execution on the vulnerable machine and get access to data & models information.\",\n", - " \"severity\": \"critical\",\n", - " \"vulnerabilities\": [\n", - " {\n", - " \"package\": {\n", - " \"ecosystem\": \"pip\",\n", - " \"name\": \"mlflow\"\n", - " },\n", - " \"vulnerable_version_range\": \"< 2.9.2\",\n", - " \"first_patched_version\": \"2.9.2\",\n", - " \"vulnerable_functions\": []\n", - " }\n", - " ],\n", - " \"cvss\": {\n", - " \"score\": 9.8,\n", - " \"vector_string\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\"\n", - " },\n", - " \"cwes\": [\n", - " {\n", - " \"cwe_id\": \"CWE-29\",\n", - " \"name\": \"Path Traversal: '..filename'\"\n", - " }\n", - " ],\n", - " \"published_at\": \"2023-12-20T06:30:25Z\",\n", - " \"updated_at\": \"2024-01-02T15:14:39Z\",\n", - " \"url\": \"https://api.github.com/advisories/GHSA-hh8p-p8mp-gqhm\",\n", - " \"html_url\": \"https://github.com/advisories/GHSA-hh8p-p8mp-gqhm\",\n", - " \"type\": \"reviewed\",\n", - " \"repository_advisory_url\": null,\n", - " \"source_code_location\": \"https://github.com/mlflow/mlflow\",\n", - " \"identifiers\": [\n", - " {\n", - " \"value\": \"GHSA-hh8p-p8mp-gqhm\",\n", - " \"type\": \"GHSA\"\n", - " },\n", - " {\n", - " \"value\": \"CVE-2023-6975\",\n", - " \"type\": \"CVE\"\n", - " }\n", - " ],\n", - " \"references\": [\n", - " \"https://nvd.nist.gov/vuln/detail/CVE-2023-6975\",\n", - " \"https://github.com/mlflow/mlflow/commit/b9ab9ed77e1deda9697fe472fb1079fd428149ee\",\n", - " \"https://huntr.com/bounties/029a3824-cee3-4cf1-b260-7138aa539b85\",\n", - " \"https://github.com/advisories/GHSA-hh8p-p8mp-gqhm\"\n", - " ],\n", - " \"github_reviewed_at\": \"2023-12-20T20:32:39Z\",\n", - " \"nvd_published_at\": \"2023-12-20T06:15:45Z\",\n", - " \"withdrawn_at\": null,\n", - " \"cvss_severities\": {\n", - " \"cvss_v3\": {\n", - " \"vector_string\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", - " \"score\": 9.8\n", - " },\n", - " \"cvss_v4\": {\n", - " \"vector_string\": null,\n", - " \"score\": 0.0\n", - " }\n", - " },\n", - " \"credits\": [],\n", - " \"epss\": {\n", - " \"percentage\": 0.01542,\n", - " \"percentile\": 0.80461\n", - " }\n", - " },\n", - " \"nvd\": {\n", - " \"cve_id\": \"CVE-2023-6975\",\n", - " \"cve_description\": \"A malicious user could use this issue to get command execution on the vulnerable machine and get access to data & models information.\",\n", - " \"cvss_vector\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", - " \"cvss_base_score\": 9.8,\n", - " \"cvss_severity\": \"CRITICAL\",\n", - " \"cwe_name\": \"CWE-29: Path Traversal: '\\\\..\\\\filename' (4.17)\",\n", - " \"cwe_description\": \"The product uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '\\\\..\\\\filename' (leading backslash dot dot) sequences that can resolve to a location that is outside of that directory.\",\n", - " \"cwe_extended_description\": \"This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.\\nThis is similar to CWE-25, except using \\\"\\\\\\\" instead of \\\"/\\\". Sometimes a program checks for \\\"..\\\\\\\" at the beginning of the input, so a \\\"\\\\..\\\\\\\" can bypass that check. It is also useful for bypassing path traversal protection schemes that only assume that the \\\"/\\\" separator is valid.\",\n", - " \"configurations\": [\n", - " {\n", - " \"package\": \"mlflow\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": \"2.9.2\",\n", - " \"versionStartIncluding\": null,\n", - " \"versionEndIncluding\": null\n", - " }\n", - " ],\n", - " \"vendor_names\": [\n", - " \"Lfprojects\"\n", - " ],\n", - " \"references\": [\n", - " \"https://github.com/mlflow/mlflow/commit/b9ab9ed77e1deda9697fe472fb1079fd428149ee\",\n", - " \"https://huntr.com/bounties/029a3824-cee3-4cf1-b260-7138aa539b85\",\n", - " \"https://github.com/mlflow/mlflow/commit/b9ab9ed77e1deda9697fe472fb1079fd428149ee\",\n", - " \"https://huntr.com/bounties/029a3824-cee3-4cf1-b260-7138aa539b85\"\n", - " ],\n", - " \"disputed\": false,\n", - " \"published_at\": \"2023-12-20T06:15:45.553\",\n", - " \"updated_at\": \"2024-11-21T08:44:57.463\"\n", - " },\n", - " \"rhsa\": {\n", - " \"bugzilla\": {\n", - " \"description\": null,\n", - " \"id\": null,\n", - " \"url\": null\n", - " },\n", - " \"details\": null,\n", - " \"statement\": null,\n", - " \"package_state\": null,\n", - " \"upstream_fix\": null,\n", - " \"cvss3\": null\n", - " },\n", - " \"ubuntu\": {\n", - " \"description\": null,\n", - " \"notes\": null,\n", - " \"notices\": null,\n", - " \"priority\": null,\n", - " \"ubuntu_description\": null,\n", - " \"impact\": null\n", - " },\n", - " \"epss\": {\n", - " \"epss\": 0.01542,\n", - " \"percentile\": 0.80467,\n", - " \"date\": \"2025-06-07\",\n", - " \"cve\": \"CVE-2023-6975\"\n", - " },\n", - " \"has_sufficient_intel_for_agent\": true\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-3f63-hfp8-52jq\",\n", - " \"ghsa\": {\n", - " \"ghsa_id\": \"GHSA-3f63-hfp8-52jq\",\n", - " \"cve_id\": \"CVE-2023-50447\",\n", - " \"summary\": \"Arbitrary Code Execution in Pillow\",\n", - " \"description\": \"Pillow through 10.1.0 allows PIL.ImageMath.eval Arbitrary Code Execution via the environment parameter, a different vulnerability than CVE-2022-22817 (which was about the expression parameter).\",\n", - " \"severity\": \"critical\",\n", - " \"vulnerabilities\": [\n", - " {\n", - " \"package\": {\n", - " \"ecosystem\": \"pip\",\n", - " \"name\": \"Pillow\"\n", - " },\n", - " \"vulnerable_version_range\": \"< 10.2.0\",\n", - " \"first_patched_version\": \"10.2.0\",\n", - " \"vulnerable_functions\": []\n", - " }\n", - " ],\n", - " \"cvss\": {\n", - " \"score\": 8.1,\n", - " \"vector_string\": \"CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\"\n", - " },\n", - " \"cwes\": [\n", - " {\n", - " \"cwe_id\": \"CWE-94\",\n", - " \"name\": \"Improper Control of Generation of Code ('Code Injection')\"\n", - " },\n", - " {\n", - " \"cwe_id\": \"CWE-95\",\n", - " \"name\": \"Improper Neutralization of Directives in Dynamically Evaluated Code ('Eval Injection')\"\n", - " }\n", - " ],\n", - " \"published_at\": \"2024-01-19T21:30:35Z\",\n", - " \"updated_at\": \"2024-11-18T16:26:35Z\",\n", - " \"url\": \"https://api.github.com/advisories/GHSA-3f63-hfp8-52jq\",\n", - " \"html_url\": \"https://github.com/advisories/GHSA-3f63-hfp8-52jq\",\n", - " \"type\": \"reviewed\",\n", - " \"repository_advisory_url\": null,\n", - " \"source_code_location\": \"https://github.com/python-pillow/Pillow\",\n", - " \"identifiers\": [\n", - " {\n", - " \"value\": \"GHSA-3f63-hfp8-52jq\",\n", - " \"type\": \"GHSA\"\n", - " },\n", - " {\n", - " \"value\": \"CVE-2023-50447\",\n", - " \"type\": \"CVE\"\n", - " }\n", - " ],\n", - " \"references\": [\n", - " \"https://nvd.nist.gov/vuln/detail/CVE-2023-50447\",\n", - " \"https://github.com/python-pillow/Pillow/commit/45c726fd4daa63236a8f3653530f297dc87b160a\",\n", - " \"https://pillow.readthedocs.io/en/stable/releasenotes/10.2.0.html#security\",\n", - " \"http://www.openwall.com/lists/oss-security/2024/01/20/1\",\n", - " \"https://github.com/python-pillow/Pillow/releases\",\n", - " \"https://lists.debian.org/debian-lts-announce/2024/01/msg00019.html\",\n", - " \"https://devhub.checkmarx.com/cve-details/CVE-2023-50447\",\n", - " \"https://duartecsantos.github.io/2023-01-02-CVE-2023-50447\",\n", - " \"https://duartecsantos.github.io/2024-01-02-CVE-2023-50447\",\n", - " \"https://github.com/advisories/GHSA-3f63-hfp8-52jq\"\n", - " ],\n", - " \"github_reviewed_at\": \"2024-01-22T21:28:18Z\",\n", - " \"nvd_published_at\": \"2024-01-19T20:15:11Z\",\n", - " \"withdrawn_at\": null,\n", - " \"cvss_severities\": {\n", - " \"cvss_v3\": {\n", - " \"vector_string\": \"CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", - " \"score\": 8.1\n", - " },\n", - " \"cvss_v4\": {\n", - " \"vector_string\": \"CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:H/VI:H/VA:H/SC:N/SI:N/SA:N\",\n", - " \"score\": 9.3\n", - " }\n", - " },\n", - " \"credits\": [],\n", - " \"epss\": {\n", - " \"percentage\": 0.00408,\n", - " \"percentile\": 0.60132\n", - " }\n", - " },\n", - " \"nvd\": {\n", - " \"cve_id\": \"CVE-2023-50447\",\n", - " \"cve_description\": \"Pillow through 10.1.0 allows PIL.ImageMath.eval Arbitrary Code Execution via the environment parameter, a different vulnerability than CVE-2022-22817 (which was about the expression parameter).\",\n", - " \"cvss_vector\": \"CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", - " \"cvss_base_score\": 8.1,\n", - " \"cvss_severity\": \"HIGH\",\n", - " \"cwe_name\": \"CWE-94: Improper Control of Generation of Code ('Code Injection') (4.17)\",\n", - " \"cwe_description\": \"The product constructs all or part of a code segment using externally-influenced input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could modify the syntax or behavior of the intended code segment.\",\n", - " \"cwe_extended_description\": null,\n", - " \"configurations\": [\n", - " {\n", - " \"package\": \"pillow\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": null,\n", - " \"versionStartIncluding\": null,\n", - " \"versionEndIncluding\": \"10.1.0\"\n", - " },\n", - " {\n", - " \"package\": \"debian_linux\",\n", - " \"system\": null,\n", - " \"versionStartExcluding\": null,\n", - " \"versionEndExcluding\": null,\n", - " \"versionStartIncluding\": \"10.0\",\n", - " \"versionEndIncluding\": \"10.0\"\n", - " }\n", - " ],\n", - " \"vendor_names\": [\n", - " \"Debian\",\n", - " \"Python\"\n", - " ],\n", - " \"references\": [\n", - " \"http://www.openwall.com/lists/oss-security/2024/01/20/1\",\n", - " \"https://devhub.checkmarx.com/cve-details/CVE-2023-50447/\",\n", - " \"https://duartecsantos.github.io/2024-01-02-CVE-2023-50447/\",\n", - " \"https://github.com/python-pillow/Pillow/releases\",\n", - " \"https://lists.debian.org/debian-lts-announce/2024/01/msg00019.html\",\n", - " \"http://www.openwall.com/lists/oss-security/2024/01/20/1\",\n", - " \"https://devhub.checkmarx.com/cve-details/CVE-2023-50447/\",\n", - " \"https://duartecsantos.github.io/2024-01-02-CVE-2023-50447/\",\n", - " \"https://github.com/python-pillow/Pillow/releases\",\n", - " \"https://lists.debian.org/debian-lts-announce/2024/01/msg00019.html\"\n", - " ],\n", - " \"disputed\": false,\n", - " \"published_at\": \"2024-01-19T20:15:11.870\",\n", - " \"updated_at\": \"2024-11-21T08:37:00.967\"\n", - " },\n", - " \"rhsa\": {\n", - " \"bugzilla\": {\n", - " \"description\": \"pillow: Arbitrary Code Execution via the environment parameter\",\n", - " \"id\": \"2259479\",\n", - " \"url\": \"https://bugzilla.redhat.com/show_bug.cgi?id=2259479\"\n", - " },\n", - " \"details\": [\n", - " \"Pillow through 10.1.0 allows PIL.ImageMath.eval Arbitrary Code Execution via the environment parameter, a different vulnerability than CVE-2022-22817 (which was about the expression parameter).\",\n", - " \"A vulnerability was found in Pillow, a popular Python imaging library. The flaw identified in the PIL.ImageMath.eval function enables arbitrary code execution by manipulating the environment parameter.\"\n", - " ],\n", - " \"statement\": \"The vulnerability in Pillow's PIL.ImageMath.eval function poses a significant threat due to its potential for arbitrary code execution. Pillow's widespread use in diverse domains makes this flaw particularly impactful, as it could lead to unauthorized access, data breaches, and compromise of entire systems. The complex exploitation method involving Python's dunder methods adds sophistication to potential attacks.\",\n", - " \"package_state\": null,\n", - " \"upstream_fix\": null,\n", - " \"cvss3\": {\n", - " \"cvss3_base_score\": 8.1,\n", - " \"cvss3_scoring_vector\": \"CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", - " \"status\": \"verified\"\n", - " },\n", - " \"threat_severity\": \"Important\",\n", - " \"public_date\": \"2024-01-19T00:00:00Z\",\n", - " \"cwe\": \"CWE-77\",\n", - " \"affected_release\": [\n", - " {\n", - " \"product_name\": \"Red Hat Ansible Automation Platform 2.4 for RHEL 8\",\n", - " \"release_date\": \"2024-06-10T00:00:00Z\",\n", - " \"advisory\": \"RHSA-2024:3781\",\n", - " \"cpe\": \"cpe:/a:redhat:ansible_automation_platform:2.4::el8\",\n", - " \"package\": \"python3x-pillow-0:10.3.0-1.el8ap\"\n", - " },\n", - " {\n", - " \"product_name\": \"Red Hat Ansible Automation Platform 2.4 for RHEL 9\",\n", - " \"release_date\": \"2024-06-10T00:00:00Z\",\n", - " \"advisory\": \"RHSA-2024:3781\",\n", - " \"cpe\": \"cpe:/a:redhat:ansible_automation_platform:2.4::el9\",\n", - " \"package\": \"python-pillow-0:10.3.0-1.el9ap\"\n", - " },\n", - " {\n", - " \"product_name\": \"Red Hat Enterprise Linux 7\",\n", - " \"release_date\": \"2024-02-19T00:00:00Z\",\n", - " \"advisory\": \"RHSA-2024:0857\",\n", - " \"cpe\": \"cpe:/o:redhat:enterprise_linux:7\",\n", - " \"package\": \"python-pillow-0:2.0.0-25.gitd1c6db8.el7_9\"\n", - " },\n", - " {\n", - " \"product_name\": \"Red Hat Enterprise Linux 8\",\n", - " \"release_date\": \"2024-02-20T00:00:00Z\",\n", - " \"advisory\": \"RHSA-2024:0893\",\n", - " \"cpe\": \"cpe:/a:redhat:enterprise_linux:8\",\n", - " \"package\": \"python-pillow-0:5.1.1-18.el8_9.1\"\n", - " },\n", - " {\n", - " \"product_name\": \"Red Hat Enterprise Linux 8.2 Advanced Update Support\",\n", - " \"release_date\": \"2024-02-29T00:00:00Z\",\n", - " \"advisory\": \"RHSA-2024:1059\",\n", - " \"cpe\": \"cpe:/a:redhat:rhel_aus:8.2\",\n", - " \"package\": \"python-pillow-0:5.1.1-15.el8_2\"\n", - " },\n", - " {\n", - " \"product_name\": \"Red Hat Enterprise Linux 8.4 Advanced Mission Critical Update Support\",\n", - " \"release_date\": \"2024-02-29T00:00:00Z\",\n", - " \"advisory\": \"RHSA-2024:1060\",\n", - " \"cpe\": \"cpe:/a:redhat:rhel_aus:8.4\",\n", - " \"package\": \"python-pillow-0:5.1.1-15.el8_4\"\n", - " },\n", - " {\n", - " \"product_name\": \"Red Hat Enterprise Linux 8.4 Telecommunications Update Service\",\n", - " \"release_date\": \"2024-02-29T00:00:00Z\",\n", - " \"advisory\": \"RHSA-2024:1060\",\n", - " \"cpe\": \"cpe:/a:redhat:rhel_tus:8.4\",\n", - " \"package\": \"python-pillow-0:5.1.1-15.el8_4\"\n", - " },\n", - " {\n", - " \"product_name\": \"Red Hat Enterprise Linux 8.4 Update Services for SAP Solutions\",\n", - " \"release_date\": \"2024-02-29T00:00:00Z\",\n", - " \"advisory\": \"RHSA-2024:1060\",\n", - " \"cpe\": \"cpe:/a:redhat:rhel_e4s:8.4\",\n", - " \"package\": \"python-pillow-0:5.1.1-15.el8_4\"\n", - " },\n", - " {\n", - " \"product_name\": \"Red Hat Enterprise Linux 8.6 Extended Update Support\",\n", - " \"release_date\": \"2024-02-29T00:00:00Z\",\n", - " \"advisory\": \"RHSA-2024:1058\",\n", - " \"cpe\": \"cpe:/a:redhat:rhel_eus:8.6\",\n", - " \"package\": \"python-pillow-0:5.1.1-19.el8_6\"\n", - " },\n", - " {\n", - " \"product_name\": \"Red Hat Enterprise Linux 8.8 Extended Update Support\",\n", - " \"release_date\": \"2024-02-08T00:00:00Z\",\n", - " \"advisory\": \"RHSA-2024:0754\",\n", - " \"cpe\": \"cpe:/a:redhat:rhel_eus:8.8\",\n", - " \"package\": \"python-pillow-0:5.1.1-19.el8_8\"\n", - " }\n", - " ],\n", - " \"references\": [\n", - " \"https://www.cve.org/CVERecord?id=CVE-2023-50447\\nhttps://nvd.nist.gov/vuln/detail/CVE-2023-50447\\nhttp://www.openwall.com/lists/oss-security/2024/01/20/1\\nhttps://devhub.checkmarx.com/cve-details/CVE-2023-50447/\\nhttps://duartecsantos.github.io/2023-01-02-CVE-2023-50447/\\nhttps://github.com/python-pillow/Pillow/releases\"\n", - " ],\n", - " \"name\": \"CVE-2023-50447\",\n", - " \"mitigation\": {\n", - " \"value\": \"Mitigation for this issue is either not available or the currently available options do not meet the Red Hat Product Security criteria comprising ease of use and deployment, applicability to widespread installation base or stability.\",\n", - " \"lang\": \"en:us\"\n", - " },\n", - " \"csaw\": false\n", - " },\n", - " \"ubuntu\": {\n", - " \"description\": \"\\nPillow through 10.1.0 allows PIL.ImageMath.eval Arbitrary Code Execution\\nvia the environment parameter, a different vulnerability than\\nCVE-2022-22817 (which was about the expression parameter).\",\n", - " \"notes\": [],\n", - " \"notices\": [\n", - " {\n", - " \"cves_ids\": [\n", - " \"CVE-2023-44271\",\n", - " \"CVE-2023-50447\"\n", - " ],\n", - " \"description\": \"It was discovered that Pillow incorrectly handled certain long text\\narguments. An attacker could possibly use this issue to cause Pillow to\\nconsume resources, leading to a denial of service. This issue only affected\\nUbuntu 20.04 LTS, and Ubuntu 22.04 LTS. (CVE-2023-44271)\\n\\nDuarte Santos discovered that Pillow incorrectly handled the environment\\nparameter to PIL.ImageMath.eval. An attacker could possibly use this issue\\nto execute arbitrary code. (CVE-2023-50447)\\n\",\n", - " \"id\": \"USN-6618-1\",\n", - " \"instructions\": \"In general, a standard system update will make all the necessary changes.\\n\",\n", - " \"is_hidden\": false,\n", - " \"published\": \"2024-01-30T15:17:54.445907\",\n", - " \"references\": [],\n", - " \"release_packages\": {\n", - " \"focal\": [\n", - " {\n", - " \"description\": \"Python Imaging Library\",\n", - " \"is_source\": true,\n", - " \"name\": \"pillow\",\n", - " \"version\": \"7.0.0-4ubuntu0.8\"\n", - " },\n", - " {\n", - " \"is_source\": false,\n", - " \"is_visible\": false,\n", - " \"name\": \"python-pil-doc\",\n", - " \"pocket\": \"security\",\n", - " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", - " \"version\": \"7.0.0-4ubuntu0.8\",\n", - " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/7.0.0-4ubuntu0.8\"\n", - " },\n", - " {\n", - " \"is_source\": false,\n", - " \"is_visible\": true,\n", - " \"name\": \"python3-pil\",\n", - " \"pocket\": \"security\",\n", - " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", - " \"version\": \"7.0.0-4ubuntu0.8\",\n", - " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/7.0.0-4ubuntu0.8\"\n", - " },\n", - " {\n", - " \"is_source\": false,\n", - " \"is_visible\": false,\n", - " \"name\": \"python3-pil.imagetk\",\n", - " \"pocket\": \"security\",\n", - " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", - " \"version\": \"7.0.0-4ubuntu0.8\",\n", - " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/7.0.0-4ubuntu0.8\"\n", - " }\n", - " ],\n", - " \"jammy\": [\n", - " {\n", - " \"description\": \"Python Imaging Library\",\n", - " \"is_source\": true,\n", - " \"name\": \"pillow\",\n", - " \"version\": \"9.0.1-1ubuntu0.2\"\n", - " },\n", - " {\n", - " \"is_source\": false,\n", - " \"is_visible\": false,\n", - " \"name\": \"python-pil-doc\",\n", - " \"pocket\": \"security\",\n", - " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", - " \"version\": \"9.0.1-1ubuntu0.2\",\n", - " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/9.0.1-1ubuntu0.2\"\n", - " },\n", - " {\n", - " \"is_source\": false,\n", - " \"is_visible\": true,\n", - " \"name\": \"python3-pil\",\n", - " \"pocket\": \"security\",\n", - " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", - " \"version\": \"9.0.1-1ubuntu0.2\",\n", - " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/9.0.1-1ubuntu0.2\"\n", - " },\n", - " {\n", - " \"is_source\": false,\n", - " \"is_visible\": false,\n", - " \"name\": \"python3-pil.imagetk\",\n", - " \"pocket\": \"security\",\n", - " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", - " \"version\": \"9.0.1-1ubuntu0.2\",\n", - " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/9.0.1-1ubuntu0.2\"\n", - " }\n", - " ],\n", - " \"mantic\": [\n", - " {\n", - " \"description\": \"Python Imaging Library\",\n", - " \"is_source\": true,\n", - " \"name\": \"pillow\",\n", - " \"version\": \"10.0.0-1ubuntu0.1\"\n", - " },\n", - " {\n", - " \"is_source\": false,\n", - " \"is_visible\": false,\n", - " \"name\": \"python-pil-doc\",\n", - " \"pocket\": \"security\",\n", - " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", - " \"version\": \"10.0.0-1ubuntu0.1\",\n", - " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/10.0.0-1ubuntu0.1\"\n", - " },\n", - " {\n", - " \"is_source\": false,\n", - " \"is_visible\": true,\n", - " \"name\": \"python3-pil\",\n", - " \"pocket\": \"security\",\n", - " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", - " \"version\": \"10.0.0-1ubuntu0.1\",\n", - " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/10.0.0-1ubuntu0.1\"\n", - " },\n", - " {\n", - " \"is_source\": false,\n", - " \"is_visible\": false,\n", - " \"name\": \"python3-pil.imagetk\",\n", - " \"pocket\": \"security\",\n", - " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", - " \"version\": \"10.0.0-1ubuntu0.1\",\n", - " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/10.0.0-1ubuntu0.1\"\n", - " }\n", - " ]\n", - " },\n", - " \"summary\": \"Several security issues were fixed in Pillow.\\n\",\n", - " \"title\": \"Pillow vulnerabilities\",\n", - " \"type\": \"USN\"\n", - " }\n", - " ],\n", - " \"priority\": \"medium\",\n", - " \"ubuntu_description\": \"\",\n", - " \"impact\": {\n", - " \"baseMetricV3\": {\n", - " \"cvssV3\": {\n", - " \"attackComplexity\": \"HIGH\",\n", - " \"attackVector\": \"NETWORK\",\n", - " \"availabilityImpact\": \"HIGH\",\n", - " \"baseScore\": 8.1,\n", - " \"baseSeverity\": \"HIGH\",\n", - " \"confidentialityImpact\": \"HIGH\",\n", - " \"integrityImpact\": \"HIGH\",\n", - " \"privilegesRequired\": \"NONE\",\n", - " \"scope\": \"UNCHANGED\",\n", - " \"userInteraction\": \"NONE\",\n", - " \"vectorString\": \"CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", - " \"version\": \"3.1\"\n", - " },\n", - " \"exploitabilityScore\": 2.2,\n", - " \"impactScore\": 5.9\n", - " }\n", - " },\n", - " \"bugs\": [\n", - " \"http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1061172\"\n", - " ],\n", - " \"codename\": null,\n", - " \"cvss3\": 8.1,\n", - " \"id\": \"CVE-2023-50447\",\n", - " \"mitigation\": \"\",\n", - " \"notices_ids\": [\n", - " \"USN-6618-1\"\n", - " ],\n", - " \"packages\": [\n", - " {\n", - " \"debian\": \"https://tracker.debian.org/pkg/pillow\",\n", - " \"name\": \"pillow\",\n", - " \"source\": \"https://ubuntu.com/security/cve?package=pillow\",\n", - " \"statuses\": [\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"bionic\",\n", - " \"status\": \"needs-triage\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"7.0.0-4ubuntu0.8\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"focal\",\n", - " \"status\": \"released\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"9.0.1-1ubuntu0.2\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"jammy\",\n", - " \"status\": \"released\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"end of life, was needed\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"lunar\",\n", - " \"status\": \"ignored\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"10.0.0-1ubuntu0.1\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"mantic\",\n", - " \"status\": \"released\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"10.2.0-1\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"noble\",\n", - " \"status\": \"released\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"10.2.0-1\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"oracular\",\n", - " \"status\": \"released\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"10.2.0-1\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"plucky\",\n", - " \"status\": \"released\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"trusty\",\n", - " \"status\": \"needs-triage\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"10.2.0-1\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"upstream\",\n", - " \"status\": \"released\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"xenial\",\n", - " \"status\": \"needs-triage\"\n", - " }\n", - " ],\n", - " \"ubuntu\": \"https://packages.ubuntu.com/search?suite=all§ion=all&arch=any&searchon=sourcenames&keywords=pillow\"\n", - " },\n", - " {\n", - " \"debian\": \"https://tracker.debian.org/pkg/pillow-python2\",\n", - " \"name\": \"pillow-python2\",\n", - " \"source\": \"https://ubuntu.com/security/cve?package=pillow-python2\",\n", - " \"statuses\": [\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"bionic\",\n", - " \"status\": \"DNE\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"focal\",\n", - " \"status\": \"needs-triage\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"jammy\",\n", - " \"status\": \"DNE\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"lunar\",\n", - " \"status\": \"DNE\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"mantic\",\n", - " \"status\": \"DNE\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"noble\",\n", - " \"status\": \"DNE\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"oracular\",\n", - " \"status\": \"DNE\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"plucky\",\n", - " \"status\": \"DNE\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"trusty\",\n", - " \"status\": \"DNE\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"upstream\",\n", - " \"status\": \"needs-triage\"\n", - " },\n", - " {\n", - " \"component\": null,\n", - " \"description\": \"\",\n", - " \"pocket\": \"security\",\n", - " \"release_codename\": \"xenial\",\n", - " \"status\": \"DNE\"\n", - " }\n", - " ],\n", - " \"ubuntu\": \"https://packages.ubuntu.com/search?suite=all§ion=all&arch=any&searchon=sourcenames&keywords=pillow-python2\"\n", - " }\n", - " ],\n", - " \"patches\": {\n", - " \"pillow\": [\n", - " \"upstream: https://github.com/python-pillow/Pillow/commit/45c726fd4daa63236a8f3653530f297dc87b160a\",\n", - " \"upstream: https://github.com/python-pillow/Pillow/commit/0ca3c33c59927e1c7e0c14dbc1eea1dfb2431a80\",\n", - " \"upstream: https://github.com/python-pillow/Pillow/commit/557ba59d13de919d04b3fd4cdef8634f7d4b3348\"\n", - " ],\n", - " \"pillow-python2\": []\n", - " },\n", - " \"published\": \"2024-01-19T20:15:00\",\n", - " \"references\": [\n", - " \"https://duartecsantos.github.io/2023-01-02-CVE-2023-50447/\",\n", - " \"https://pillow.readthedocs.io/en/stable/releasenotes/10.2.0.html#imagemath-eval-restricted-environment-keys\",\n", - " \"https://devhub.checkmarx.com/cve-details/CVE-2023-50447/\",\n", - " \"http://www.openwall.com/lists/oss-security/2024/01/20/1\",\n", - " \"https://ubuntu.com/security/notices/USN-6618-1\",\n", - " \"https://www.cve.org/CVERecord?id=CVE-2023-50447\"\n", - " ],\n", - " \"status\": \"active\",\n", - " \"tags\": {},\n", - " \"updated_at\": \"2024-07-24T15:57:39.284958+00:00\"\n", - " },\n", - " \"epss\": {\n", - " \"epss\": 0.00408,\n", - " \"percentile\": 0.60242,\n", - " \"date\": \"2025-05-30\",\n", - " \"cve\": \"CVE-2023-50447\"\n", - " },\n", - " \"has_sufficient_intel_for_agent\": true\n", - " }\n", - " ],\n", - " \"sbom\": {\n", - " \"packages\": [\n", - " {\n", - " \"name\": \"Babel\",\n", - " \"version\": \"2.13.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"Brotli\",\n", - " \"version\": \"1.0.9\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"Brotli\",\n", - " \"version\": \"1.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"ConfigArgParse\",\n", - " \"version\": \"1.5.5\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"Flask\",\n", - " \"version\": \"3.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"GitPython\",\n", - " \"version\": \"3.1.40\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"Jinja2\",\n", - " \"version\": \"3.1.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"Mako\",\n", - " \"version\": \"1.3.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"Markdown\",\n", - " \"version\": \"3.5.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"MarkupSafe\",\n", - " \"version\": \"2.1.3\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"Pillow\",\n", - " \"version\": \"10.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"PyJWT\",\n", - " \"version\": \"2.8.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"PySocks\",\n", - " \"version\": \"1.7.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"PyYAML\",\n", - " \"version\": \"6.0.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"Pygments\",\n", - " \"version\": \"2.17.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"SQLAlchemy\",\n", - " \"version\": \"2.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"SciPy\",\n", - " \"version\": \"1.11.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"Simple\",\n", - " \"version\": \"Launcher\",\n", - " \"path\": null,\n", - " \"system\": \"1.1.0.14\"\n", - " },\n", - " {\n", - " \"name\": \"Sphinx\",\n", - " \"version\": \"7.2.6\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"Werkzeug\",\n", - " \"version\": \"3.0.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"absl-py\",\n", - " \"version\": \"1.4.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"adduser\",\n", - " \"version\": \"3.118ubuntu5\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"alabaster\",\n", - " \"version\": \"0.7.13\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"alembic\",\n", - " \"version\": \"1.13.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"anyio\",\n", - " \"version\": \"3.7.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"appdirs\",\n", - " \"version\": \"1.4.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"apt\",\n", - " \"version\": \"2.4.11\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"asn1crypto\",\n", - " \"version\": \"1.5.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"attrs\",\n", - " \"version\": \"23.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"base-files\",\n", - " \"version\": \"12ubuntu4.4\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"base-passwd\",\n", - " \"version\": \"3.5.52build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"bash\",\n", - " \"version\": \"5.1-6ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"bc\",\n", - " \"version\": \"1.07.1-3build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"beautifulsoup4\",\n", - " \"version\": \"4.12.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"betterproto\",\n", - " \"version\": \"1.2.5\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"binutils\",\n", - " \"version\": \"2.38-4ubuntu2.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"binutils-common\",\n", - " \"version\": \"2.38-4ubuntu2.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"binutils-x86-64-linux-gnu\",\n", - " \"version\": \"2.38-4ubuntu2.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"blinker\",\n", - " \"version\": \"1.7.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"boa\",\n", - " \"version\": \"0.16.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"bokeh\",\n", - " \"version\": \"2.4.3\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"boltons\",\n", - " \"version\": \"23.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"bsdutils\",\n", - " \"version\": \"1:2.37.2-4ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"build-essential\",\n", - " \"version\": \"12.9ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"bzip2\",\n", - " \"version\": \"1.0.8-5build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"ca-certificates\",\n", - " \"version\": \"20230311ubuntu0.22.04.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"ca-certificates-java\",\n", - " \"version\": \"java-archive\",\n", - " \"path\": null,\n", - " \"system\": \"\\u001b[38;2;119;119;119m\\u001b[0m\"\n", - " },\n", - " {\n", - " \"name\": \"ca-certificates-java\",\n", - " \"version\": \"20190909ubuntu1.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cachetools\",\n", - " \"version\": \"5.3.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cattrs\",\n", - " \"version\": \"23.2.3\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"certifi\",\n", - " \"version\": \"2023.11.17\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cffi\",\n", - " \"version\": \"1.15.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cffi\",\n", - " \"version\": \"1.16.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"chardet\",\n", - " \"version\": \"5.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"charset-normalizer\",\n", - " \"version\": \"3.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"charset-normalizer\",\n", - " \"version\": \"3.3.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"click\",\n", - " \"version\": \"8.1.7\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cloudpickle\",\n", - " \"version\": \"3.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"colorama\",\n", - " \"version\": \"0.4.6\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"conda\",\n", - " \"version\": \"23.3.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"conda-build\",\n", - " \"version\": \"3.28.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"conda-libmamba-solver\",\n", - " \"version\": \"23.3.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"conda-package-handling\",\n", - " \"version\": \"2.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"conda_index\",\n", - " \"version\": \"0.3.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"conda_package_streaming\",\n", - " \"version\": \"0.9.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"configparser\",\n", - " \"version\": \"5.3.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"confluent-kafka\",\n", - " \"version\": \"1.9.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"contourpy\",\n", - " \"version\": \"1.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"coreutils\",\n", - " \"version\": \"8.32-4.1ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cpp\",\n", - " \"version\": \"4:11.2.0-1ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cpp-11\",\n", - " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cryptography\",\n", - " \"version\": \"41.0.3\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cryptography\",\n", - " \"version\": \"41.0.7\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cubinlinker\",\n", - " \"version\": \"0.3.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-cccl-11-8\",\n", - " \"version\": \"11.8.89-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-compat-11-8\",\n", - " \"version\": \"520.61.05-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-compiler-11-8\",\n", - " \"version\": \"11.8.0-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-cudart-11-8\",\n", - " \"version\": \"11.8.89-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-cudart-dev-11-8\",\n", - " \"version\": \"11.8.89-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-cuobjdump-11-8\",\n", - " \"version\": \"11.8.86-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-cupti-11-8\",\n", - " \"version\": \"11.8.87-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-cuxxfilt-11-8\",\n", - " \"version\": \"11.8.86-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-driver-dev-11-8\",\n", - " \"version\": \"11.8.89-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-gdb-11-8\",\n", - " \"version\": \"11.8.86-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-keyring\",\n", - " \"version\": \"1.1-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-nvcc-11-8\",\n", - " \"version\": \"11.8.89-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-nvdisasm-11-8\",\n", - " \"version\": \"11.8.86-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-nvprune-11-8\",\n", - " \"version\": \"11.8.86-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-python\",\n", - " \"version\": \"11.8.3\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-toolkit-11-8-config-common\",\n", - " \"version\": \"11.8.89-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-toolkit-11-config-common\",\n", - " \"version\": \"11.8.89-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cuda-toolkit-config-common\",\n", - " \"version\": \"12.3.101-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cudf\",\n", - " \"version\": \"23.6.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cudf-kafka\",\n", - " \"version\": \"23.6.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cupy\",\n", - " \"version\": \"12.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"curl\",\n", - " \"version\": \"7.81.0-1ubuntu1.15\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"cycler\",\n", - " \"version\": \"0.12.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"cytoolz\",\n", - " \"version\": \"0.12.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"dash\",\n", - " \"version\": \"0.5.11+git20210903+057cd650a4ed-3build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"dask\",\n", - " \"version\": \"2023.3.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"dask-cuda\",\n", - " \"version\": \"23.6.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"dask-cudf\",\n", - " \"version\": \"23.6.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"databricks-cli\",\n", - " \"version\": \"0.18.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"dataclasses\",\n", - " \"version\": \"0.8\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"datacompy\",\n", - " \"version\": \"0.8.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"debconf\",\n", - " \"version\": \"1.5.79ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"debianutils\",\n", - " \"version\": \"5.5-1ubuntu2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"diffutils\",\n", - " \"version\": \"1:3.8-0ubuntu2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"dill\",\n", - " \"version\": \"0.3.7\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"dirmngr\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"distributed\",\n", - " \"version\": \"2023.3.2.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"docker\",\n", - " \"version\": \"5.0.3\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"docker-pycreds\",\n", - " \"version\": \"0.4.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"docutils\",\n", - " \"version\": \"0.20.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"dpkg\",\n", - " \"version\": \"1.21.1ubuntu2.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"dpkg-dev\",\n", - " \"version\": \"1.21.1ubuntu2.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"e2fsprogs\",\n", - " \"version\": \"1.46.5-2ubuntu1.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"entrypoints\",\n", - " \"version\": \"0.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"exceptiongroup\",\n", - " \"version\": \"1.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"fastrlock\",\n", - " \"version\": \"0.8.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"filelock\",\n", - " \"version\": \"3.13.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"findutils\",\n", - " \"version\": \"4.8.0-1ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"fontconfig-config\",\n", - " \"version\": \"2.13.1-4.2ubuntu5\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"fonts-dejavu-core\",\n", - " \"version\": \"2.37-2build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"fonttools\",\n", - " \"version\": \"4.46.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"fsspec\",\n", - " \"version\": \"2023.12.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"g++\",\n", - " \"version\": \"4:11.2.0-1ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"g++-11\",\n", - " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gcc\",\n", - " \"version\": \"4:11.2.0-1ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gcc-11\",\n", - " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gcc-11-base\",\n", - " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gcc-12-base\",\n", - " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gitdb\",\n", - " \"version\": \"4.0.11\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"gmpy2\",\n", - " \"version\": \"2.1.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"gnupg\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gnupg-l10n\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gnupg-utils\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gnupg2\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"googleapis-common-protos\",\n", - " \"version\": \"1.61.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"gpg\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gpg-agent\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gpg-wks-client\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gpg-wks-server\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gpgconf\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gpgsm\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"gpgv\",\n", - " \"version\": \"2.2.27-3ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"greenlet\",\n", - " \"version\": \"3.0.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"grep\",\n", - " \"version\": \"3.7-1build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"grpcio\",\n", - " \"version\": \"1.54.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"grpclib\",\n", - " \"version\": \"0.4.6\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"gunicorn\",\n", - " \"version\": \"21.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"gzip\",\n", - " \"version\": \"1.10-4ubuntu4.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"h2\",\n", - " \"version\": \"4.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"hostname\",\n", - " \"version\": \"3.23ubuntu2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"hpack\",\n", - " \"version\": \"4.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"hyperframe\",\n", - " \"version\": \"6.0.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"idna\",\n", - " \"version\": \"3.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"idna\",\n", - " \"version\": \"3.6\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"imagesize\",\n", - " \"version\": \"1.4.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"importlib-metadata\",\n", - " \"version\": \"7.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"importlib-resources\",\n", - " \"version\": \"6.1.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"init-system-helpers\",\n", - " \"version\": \"1.62\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"itsdangerous\",\n", - " \"version\": \"2.1.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"java-common\",\n", - " \"version\": \"0.72build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"joblib\",\n", - " \"version\": \"1.3.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"jq\",\n", - " \"version\": \"1.6-2.1ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"jrt-fs\",\n", - " \"version\": \"11.0.21\",\n", - " \"path\": null,\n", - " \"system\": \"java-archive\"\n", - " },\n", - " {\n", - " \"name\": \"js\",\n", - " \"version\": \"1.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"npm\"\n", - " },\n", - " {\n", - " \"name\": \"json5\",\n", - " \"version\": \"0.9.14\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"jsonpatch\",\n", - " \"version\": \"1.32\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"jsonpointer\",\n", - " \"version\": \"2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"jsonschema\",\n", - " \"version\": \"4.20.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"jsonschema-specifications\",\n", - " \"version\": \"2023.11.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"jupyter_client\",\n", - " \"version\": \"8.6.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"jupyter_core\",\n", - " \"version\": \"5.5.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"kiwisolver\",\n", - " \"version\": \"1.4.5\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"libacl1\",\n", - " \"version\": \"2.3.1-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libapt-pkg6.0\",\n", - " \"version\": \"2.4.11\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libarchive-c\",\n", - " \"version\": \"5.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"libasan6\",\n", - " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libasound2\",\n", - " \"version\": \"1.2.6.1-1ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libasound2-data\",\n", - " \"version\": \"1.2.6.1-1ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libassuan0\",\n", - " \"version\": \"2.5.5-1build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libatomic1\",\n", - " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libattr1\",\n", - " \"version\": \"1:2.5.1-1build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libaudit-common\",\n", - " \"version\": \"1:3.0.7-1build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libaudit1\",\n", - " \"version\": \"1:3.0.7-1build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libavahi-client3\",\n", - " \"version\": \"0.8-5ubuntu5.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libavahi-common-data\",\n", - " \"version\": \"0.8-5ubuntu5.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libavahi-common3\",\n", - " \"version\": \"0.8-5ubuntu5.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libbinutils\",\n", - " \"version\": \"2.38-4ubuntu2.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libblkid1\",\n", - " \"version\": \"2.37.2-4ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libbrotli1\",\n", - " \"version\": \"1.0.9-2build6\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libbsd0\",\n", - " \"version\": \"0.11.5-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libbz2-1.0\",\n", - " \"version\": \"1.0.8-5build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libc-bin\",\n", - " \"version\": \"2.35-0ubuntu3.5\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libc-dev-bin\",\n", - " \"version\": \"2.35-0ubuntu3.5\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libc6\",\n", - " \"version\": \"2.35-0ubuntu3.5\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libc6-dev\",\n", - " \"version\": \"2.35-0ubuntu3.5\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcap-ng0\",\n", - " \"version\": \"0.7.9-2.2build3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcap2\",\n", - " \"version\": \"1:2.44-1ubuntu0.22.04.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcbor0.8\",\n", - " \"version\": \"0.8.0-2ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcc1-0\",\n", - " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcom-err2\",\n", - " \"version\": \"1.46.5-2ubuntu1.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcrypt-dev\",\n", - " \"version\": \"1:4.4.27-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcrypt1\",\n", - " \"version\": \"1:4.4.27-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libctf-nobfd0\",\n", - " \"version\": \"2.38-4ubuntu2.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libctf0\",\n", - " \"version\": \"2.38-4ubuntu2.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcublas-11-8\",\n", - " \"version\": \"11.11.3.6-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcufft-11-8\",\n", - " \"version\": \"10.9.0.58-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcups2\",\n", - " \"version\": \"2.4.1op1-1ubuntu4.7\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcurand-11-8\",\n", - " \"version\": \"10.3.0.86-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcurl4\",\n", - " \"version\": \"7.81.0-1ubuntu1.15\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libcusolver-11-8\",\n", - " \"version\": \"11.4.1.48-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libdb5.3\",\n", - " \"version\": \"5.3.28+dfsg1-0.8ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libdbus-1-3\",\n", - " \"version\": \"1.12.20-2ubuntu4.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libdebconfclient0\",\n", - " \"version\": \"0.261ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libdpkg-perl\",\n", - " \"version\": \"1.21.1ubuntu2.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libedit2\",\n", - " \"version\": \"3.1-20210910-1build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libexpat1\",\n", - " \"version\": \"2.4.7-1ubuntu0.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libext2fs2\",\n", - " \"version\": \"1.46.5-2ubuntu1.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libffi8\",\n", - " \"version\": \"3.4.2-4\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libfido2-1\",\n", - " \"version\": \"1.10.0-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libfontconfig1\",\n", - " \"version\": \"2.13.1-4.2ubuntu5\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libfreetype6\",\n", - " \"version\": \"2.11.1+dfsg-1ubuntu0.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgcc-11-dev\",\n", - " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgcc-s1\",\n", - " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgcrypt20\",\n", - " \"version\": \"1.9.4-3ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgdbm-compat4\",\n", - " \"version\": \"1.23-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgdbm6\",\n", - " \"version\": \"1.23-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libglib2.0-0\",\n", - " \"version\": \"2.72.4-0ubuntu2.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgmp10\",\n", - " \"version\": \"2:6.2.1+dfsg-3ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgnutls30\",\n", - " \"version\": \"3.7.3-4ubuntu1.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgomp1\",\n", - " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgpg-error0\",\n", - " \"version\": \"1.43-3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgraphite2-3\",\n", - " \"version\": \"1.3.14-1build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libgssapi-krb5-2\",\n", - " \"version\": \"1.19.2-2ubuntu0.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libharfbuzz0b\",\n", - " \"version\": \"2.7.4-1ubuntu3.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libhogweed6\",\n", - " \"version\": \"3.7.3-1build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libidn2-0\",\n", - " \"version\": \"2.3.2-2build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libisl23\",\n", - " \"version\": \"0.24-2build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libitm1\",\n", - " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libjpeg-turbo8\",\n", - " \"version\": \"2.1.2-0ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libjpeg8\",\n", - " \"version\": \"8c-2ubuntu10\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libjq1\",\n", - " \"version\": \"1.6-2.1ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libk5crypto3\",\n", - " \"version\": \"1.19.2-2ubuntu0.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libkeyutils1\",\n", - " \"version\": \"1.6.1-2ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libkrb5-3\",\n", - " \"version\": \"1.19.2-2ubuntu0.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libkrb5support0\",\n", - " \"version\": \"1.19.2-2ubuntu0.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libksba8\",\n", - " \"version\": \"1.6.0-2ubuntu0.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"liblcms2-2\",\n", - " \"version\": \"2.12~rc1-2build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libldap-2.5-0\",\n", - " \"version\": \"2.5.16+dfsg-0ubuntu0.22.04.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"liblsan0\",\n", - " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"liblz4-1\",\n", - " \"version\": \"1.9.3-2build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"liblzma5\",\n", - " \"version\": \"5.2.5-2ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libmambapy\",\n", - " \"version\": \"1.5.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"libmd0\",\n", - " \"version\": \"1.0.4-1build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libmount1\",\n", - " \"version\": \"2.37.2-4ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libmpc3\",\n", - " \"version\": \"1.2.1-2build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libmpfr6\",\n", - " \"version\": \"4.1.0-3build3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libncurses6\",\n", - " \"version\": \"6.3-2ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libncursesw6\",\n", - " \"version\": \"6.3-2ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libnettle8\",\n", - " \"version\": \"3.7.3-1build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libnghttp2-14\",\n", - " \"version\": \"1.43.0-1ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libnpth0\",\n", - " \"version\": \"1.6-3build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libnsl-dev\",\n", - " \"version\": \"1.3.0-2build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libnsl2\",\n", - " \"version\": \"1.3.0-2build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libnspr4\",\n", - " \"version\": \"2:4.32-3build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libnss3\",\n", - " \"version\": \"2:3.68.2-0ubuntu1.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libnuma1\",\n", - " \"version\": \"2.0.14-3ubuntu2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libonig5\",\n", - " \"version\": \"6.9.7.1-2build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libp11-kit0\",\n", - " \"version\": \"0.24.0-6build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libpam-modules\",\n", - " \"version\": \"1.4.0-11ubuntu2.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libpam-modules-bin\",\n", - " \"version\": \"1.4.0-11ubuntu2.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libpam-runtime\",\n", - " \"version\": \"1.4.0-11ubuntu2.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libpam0g\",\n", - " \"version\": \"1.4.0-11ubuntu2.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libpcre2-8-0\",\n", - " \"version\": \"10.39-3ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libpcre3\",\n", - " \"version\": \"2:8.39-13ubuntu0.22.04.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libpcsclite1\",\n", - " \"version\": \"1.9.5-3ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libperl5.34\",\n", - " \"version\": \"5.34.0-3ubuntu1.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libpng16-16\",\n", - " \"version\": \"1.6.37-3build5\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libprocps8\",\n", - " \"version\": \"2:3.3.17-6ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libpsl5\",\n", - " \"version\": \"0.21.0-1.2build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libquadmath0\",\n", - " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libreadline8\",\n", - " \"version\": \"8.1.2-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"librtmp1\",\n", - " \"version\": \"2.4+20151223.gitfa8646d.1-2build4\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libsasl2-2\",\n", - " \"version\": \"2.1.27+dfsg2-3ubuntu1.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libsasl2-modules-db\",\n", - " \"version\": \"2.1.27+dfsg2-3ubuntu1.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libseccomp2\",\n", - " \"version\": \"2.5.3-2ubuntu2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libselinux1\",\n", - " \"version\": \"3.3-1build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libsemanage-common\",\n", - " \"version\": \"3.3-1build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libsemanage2\",\n", - " \"version\": \"3.3-1build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libsepol2\",\n", - " \"version\": \"3.3-1build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libsmartcols1\",\n", - " \"version\": \"2.37.2-4ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libsqlite3-0\",\n", - " \"version\": \"3.37.2-2ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libss2\",\n", - " \"version\": \"1.46.5-2ubuntu1.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libssh-4\",\n", - " \"version\": \"0.9.6-2ubuntu0.22.04.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libssl3\",\n", - " \"version\": \"3.0.2-0ubuntu1.12\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libstdc++-11-dev\",\n", - " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libstdc++6\",\n", - " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libsystemd0\",\n", - " \"version\": \"249.11-0ubuntu3.11\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libtasn1-6\",\n", - " \"version\": \"4.18.0-4build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libtinfo6\",\n", - " \"version\": \"6.3-2ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libtirpc-common\",\n", - " \"version\": \"1.3.2-2ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libtirpc-dev\",\n", - " \"version\": \"1.3.2-2ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libtirpc3\",\n", - " \"version\": \"1.3.2-2ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libtsan0\",\n", - " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libubsan1\",\n", - " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libudev1\",\n", - " \"version\": \"249.11-0ubuntu3.11\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libunistring2\",\n", - " \"version\": \"1.0-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libuuid1\",\n", - " \"version\": \"2.37.2-4ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libxxhash0\",\n", - " \"version\": \"0.8.1-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"libzstd1\",\n", - " \"version\": \"1.4.8+dfsg-3build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"linux-libc-dev\",\n", - " \"version\": \"5.15.0-89.99\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"llvmlite\",\n", - " \"version\": \"0.41.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"locket\",\n", - " \"version\": \"1.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"login\",\n", - " \"version\": \"1:4.8.1-2ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"logsave\",\n", - " \"version\": \"1.46.5-2ubuntu1.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"lsb-base\",\n", - " \"version\": \"11.1.0ubuntu4\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"lto-disabled-list\",\n", - " \"version\": \"24\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"lz4\",\n", - " \"version\": \"4.3.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"make\",\n", - " \"version\": \"4.3-4.1build1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"mamba\",\n", - " \"version\": \"1.5.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"markdown-it-py\",\n", - " \"version\": \"3.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"matplotlib\",\n", - " \"version\": \"3.8.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"mawk\",\n", - " \"version\": \"1.3.4.20200120-3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"mdurl\",\n", - " \"version\": \"0.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"menuinst\",\n", - " \"version\": \"2.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"merlin-core\",\n", - " \"version\": \"23.6.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"merlin-dataloader\",\n", - " \"version\": \"23.6.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"mlflow\",\n", - " \"version\": \"2.9.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"more-itertools\",\n", - " \"version\": \"10.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"morpheus\",\n", - " \"version\": \"23.11.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"mount\",\n", - " \"version\": \"2.37.2-4ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"mpmath\",\n", - " \"version\": \"1.3.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"mrc\",\n", - " \"version\": \"23.11.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"msgpack\",\n", - " \"version\": \"1.0.7\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"multidict\",\n", - " \"version\": \"6.0.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"munkres\",\n", - " \"version\": \"1.1.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"nb-conda-kernels\",\n", - " \"version\": \"2.3.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"ncurses-base\",\n", - " \"version\": \"6.3-2ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"ncurses-bin\",\n", - " \"version\": \"6.3-2ubuntu0.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"networkx\",\n", - " \"version\": \"3.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"npy-append-array\",\n", - " \"version\": \"0.9.16\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"numba\",\n", - " \"version\": \"0.58.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"numpy\",\n", - " \"version\": \"1.26.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"numpydoc\",\n", - " \"version\": \"1.4.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"nvtabular\",\n", - " \"version\": \"23.6.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"nvtx\",\n", - " \"version\": \"0.2.8\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"oauthlib\",\n", - " \"version\": \"3.2.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"openjdk-11-jre-headless\",\n", - " \"version\": \"11.0.21+9-0ubuntu1~22.04\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"openssh-client\",\n", - " \"version\": \"1:8.9p1-3ubuntu0.4\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"openssl\",\n", - " \"version\": \"3.0.2-0ubuntu1.12\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"openssl\",\n", - " \"version\": \"3.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"binary\"\n", - " },\n", - " {\n", - " \"name\": \"ordered-set\",\n", - " \"version\": \"4.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"packaging\",\n", - " \"version\": \"23.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"packaging\",\n", - " \"version\": \"23.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pandas\",\n", - " \"version\": \"1.3.5\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"partd\",\n", - " \"version\": \"1.4.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"passwd\",\n", - " \"version\": \"1:4.8.1-2ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"patch\",\n", - " \"version\": \"2.7.6-7build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"perl\",\n", - " \"version\": \"5.34.0-3ubuntu1.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"perl-base\",\n", - " \"version\": \"5.34.0-3ubuntu1.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"perl-modules-5.34\",\n", - " \"version\": \"5.34.0-3ubuntu1.3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"pinentry-curses\",\n", - " \"version\": \"1.1.1-1build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"pip\",\n", - " \"version\": \"23.3.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pkg-config\",\n", - " \"version\": \"0.29.2-1ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"pkginfo\",\n", - " \"version\": \"1.9.6\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pkgutil_resolve_name\",\n", - " \"version\": \"1.3.10\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"platformdirs\",\n", - " \"version\": \"4.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pluggy\",\n", - " \"version\": \"1.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pluggy\",\n", - " \"version\": \"1.3.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"procps\",\n", - " \"version\": \"2:3.3.17-6ubuntu2.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"prometheus-client\",\n", - " \"version\": \"0.19.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"prometheus-flask-exporter\",\n", - " \"version\": \"0.23.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"prompt-toolkit\",\n", - " \"version\": \"3.0.41\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"protobuf\",\n", - " \"version\": \"4.21.12\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"psutil\",\n", - " \"version\": \"5.9.5\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"ptxcompiler\",\n", - " \"version\": \"0.8.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pyOpenSSL\",\n", - " \"version\": \"23.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pyOpenSSL\",\n", - " \"version\": \"23.3.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pyarrow\",\n", - " \"version\": \"11.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pyarrow-hotfix\",\n", - " \"version\": \"0.6\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pycosat\",\n", - " \"version\": \"0.6.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pycparser\",\n", - " \"version\": \"2.21\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pynvml\",\n", - " \"version\": \"11.4.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pyparsing\",\n", - " \"version\": \"3.1.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"python\",\n", - " \"version\": \"3.10.12\",\n", - " \"path\": null,\n", - " \"system\": \"binary\"\n", - " },\n", - " {\n", - " \"name\": \"python\",\n", - " \"version\": \"3.10.13\",\n", - " \"path\": null,\n", - " \"system\": \"binary\"\n", - " },\n", - " {\n", - " \"name\": \"python-dateutil\",\n", - " \"version\": \"2.8.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"python-rapidjson\",\n", - " \"version\": \"1.13\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pytz\",\n", - " \"version\": \"2023.3.post1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"pyzmq\",\n", - " \"version\": \"25.1.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"querystring-parser\",\n", - " \"version\": \"1.2.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"readline-common\",\n", - " \"version\": \"8.1.2-1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"referencing\",\n", - " \"version\": \"0.31.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"requests\",\n", - " \"version\": \"2.31.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"requests-cache\",\n", - " \"version\": \"1.1.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"rich\",\n", - " \"version\": \"13.7.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"rmm\",\n", - " \"version\": \"23.6.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"rpcsvc-proto\",\n", - " \"version\": \"1.4.2-0ubuntu6\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"rpds-py\",\n", - " \"version\": \"0.13.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"ruamel.yaml\",\n", - " \"version\": \"0.17.32\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"ruamel.yaml.clib\",\n", - " \"version\": \"0.2.7\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"scikit-learn\",\n", - " \"version\": \"1.2.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sed\",\n", - " \"version\": \"4.8-1ubuntu2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"sensible-utils\",\n", - " \"version\": \"0.0.17\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"setuptools\",\n", - " \"version\": \"59.8.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"setuptools\",\n", - " \"version\": \"68.1.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"six\",\n", - " \"version\": \"1.16.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"smmap\",\n", - " \"version\": \"5.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sniffio\",\n", - " \"version\": \"1.3.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"snowballstemmer\",\n", - " \"version\": \"2.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sortedcontainers\",\n", - " \"version\": \"2.4.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"soupsieve\",\n", - " \"version\": \"2.5\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sphinxcontrib-applehelp\",\n", - " \"version\": \"1.0.7\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sphinxcontrib-devhelp\",\n", - " \"version\": \"1.0.5\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sphinxcontrib-htmlhelp\",\n", - " \"version\": \"2.0.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sphinxcontrib-jsmath\",\n", - " \"version\": \"1.0.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sphinxcontrib-qthelp\",\n", - " \"version\": \"1.0.6\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sphinxcontrib-serializinghtml\",\n", - " \"version\": \"1.1.9\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sqlparse\",\n", - " \"version\": \"0.4.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"stringcase\",\n", - " \"version\": \"1.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sympy\",\n", - " \"version\": \"1.12\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"sysvinit-utils\",\n", - " \"version\": \"3.01-1ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"tabulate\",\n", - " \"version\": \"0.9.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"tar\",\n", - " \"version\": \"1.34+dfsg-1ubuntu0.1.22.04.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"tblib\",\n", - " \"version\": \"2.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"tensorflow-metadata\",\n", - " \"version\": \"1.13.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"threadpoolctl\",\n", - " \"version\": \"3.2.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"tomli\",\n", - " \"version\": \"2.0.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"toolz\",\n", - " \"version\": \"0.12.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"torch\",\n", - " \"version\": \"2.0.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"tornado\",\n", - " \"version\": \"6.3.3\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"tqdm\",\n", - " \"version\": \"4.66.1\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"traitlets\",\n", - " \"version\": \"5.14.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"triton\",\n", - " \"version\": \"2.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"tritonclient\",\n", - " \"version\": \"2.26.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"typing-utils\",\n", - " \"version\": \"0.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"typing_extensions\",\n", - " \"version\": \"4.8.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"ubuntu-keyring\",\n", - " \"version\": \"2021.03.26\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"ucf\",\n", - " \"version\": \"3.0043\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"ujson\",\n", - " \"version\": \"5.8.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"unicodedata2\",\n", - " \"version\": \"15.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"unzip\",\n", - " \"version\": \"6.0-26ubuntu3.1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"url-normalize\",\n", - " \"version\": \"1.4.3\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"urllib3\",\n", - " \"version\": \"2.0.4\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"urllib3\",\n", - " \"version\": \"2.1.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"usrmerge\",\n", - " \"version\": \"25ubuntu2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"util-linux\",\n", - " \"version\": \"2.37.2-4ubuntu3\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"watchdog\",\n", - " \"version\": \"2.1.9\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"watchgod\",\n", - " \"version\": \"0.8.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"wcwidth\",\n", - " \"version\": \"0.2.12\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"websocket-client\",\n", - " \"version\": \"1.7.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"websockets\",\n", - " \"version\": \"12.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"wget\",\n", - " \"version\": \"1.21.2-2ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"wheel\",\n", - " \"version\": \"0.41.2\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"wheel\",\n", - " \"version\": \"0.42.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"xz-utils\",\n", - " \"version\": \"5.2.5-2ubuntu1\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"zict\",\n", - " \"version\": \"3.0.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"zip\",\n", - " \"version\": \"3.0-12build2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"zipp\",\n", - " \"version\": \"3.17.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " },\n", - " {\n", - " \"name\": \"zlib1g\",\n", - " \"version\": \"1:1.2.11.dfsg-2ubuntu9.2\",\n", - " \"path\": null,\n", - " \"system\": \"deb\"\n", - " },\n", - " {\n", - " \"name\": \"zstandard\",\n", - " \"version\": \"0.19.0\",\n", - " \"path\": null,\n", - " \"system\": \"python\"\n", - " }\n", - " ]\n", - " },\n", - " \"vulnerable_dependencies\": [\n", - " {\n", - " \"vuln_id\": \"CVE-2024-21762\",\n", - " \"vuln_package_intel_sources\": [\n", - " \"nvd\"\n", - " ],\n", - " \"vulnerable_sbom_packages\": []\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-hh8p-p8mp-gqhm\",\n", - " \"vuln_package_intel_sources\": [\n", - " \"ghsa\",\n", - " \"nvd\"\n", - " ],\n", - " \"vulnerable_sbom_packages\": [\n", - " {\n", - " \"name\": \"mlflow\",\n", - " \"version\": \"2.9.1\",\n", - " \"vulnerable_dependency_package\": {\n", - " \"system\": \"PYPI\",\n", - " \"name\": \"mlflow\",\n", - " \"version\": \"2.9.1\",\n", - " \"relation\": \"SELF\"\n", - " }\n", - " },\n", - " {\n", - " \"name\": \"mlflow\",\n", - " \"version\": \"2.9.1\",\n", - " \"vulnerable_dependency_package\": {\n", - " \"system\": \"PYPI\",\n", - " \"name\": \"mlflow\",\n", - " \"version\": \"2.9.1\",\n", - " \"relation\": \"SELF\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-3f63-hfp8-52jq\",\n", - " \"vuln_package_intel_sources\": [\n", - " \"ghsa\",\n", - " \"nvd\",\n", - " \"ubuntu\",\n", - " \"rhsa\"\n", - " ],\n", - " \"vulnerable_sbom_packages\": [\n", - " {\n", - " \"name\": \"pillow\",\n", - " \"version\": \"10.1.0\",\n", - " \"vulnerable_dependency_package\": {\n", - " \"system\": \"PYPI\",\n", - " \"name\": \"pillow\",\n", - " \"version\": \"10.1.0\",\n", - " \"relation\": \"SELF\"\n", - " }\n", - " },\n", - " {\n", - " \"name\": \"pillow\",\n", - " \"version\": \"10.1.0\",\n", - " \"vulnerable_dependency_package\": {\n", - " \"system\": \"PYPI\",\n", - " \"name\": \"pillow\",\n", - " \"version\": \"10.1.0\",\n", - " \"relation\": \"SELF\"\n", - " }\n", - " }\n", - " ]\n", - " }\n", - " ]\n", - " },\n", - " \"output\": [\n", - " {\n", - " \"vuln_id\": \"CVE-2024-21762\",\n", - " \"checklist\": [\n", - " {\n", - " \"input\": \"Agent bypassed: no vulnerable packages detected. Checklist not generated.\",\n", - " \"response\": \"The VulnerableDependencyChecker did not find any vulnerable packages or dependencies in the SBOM and so the agent was bypassed.\",\n", - " \"intermediate_steps\": null\n", - " }\n", - " ],\n", - " \"summary\": \"The VulnerableDependencyChecker did not find any vulnerable packages or dependencies in the SBOM.\",\n", - " \"justification\": {\n", - " \"label\": \"false_positive\",\n", - " \"reason\": \"No vulnerable packages or dependencies were detected in the SBOM.\",\n", - " \"status\": \"FALSE\"\n", - " }\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-hh8p-p8mp-gqhm\",\n", - " \"checklist\": [\n", - " {\n", - " \"input\": \"Verify Usage of Vulnerable Path Construction: Review the application code within the container image to check if it constructs pathnames using external input. Specifically, look for instances where user input is used to build file paths, which could be exploited to traverse the file system.\",\n", - " \"response\": \"The application code within the container image does construct pathnames using external input, which could potentially be exploited to traverse the file system. Specifically, the code patterns found in the `path.join()` and string concatenation functions may be vulnerable to path traversal attacks. However, without further information about the container image and its configuration, it is difficult to determine the exact risk and potential impact of these vulnerabilities.\",\n", - " \"intermediate_steps\": null\n", - " },\n", - " {\n", - " \"input\": \"Assess Input Validation and Sanitization: Evaluate the robustness of input validation and sanitization practices within the application, particularly for inputs that are used in constructing file paths. Check if the application properly neutralizes '\\\\..\\\\filename' sequences that could resolve to locations outside the intended directory.\",\n", - " \"response\": \"The application does not properly neutralize '\\\\..\\\\filename' sequences that could resolve to locations outside the intended directory. The custom class `MorpheusRelativePath` and the function `get_package_relative_file` do not validate and sanitize file paths constructed from user input, making the application vulnerable to directory traversal attacks. To fix this, the code should use `os.path.normpath` or `os.path.abspath` to normalize the path and remove any directory traversal sequences, and also validate the resulting path to ensure it's within the intended directory.\",\n", - " \"intermediate_steps\": null\n", - " },\n", - " {\n", - " \"input\": \"Inspect File System Access Controls: Review the file system access controls and permissions within the container to determine if an attacker could exploit the vulnerability to access sensitive files or directories. Consider the potential impact of an attacker gaining access to data and models information.\",\n", - " \"response\": \"The container image likely has restricted access controls and permissions to prevent unauthorized access to sensitive files and directories, but the exact specifics couldn't be determined.\",\n", - " \"intermediate_steps\": null\n", - " }\n", - " ],\n", - " \"summary\": \"Based on the provided Checklist and Findings, the CVE is exploitable. Specifically, the application code constructs pathnames using external input, which could be exploited to traverse the file system (Checklist Item 1). Moreover, the application does not properly neutralize directory traversal sequences, making it vulnerable to directory traversal attacks (Checklist Item 2). These findings indicate that the CVE is exploitable, allowing an attacker to potentially access sensitive files or directories.\",\n", - " \"justification\": {\n", - " \"label\": \"vulnerable\",\n", - " \"reason\": \"The analysis indicates that the CVE is exploitable due to the application's construction of pathnames using external input and failure to neutralize directory traversal sequences, making it vulnerable to directory traversal attacks.\",\n", - " \"status\": \"TRUE\"\n", - " }\n", - " },\n", - " {\n", - " \"vuln_id\": \"GHSA-3f63-hfp8-52jq\",\n", - " \"checklist\": [\n", - " {\n", - " \"input\": \"Verify Usage of PIL.ImageMath.eval: Check if the application code within the container image uses the PIL.ImageMath.eval function, specifically with the environment parameter. This function is the target of the vulnerability, and its usage with untrusted input could lead to arbitrary code execution.\",\n", - " \"response\": \"Based on the observations from the Container Image Code QA System and the Container Image Developer Guide QA System, it appears that the application code within the container image does not use the PIL.ImageMath.eval function with the environment parameter, and there is no indication of any dependencies or libraries that might be using it. Therefore, the usage of PIL.ImageMath.eval with untrusted input, which could lead to arbitrary code execution, is unlikely to be a concern for this container image.\",\n", - " \"intermediate_steps\": null\n", - " },\n", - " {\n", - " \"input\": \"Assess Input Handling for PIL.ImageMath.eval: If PIL.ImageMath.eval is used, evaluate how the application handles input data passed to this function, especially focusing on the environment parameter. Ensure that inputs are properly sanitized and validated to prevent potential code injection attacks.\",\n", - " \"response\": \"The container image does not use PIL.ImageMath.eval, but it uses other libraries such as CuPy and Morpheus for mathematical operations. While the provided context does not provide enough information to determine if the inputs are properly sanitized and validated, it is essential to handle input data carefully to prevent potential security risks, such as information leakage across user boundaries. Therefore, it is recommended to implement proper input data sanitization and validation mechanisms when using these libraries.\",\n", - " \"intermediate_steps\": null\n", - " },\n", - " {\n", - " \"input\": \"Review Environment Variable Management: Since the vulnerability involves manipulation of the environment parameter, assess how environment variables are managed within the application and the container. Ensure that sensitive data is not exposed through environment variables and that variables are properly secured to prevent unauthorized access or manipulation.\",\n", - " \"response\": \"The container image's management of environment variables is not clearly documented, and it is unclear if sensitive data is exposed through environment variables or if variables are properly secured to prevent unauthorized access or manipulation. Further investigation is needed to fully assess the security of environment variable management within the container image.\",\n", - " \"intermediate_steps\": null\n", - " }\n", - " ],\n", - " \"summary\": \"Based on the provided Checklist and Findings, the CVE is not exploitable. The investigation found that the application code within the container image does not use the PIL.ImageMath.eval function with the environment parameter, which is the target of the vulnerability. This suggests that the risk of arbitrary code execution through this specific vulnerability is unlikely.\",\n", - " \"justification\": {\n", - " \"label\": \"code_not_reachable\",\n", - " \"reason\": \"The vulnerable code is not executed during runtime because the application code within the container image does not use the PIL.ImageMath.eval function with the environment parameter, which is the target of the vulnerability.\",\n", - " \"status\": \"FALSE\"\n", - " }\n", - " }\n", - " ]\n", - "}\n" - ] - } - ], + "outputs": [], "source": [ "print(json.dumps(first_result, indent=2))" ] @@ -4681,7 +442,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": { "tags": [] }, @@ -4702,94 +463,11 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": { "tags": [] }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
vuln_idchecklistsummarylabelreasonaffected
0CVE-2024-21762[{'input': 'Agent bypassed: no vulnerable pack...The VulnerableDependencyChecker did not find a...false_positiveNo vulnerable packages or dependencies were de...FALSE
1GHSA-hh8p-p8mp-gqhm[{'input': 'Verify Usage of Vulnerable Path Co...Based on the provided Checklist and Findings, ...vulnerableThe analysis indicates that the CVE is exploit...TRUE
2GHSA-3f63-hfp8-52jq[{'input': 'Verify Usage of PIL.ImageMath.eval...Based on the provided Checklist and Findings, ...code_not_reachableThe vulnerable code is not executed during run...FALSE
\n", - "
" - ], - "text/plain": [ - " vuln_id checklist \\\n", - "0 CVE-2024-21762 [{'input': 'Agent bypassed: no vulnerable pack... \n", - "1 GHSA-hh8p-p8mp-gqhm [{'input': 'Verify Usage of Vulnerable Path Co... \n", - "2 GHSA-3f63-hfp8-52jq [{'input': 'Verify Usage of PIL.ImageMath.eval... \n", - "\n", - " summary label \\\n", - "0 The VulnerableDependencyChecker did not find a... false_positive \n", - "1 Based on the provided Checklist and Findings, ... vulnerable \n", - "2 Based on the provided Checklist and Findings, ... code_not_reachable \n", - "\n", - " reason affected \n", - "0 No vulnerable packages or dependencies were de... FALSE \n", - "1 The analysis indicates that the CVE is exploit... TRUE \n", - "2 The vulnerable code is not executed during run... FALSE " - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "process_results(output_file_path)" ] @@ -4822,7 +500,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.10" + "version": "3.13.7" } }, "nbformat": 4, diff --git a/src/exploit_iq_commons/__init__.py b/src/exploit_iq_commons/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/exploit_iq_commons/data/__init__.py b/src/exploit_iq_commons/data/__init__.py new file mode 100644 index 000000000..cf7c586a5 --- /dev/null +++ b/src/exploit_iq_commons/data/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/exploit_iq_commons/data/hardening_kb/__init__.py b/src/exploit_iq_commons/data/hardening_kb/__init__.py new file mode 100644 index 000000000..cf7c586a5 --- /dev/null +++ b/src/exploit_iq_commons/data/hardening_kb/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/exploit_iq_commons/data/hardening_kb/hardening_kb.json b/src/exploit_iq_commons/data/hardening_kb/hardening_kb.json new file mode 100644 index 000000000..92ab9a8f9 --- /dev/null +++ b/src/exploit_iq_commons/data/hardening_kb/hardening_kb.json @@ -0,0 +1,395 @@ +{ + "kb_version": "1.1", + "last_updated": "2026-05-09", + "flag_type_definitions": { + "warning": "Compile-time warnings only. Does not add runtime protection. Use for 'Best Practices' audits, NOT for mitigation claims.", + "runtime": "Actual runtime protection or detection. Valid for 'Mitigated' status if maps to the specific CWE.", + "optimization": "Changes compiler optimization behavior but does not add runtime detection/prevention. Not valid for mitigation.", + "linker": "Linker-level hardening that affects runtime binary layout/behavior. Valid for mitigation if maps to CWE.", + "architecture": "Platform-specific build flag. Valid for mitigation ONLY if CVE advisory states the architecture is not affected." + }, + "mappings": [ + { + "flag": "-Wall -Wextra", + "flag_type": "warning", + "description": "Enable warnings for constructs often associated with defects.", + "vulnerability_category": "Defensive Coding", + "cwe_ids": [ + "CWE-563", + "CWE-457", + "CWE-480" + ], + "requires": {} + }, + { + "flag": "-Wformat -Wformat=2", + "flag_type": "warning", + "description": "Enable additional format function warnings.", + "vulnerability_category": "Input Validation", + "cwe_ids": [ + "CWE-134" + ], + "requires": {} + }, + { + "flag": "-Wconversion -Wsign-conversion", + "flag_type": "warning", + "description": "Enable implicit conversion warnings.", + "vulnerability_category": "Arithmetic Safety", + "cwe_ids": [ + "CWE-190", + "CWE-681" + ], + "requires": {} + }, + { + "flag": "-Wtrampolines", + "flag_type": "warning", + "description": "Enable warnings about trampolines that require executable stacks.", + "vulnerability_category": "Control Flow Integrity", + "cwe_ids": [ + "CWE-693" + ], + "requires": {} + }, + { + "flag": "-Wimplicit-fallthrough", + "flag_type": "warning", + "description": "Warn when a switch case falls through.", + "vulnerability_category": "Defensive Coding", + "cwe_ids": [ + "CWE-484" + ], + "requires": {} + }, + { + "flag": "-Wbidi-chars=any", + "flag_type": "warning", + "description": "Enable warnings for possibly misleading Unicode bidirectional control characters.", + "vulnerability_category": "Code Integrity", + "cwe_ids": [ + "CWE-1301" + ], + "requires": {} + }, + { + "flag": "-Werror", + "flag_type": "warning", + "description": "Treat all or selected compiler warnings as errors.", + "vulnerability_category": "Policy Enforcement", + "cwe_ids": [ + "N/A" + ], + "requires": {} + }, + { + "flag": "-Werror=format-security", + "flag_type": "warning", + "description": "Treat format strings that are not string literals and used without arguments as errors.", + "vulnerability_category": "Input Validation", + "cwe_ids": [ + "CWE-134" + ], + "requires": {} + }, + { + "flag": "-Werror=implicit -Werror=incompatible-pointer-types -Werror=int-conversion", + "flag_type": "warning", + "description": "Treat obsolete C constructs as errors.", + "vulnerability_category": "Type Safety", + "cwe_ids": [ + "CWE-704", + "CWE-843" + ], + "requires": {} + }, + { + "flag": "-D_FORTIFY_SOURCE=3", + "flag_type": "runtime", + "description": "Fortify sources with compile- and run-time checks for unsafe libc usage and buffer overflows.", + "vulnerability_category": "Memory Safety", + "cwe_ids": [ + "CWE-119", + "CWE-120", + "CWE-121", + "CWE-122" + ], + "requires": {} + }, + { + "flag": "-D_FORTIFY_SOURCE=2", + "flag_type": "runtime", + "description": "Fortify sources with compile- and run-time checks for unsafe libc usage and buffer overflows (legacy level).", + "vulnerability_category": "Memory Safety", + "cwe_ids": [ + "CWE-119", + "CWE-120", + "CWE-121", + "CWE-122" + ], + "requires": {} + }, + { + "flag": "-D_GLIBCXX_ASSERTIONS", + "flag_type": "runtime", + "description": "Precondition checks for C++ standard library calls.", + "vulnerability_category": "Memory Safety", + "cwe_ids": [ + "CWE-119", + "CWE-125", + "CWE-787" + ], + "requires": {} + }, + { + "flag": "-fstrict-flex-arrays=3", + "flag_type": "runtime", + "description": "Consider a trailing array in a struct as a flexible array if declared as [].", + "vulnerability_category": "Memory Safety", + "cwe_ids": [ + "CWE-119", + "CWE-125", + "CWE-787" + ], + "requires": {} + }, + { + "flag": "-fstack-clash-protection", + "flag_type": "runtime", + "description": "Enable run-time checks for variable-size stack allocation validity.", + "vulnerability_category": "Memory Safety", + "cwe_ids": [ + "CWE-785" + ], + "requires": {} + }, + { + "flag": "-fstack-protector-strong", + "flag_type": "runtime", + "description": "Enable run-time checks for stack-based buffer overflows.", + "vulnerability_category": "Memory Safety", + "cwe_ids": [ + "CWE-121" + ], + "requires": {} + }, + { + "flag": "-fcf-protection=full", + "flag_type": "runtime", + "description": "Enable control-flow protection against return-oriented programming (ROP) and jump-oriented programming (JOP) attacks on x86_64.", + "vulnerability_category": "Control Flow Integrity", + "cwe_ids": [ + "CWE-693" + ], + "requires": {} + }, + { + "flag": "-mbranch-protection=standard", + "flag_type": "runtime", + "description": "Enable branch protection against ROP and JOP attacks on AArch64.", + "vulnerability_category": "Control Flow Integrity", + "cwe_ids": [ + "CWE-693" + ], + "requires": {} + }, + { + "flag": "-ftrapv", + "flag_type": "runtime", + "description": "Generate traps for signed arithmetic overflow on addition, subtraction, multiplication.", + "vulnerability_category": "Arithmetic Safety", + "cwe_ids": [ + "CWE-190", + "CWE-191" + ], + "requires": {} + }, + { + "flag": "-fsanitize=signed-integer-overflow", + "flag_type": "runtime", + "description": "Enable undefined behavior sanitizer for signed integer overflow detection.", + "vulnerability_category": "Arithmetic Safety", + "cwe_ids": [ + "CWE-190", + "CWE-191" + ], + "requires": {} + }, + { + "flag": "-fsanitize=unsigned-integer-overflow", + "flag_type": "runtime", + "description": "Enable undefined behavior sanitizer for unsigned integer overflow detection.", + "vulnerability_category": "Arithmetic Safety", + "cwe_ids": [ + "CWE-190", + "CWE-191" + ], + "requires": {} + }, + { + "flag": "-Wl,-z,nodlopen", + "flag_type": "linker", + "description": "Restrict dlopen(3) calls to shared objects.", + "vulnerability_category": "Policy Enforcement", + "cwe_ids": [ + "CWE-269" + ], + "requires": {} + }, + { + "flag": "-Wl,-z,noexecstack", + "flag_type": "linker", + "description": "Enable data execution prevention by marking stack memory as non-executable.", + "vulnerability_category": "Control Flow Integrity", + "cwe_ids": [ + "CWE-693", + "CWE-94" + ], + "requires": {} + }, + { + "flag": "-Wl,-z,relro -Wl,-z,now", + "flag_type": "linker", + "description": "Mark relocation table entries resolved at load-time as read-only.", + "vulnerability_category": "Code Integrity", + "cwe_ids": [ + "CWE-123" + ], + "requires": {} + }, + { + "flag": "-fPIE -pie", + "flag_type": "linker", + "description": "Build as position-independent executable.", + "vulnerability_category": "Control Flow Integrity", + "cwe_ids": [ + "CWE-693" + ], + "requires": {} + }, + { + "flag": "-fPIC -shared", + "flag_type": "linker", + "description": "Build as position-independent code.", + "vulnerability_category": "Control Flow Integrity", + "cwe_ids": [ + "CWE-693" + ], + "requires": {} + }, + { + "flag": "-fno-delete-null-pointer-checks", + "flag_type": "optimization", + "description": "Force retention of null pointer checks.", + "vulnerability_category": "Memory Safety", + "cwe_ids": [ + "CWE-476" + ], + "requires": {} + }, + { + "flag": "-fno-strict-overflow", + "flag_type": "optimization", + "description": "Do not assume signed integer overflow is undefined behavior. Prevents aggressive optimizations but does NOT add runtime detection.", + "vulnerability_category": "Arithmetic Safety", + "cwe_ids": [ + "CWE-190" + ], + "requires": {} + }, + { + "flag": "-fno-strict-aliasing", + "flag_type": "optimization", + "description": "Do not assume strict aliasing.", + "vulnerability_category": "Memory Safety", + "cwe_ids": [ + "CWE-416" + ], + "requires": {} + }, + { + "flag": "-ftrivial-auto-var-init", + "flag_type": "runtime", + "description": "Initialize automatic variables that lack explicit initializers.", + "vulnerability_category": "Information Leakage", + "cwe_ids": [ + "CWE-457" + ], + "requires": {} + }, + { + "flag": "-fexceptions", + "flag_type": "runtime", + "description": "Enable exception propagation to harden multi-threaded C code.", + "vulnerability_category": "Error Handling", + "cwe_ids": [ + "CWE-391" + ], + "requires": {} + }, + { + "flag": "-fhardened", + "flag_type": "runtime", + "description": "Enable pre-determined set of hardening options in GCC.", + "vulnerability_category": "Full Hardening", + "cwe_ids": [ + "Multi" + ], + "requires": {} + }, + { + "flag": "-Wl,--as-needed -Wl,--no-copy-dt-needed-entries", + "flag_type": "linker", + "description": "Allow linker to omit libraries specified on the command line to link against if they are not used.", + "vulnerability_category": "Supply Chain Safety", + "cwe_ids": [ + "N/A" + ], + "requires": {} + }, + { + "flag": "-fzero-init-padding-bits=all", + "flag_type": "runtime", + "description": "Guarantee zero initialization of padding bits in all automatic variable initializers.", + "vulnerability_category": "Information Leakage", + "cwe_ids": [ + "CWE-200" + ], + "requires": {} + }, + { + "flag": "-m64", + "flag_type": "architecture", + "description": "Compile for 64-bit x86_64 architecture. Many integer overflow vulnerabilities only affect 32-bit systems.", + "vulnerability_category": "Architecture", + "cwe_ids": [ + "CWE-190", + "CWE-680", + "CWE-681" + ], + "requires": { + "advisory_states": "Mitigation valid ONLY if CVE advisory explicitly states 64-bit systems are not affected." + } + }, + { + "flag": "-m32", + "flag_type": "architecture", + "description": "Compile for 32-bit i686 architecture.", + "vulnerability_category": "Architecture", + "cwe_ids": [], + "requires": { + "advisory_states": "Check CVE advisory for 32-bit specific vulnerabilities." + } + }, + { + "flag": "-march=", + "flag_type": "architecture", + "description": "Target specific CPU architecture. May affect vulnerability applicability.", + "vulnerability_category": "Architecture", + "cwe_ids": [], + "requires": { + "advisory_states": "Check CVE advisory for architecture-specific conditions." + } + } + ] +} diff --git a/src/exploit_iq_commons/data/package_repo_mapping.json b/src/exploit_iq_commons/data/package_repo_mapping.json new file mode 100644 index 000000000..511537e56 --- /dev/null +++ b/src/exploit_iq_commons/data/package_repo_mapping.json @@ -0,0 +1,314 @@ +{ + "_meta": { + "description": "Package name to upstream git repository mapping for RPM/C/C++ packages", + "version": "1.0.0", + "last_updated": "2026-06-17", + "usage": "PRIMARY resolution strategy for REPO_RESOLUTION_NODE" + }, + "packages": { + "httpd": "https://github.com/apache/httpd", + "curl": "https://github.com/curl/curl", + "openssl": "https://github.com/openssl/openssl", + "nginx": "https://github.com/nginx/nginx", + "postgresql": "https://github.com/postgres/postgres", + "mysql": "https://github.com/mysql/mysql-server", + "sqlite": "https://github.com/sqlite/sqlite", + "redis": "https://github.com/redis/redis", + "memcached": "https://github.com/memcached/memcached", + "openssh": "https://github.com/openssh/openssh-portable", + "bind": "https://gitlab.isc.org/isc-projects/bind9", + "linux": "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git", + "glibc": "https://sourceware.org/git/glibc.git", + "gcc": "https://gcc.gnu.org/git/gcc.git", + "gnutls": "https://gitlab.com/gnutls/gnutls", + "libxml2": "https://gitlab.gnome.org/GNOME/libxml2", + "libxslt": "https://gitlab.gnome.org/GNOME/libxslt", + "python": "https://github.com/python/cpython", + "php": "https://github.com/php/php-src", + "ruby": "https://github.com/ruby/ruby", + "nodejs": "https://github.com/nodejs/node", + "golang": "https://github.com/golang/go", + "rust": "https://github.com/rust-lang/rust", + "git": "https://github.com/git/git", + "vim": "https://github.com/vim/vim", + "tmux": "https://github.com/tmux/tmux", + "bash": "https://git.savannah.gnu.org/git/bash.git", + "coreutils": "https://git.savannah.gnu.org/git/coreutils.git", + "systemd": "https://github.com/systemd/systemd", + "dbus": "https://gitlab.freedesktop.org/dbus/dbus", + "cups": "https://github.com/OpenPrinting/cups", + "samba": "https://git.samba.org/samba.git", + "openvpn": "https://github.com/OpenVPN/openvpn", + "strongswan": "https://github.com/strongswan/strongswan", + "iptables": "https://git.netfilter.org/iptables", + "nftables": "https://git.netfilter.org/nftables", + "wireshark": "https://gitlab.com/wireshark/wireshark", + "tcpdump": "https://github.com/the-tcpdump-group/tcpdump", + "libpcap": "https://github.com/the-tcpdump-group/libpcap", + "zlib": "https://github.com/madler/zlib", + "xz": "https://github.com/tukaani-project/xz", + "bzip2": "https://sourceware.org/git/bzip2.git", + "lz4": "https://github.com/lz4/lz4", + "zstd": "https://github.com/facebook/zstd", + "libpng": "https://github.com/pnggroup/libpng", + "libjpeg-turbo": "https://github.com/libjpeg-turbo/libjpeg-turbo", + "imagemagick": "https://github.com/ImageMagick/ImageMagick", + "ffmpeg": "https://git.ffmpeg.org/ffmpeg.git", + "vlc": "https://code.videolan.org/videolan/vlc", + "haproxy": "https://github.com/haproxy/haproxy", + "traefik": "https://github.com/traefik/traefik", + "envoy": "https://github.com/envoyproxy/envoy", + "kubernetes": "https://github.com/kubernetes/kubernetes", + "docker": "https://github.com/moby/moby", + "containerd": "https://github.com/containerd/containerd", + "runc": "https://github.com/opencontainers/runc", + "podman": "https://github.com/containers/podman", + "buildah": "https://github.com/containers/buildah", + "skopeo": "https://github.com/containers/skopeo", + "cri-o": "https://github.com/cri-o/cri-o", + "etcd": "https://github.com/etcd-io/etcd", + "consul": "https://github.com/hashicorp/consul", + "vault": "https://github.com/hashicorp/vault", + "terraform": "https://github.com/hashicorp/terraform", + "ansible": "https://github.com/ansible/ansible", + "puppet": "https://github.com/puppetlabs/puppet", + "chef": "https://github.com/chef/chef", + "salt": "https://github.com/saltstack/salt", + "jenkins": "https://github.com/jenkinsci/jenkins", + "grafana": "https://github.com/grafana/grafana", + "prometheus": "https://github.com/prometheus/prometheus", + "elasticsearch": "https://github.com/elastic/elasticsearch", + "logstash": "https://github.com/elastic/logstash", + "kibana": "https://github.com/elastic/kibana", + "kafka": "https://github.com/apache/kafka", + "zookeeper": "https://github.com/apache/zookeeper", + "cassandra": "https://github.com/apache/cassandra", + "mongodb": "https://github.com/mongodb/mongo", + "mariadb": "https://github.com/MariaDB/server", + "tomcat": "https://github.com/apache/tomcat", + "jetty": "https://github.com/jetty/jetty.project", + "spring-boot": "https://github.com/spring-projects/spring-boot", + "spring-framework": "https://github.com/spring-projects/spring-framework", + "struts": "https://github.com/apache/struts", + "log4j": "https://github.com/apache/logging-log4j2", + "jackson": "https://github.com/FasterXML/jackson-databind", + "guava": "https://github.com/google/guava", + "netty": "https://github.com/netty/netty", + "grpc": "https://github.com/grpc/grpc", + "protobuf": "https://github.com/protocolbuffers/protobuf", + "flatbuffers": "https://github.com/google/flatbuffers", + "capnproto": "https://github.com/capnproto/capnproto", + "thrift": "https://github.com/apache/thrift", + "avro": "https://github.com/apache/avro", + "arrow": "https://github.com/apache/arrow", + "parquet": "https://github.com/apache/parquet-java", + "spark": "https://github.com/apache/spark", + "flink": "https://github.com/apache/flink", + "hadoop": "https://github.com/apache/hadoop", + "hive": "https://github.com/apache/hive", + "presto": "https://github.com/prestodb/presto", + "trino": "https://github.com/trinodb/trino", + "airflow": "https://github.com/apache/airflow", + "celery": "https://github.com/celery/celery", + "rabbitmq": "https://github.com/rabbitmq/rabbitmq-server", + "activemq": "https://github.com/apache/activemq", + "pulsar": "https://github.com/apache/pulsar", + "nats": "https://github.com/nats-io/nats-server", + "zeromq": "https://github.com/zeromq/libzmq", + "expat": "https://github.com/libexpat/libexpat", + "freetype": "https://gitlab.freedesktop.org/freetype/freetype", + "fontconfig": "https://gitlab.freedesktop.org/fontconfig/fontconfig", + "cairo": "https://gitlab.freedesktop.org/cairo/cairo", + "pango": "https://gitlab.gnome.org/GNOME/pango", + "gdk-pixbuf": "https://gitlab.gnome.org/GNOME/gdk-pixbuf", + "gtk": "https://gitlab.gnome.org/GNOME/gtk", + "glib": "https://gitlab.gnome.org/GNOME/glib", + "libsoup": "https://gitlab.gnome.org/GNOME/libsoup", + "webkit": "https://github.com/nickthecoder/nickthecoder-website", + "webkitgtk": "https://github.com/nickthecoder/nickthecoder-website", + "chromium": "https://chromium.googlesource.com/chromium/src.git", + "firefox": "https://github.com/nickthecoder/nickthecoder-website", + "libtiff": "https://gitlab.com/libtiff/libtiff", + "giflib": "https://sourceforge.net/projects/giflib", + "libwebp": "https://chromium.googlesource.com/webm/libwebp", + "openjpeg": "https://github.com/uclouvain/openjpeg", + "libvpx": "https://chromium.googlesource.com/webm/libvpx", + "x264": "https://code.videolan.org/videolan/x264", + "x265": "https://bitbucket.org/multicoreware/x265_git", + "dav1d": "https://code.videolan.org/videolan/dav1d", + "aom": "https://aomedia.googlesource.com/aom", + "opus": "https://gitlab.xiph.org/xiph/opus", + "flac": "https://github.com/xiph/flac", + "vorbis": "https://gitlab.xiph.org/xiph/vorbis", + "ogg": "https://gitlab.xiph.org/xiph/ogg", + "libsndfile": "https://github.com/libsndfile/libsndfile", + "pulseaudio": "https://gitlab.freedesktop.org/pulseaudio/pulseaudio", + "pipewire": "https://gitlab.freedesktop.org/pipewire/pipewire", + "alsa-lib": "https://github.com/alsa-project/alsa-lib", + "openldap": "https://git.openldap.org/openldap/openldap", + "cyrus-sasl": "https://github.com/cyrusimap/cyrus-sasl", + "krb5": "https://github.com/krb5/krb5", + "nss": "https://github.com/nickthecoder/nickthecoder-website", + "nspr": "https://github.com/nickthecoder/nickthecoder-website", + "libssh": "https://git.libssh.org/projects/libssh.git", + "libssh2": "https://github.com/libssh2/libssh2", + "libevent": "https://github.com/libevent/libevent", + "libuv": "https://github.com/libuv/libuv", + "libev": "https://github.com/enki/libev", + "boost": "https://github.com/boostorg/boost", + "poco": "https://github.com/pocoproject/poco", + "cpprestsdk": "https://github.com/microsoft/cpprestsdk", + "libcurl": "https://github.com/curl/curl", + "wget": "https://git.savannah.gnu.org/git/wget.git", + "rsync": "https://github.com/WayneD/rsync", + "sudo": "https://github.com/sudo-project/sudo", + "polkit": "https://gitlab.freedesktop.org/polkit/polkit", + "shadow": "https://github.com/shadow-maint/shadow", + "pam": "https://github.com/linux-pam/linux-pam", + "util-linux": "https://github.com/util-linux/util-linux", + "e2fsprogs": "https://git.kernel.org/pub/scm/fs/ext2/e2fsprogs.git", + "xfsprogs": "https://git.kernel.org/pub/scm/fs/xfs/xfsprogs-dev.git", + "btrfs-progs": "https://github.com/kdave/btrfs-progs", + "lvm2": "https://sourceware.org/git/lvm2.git", + "mdadm": "https://git.kernel.org/pub/scm/utils/mdadm/mdadm.git", + "parted": "https://git.savannah.gnu.org/git/parted.git", + "grub": "https://git.savannah.gnu.org/git/grub.git", + "dracut": "https://github.com/dracutdevs/dracut", + "systemd-boot": "https://github.com/systemd/systemd", + "dhcp": "https://gitlab.isc.org/isc-projects/dhcp", + "dnsmasq": "https://thekelleys.org.uk/gitweb/?p=dnsmasq.git", + "unbound": "https://github.com/NLnetLabs/unbound", + "nsd": "https://github.com/NLnetLabs/nsd", + "postfix": "https://github.com/vdukhovni/postfix", + "dovecot": "https://github.com/dovecot/core", + "exim": "https://git.exim.org/exim.git", + "sendmail": "https://github.com/nickthecoder/nickthecoder-website", + "spamassassin": "https://github.com/apache/spamassassin", + "clamav": "https://github.com/Cisco-Talos/clamav", + "fail2ban": "https://github.com/fail2ban/fail2ban", + "iptables-nft": "https://git.netfilter.org/iptables", + "nmap": "https://github.com/nmap/nmap", + "socat": "https://repo.or.cz/socat.git", + "netcat": "https://sourceforge.net/projects/netcat", + "iproute2": "https://git.kernel.org/pub/scm/network/iproute2/iproute2.git", + "ethtool": "https://git.kernel.org/pub/scm/network/ethtool/ethtool.git", + "bridge-utils": "https://git.kernel.org/pub/scm/network/bridge/bridge-utils.git", + "iw": "https://git.kernel.org/pub/scm/linux/kernel/git/jberg/iw.git", + "wpa_supplicant": "https://w1.fi/cgit/hostap", + "hostapd": "https://w1.fi/cgit/hostap", + "NetworkManager": "https://gitlab.freedesktop.org/NetworkManager/NetworkManager", + "connman": "https://git.kernel.org/pub/scm/network/connman/connman.git", + "avahi": "https://github.com/avahi/avahi", + "bluez": "https://github.com/bluez/bluez", + "qemu": "https://gitlab.com/qemu-project/qemu", + "libvirt": "https://gitlab.com/libvirt/libvirt", + "virt-manager": "https://github.com/virt-manager/virt-manager", + "lxc": "https://github.com/lxc/lxc", + "lxd": "https://github.com/canonical/lxd", + "firecracker": "https://github.com/firecracker-microvm/firecracker", + "kata-containers": "https://github.com/kata-containers/kata-containers", + "crun": "https://github.com/containers/crun", + "bubblewrap": "https://github.com/containers/bubblewrap", + "flatpak": "https://github.com/flatpak/flatpak", + "ostree": "https://github.com/ostreedev/ostree", + "rpm": "https://github.com/rpm-software-management/rpm", + "dnf": "https://github.com/rpm-software-management/dnf", + "yum": "https://github.com/rpm-software-management/yum", + "libsolv": "https://github.com/openSUSE/libsolv", + "librepo": "https://github.com/rpm-software-management/librepo", + "createrepo": "https://github.com/rpm-software-management/createrepo_c", + "dpkg": "https://git.dpkg.org/git/dpkg/dpkg.git", + "apt": "https://salsa.debian.org/apt-team/apt", + "pacman": "https://gitlab.archlinux.org/pacman/pacman", + "zypper": "https://github.com/openSUSE/zypper", + "snapd": "https://github.com/snapcore/snapd" + }, + "aliases": { + "apache": "httpd", + "apache2": "httpd", + "apache-httpd": "httpd", + "postgres": "postgresql", + "pgsql": "postgresql", + "cpython": "python", + "python3": "python", + "node": "nodejs", + "go": "golang", + "moby": "docker", + "mongo": "mongodb", + "mysql-server": "mysql", + "mariadb-server": "mariadb", + "kernel": "linux", + "linux-kernel": "linux", + "openssh-server": "openssh", + "openssh-client": "openssh", + "ssh": "openssh", + "sshd": "openssh", + "named": "bind", + "bind9": "bind", + "nginx-core": "nginx", + "libcurl4": "curl", + "curl-minimal": "curl", + "openssl-libs": "openssl", + "libssl": "openssl", + "libopenssl": "openssl", + "gnutls-utils": "gnutls", + "libgnutls": "gnutls", + "vim-minimal": "vim", + "vim-enhanced": "vim", + "neovim": "vim", + "ntp": "chrony", + "ntpd": "chrony", + "libxml2-dev": "libxml2", + "libxml2-devel": "libxml2", + "libxslt-dev": "libxslt", + "libxslt-devel": "libxslt", + "php-fpm": "php", + "php-cli": "php", + "postgresql-server": "postgresql", + "postgresql-client": "postgresql", + "mysql-client": "mysql", + "redis-server": "redis", + "redis-cli": "redis", + "memcached-devel": "memcached", + "sqlite3": "sqlite", + "libsqlite3": "sqlite", + "systemd-libs": "systemd", + "libsystemd": "systemd", + "cups-libs": "cups", + "libcups": "cups", + "samba-client": "samba", + "samba-common": "samba", + "samba-libs": "samba", + "libsmbclient": "samba", + "qemu-kvm": "qemu", + "qemu-system": "qemu", + "qemu-img": "qemu", + "libvirt-daemon": "libvirt", + "libvirt-client": "libvirt", + "docker-ce": "docker", + "docker-engine": "docker", + "containerd.io": "containerd", + "podman-docker": "podman", + "zlib-devel": "zlib", + "zlib1g": "zlib", + "xz-libs": "xz", + "liblzma": "xz", + "bzip2-libs": "bzip2", + "libbz2": "bzip2", + "zstd-devel": "zstd", + "libzstd": "zstd", + "libpng-devel": "libpng", + "libpng16": "libpng", + "libjpeg": "libjpeg-turbo", + "jpeg": "libjpeg-turbo", + "turbojpeg": "libjpeg-turbo", + "ImageMagick-libs": "imagemagick", + "libMagick": "imagemagick", + "ffmpeg-libs": "ffmpeg", + "libavcodec": "ffmpeg", + "libavformat": "ffmpeg", + "libavutil": "ffmpeg", + "vlc-libs": "vlc", + "libvlc": "vlc" + } +} diff --git a/src/exploit_iq_commons/data/standard_libs/__init__.py b/src/exploit_iq_commons/data/standard_libs/__init__.py new file mode 100644 index 000000000..cf7c586a5 --- /dev/null +++ b/src/exploit_iq_commons/data/standard_libs/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/exploit_iq_commons/data/standard_libs/standard_libraries.json b/src/exploit_iq_commons/data/standard_libs/standard_libraries.json new file mode 100644 index 000000000..0f5e4412f --- /dev/null +++ b/src/exploit_iq_commons/data/standard_libs/standard_libraries.json @@ -0,0 +1,280 @@ +{ + "_metadata": { + "_comment": "AUTO-GENERATED FILE - Do not manually edit. Generated using AI. Review changes.", + "_generation_prompt": "For {ecosystem}, generate a comprehensive JSON list of ALL standard library packages/modules. Include built-in modules that come with the language installation. Format as a flat array of strings. Be exhaustive and accurate.", + "_last_updated": "2025-10-21" + }, + "go": [ + "archive/tar", + "archive/zip", + "bufio", + "bytes", + "compress/gzip", + "compress/zlib", + "container/heap", + "container/list", + "context", + "crypto", + "crypto/aes", + "crypto/cipher", + "crypto/des", + "crypto/ecdsa", + "crypto/ed25519", + "crypto/hmac", + "crypto/md5", + "crypto/rand", + "crypto/rsa", + "crypto/sha1", + "crypto/sha256", + "crypto/sha512", + "crypto/tls", + "crypto/x509", + "database/sql", + "database/sql/driver", + "debug/pprof", + "encoding", + "encoding/ascii85", + "encoding/base32", + "encoding/base64", + "encoding/binary", + "encoding/csv", + "encoding/gob", + "encoding/hex", + "encoding/json", + "encoding/pem", + "encoding/xml", + "errors", + "expvar", + "flag", + "fmt", + "go/ast", + "go/parser", + "go/token", + "hash", + "hash/adler32", + "hash/crc32", + "hash/fnv", + "html", + "html/template", + "image", + "image/color", + "image/draw", + "image/gif", + "image/jpeg", + "image/png", + "index/suffixarray", + "io", + "io/fs", + "io/ioutil", + "log", + "log/syslog", + "math", + "math/big", + "math/cmplx", + "math/rand", + "mime", + "mime/multipart", + "net", + "net/http", + "net/http/cgi", + "net/http/cookiejar", + "net/http/httptest", + "net/http/httputil", + "net/http/pprof", + "net/mail", + "net/rpc", + "net/smtp", + "net/textproto", + "net/url", + "os", + "os/exec", + "os/signal", + "os/user", + "path", + "path/filepath", + "reflect", + "regexp", + "regexp/syntax", + "runtime", + "runtime/cgo", + "runtime/debug", + "runtime/pprof", + "sort", + "strconv", + "strings", + "sync", + "sync/atomic", + "syscall", + "testing", + "testing/quick", + "text/scanner", + "text/tabwriter", + "text/template", + "time", + "time/tzdata", + "unicode", + "unicode/utf16", + "unicode/utf8", + "unsafe" + ], + "python": [ + "abc", + "argparse", + "array", + "ast", + "asyncio", + "base64", + "bisect", + "bz2", + "calendar", + "cProfile", + "codecs", + "collections", + "collections.abc", + "concurrent.futures", + "configparser", + "contextlib", + "copy", + "csv", + "ctypes", + "dataclasses", + "datetime", + "decimal", + "difflib", + "dis", + "email", + "enum", + "errno", + "fractions", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "gzip", + "hashlib", + "heapq", + "hmac", + "http", + "http.client", + "http.cookiejar", + "http.cookies", + "http.server", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "logging", + "logging.config", + "logging.handlers", + "lzma", + "math", + "mmap", + "multiprocessing", + "netrc", + "operator", + "os", + "os.path", + "pathlib", + "pdb", + "pickle", + "platform", + "pprint", + "pty", + "queue", + "random", + "re", + "sched", + "secrets", + "select", + "shelve", + "shlex", + "shutil", + "signal", + "socket", + "socketserver", + "sqlite3", + "ssl", + "statistics", + "string", + "struct", + "subprocess", + "sys", + "sysconfig", + "tarfile", + "tempfile", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "traceback", + "tracemalloc", + "types", + "typing", + "unittest", + "urllib", + "urllib.error", + "urllib.parse", + "urllib.request", + "urllib.robotparser", + "uuid", + "warnings", + "weakref", + "webbrowser", + "wsgiref", + "xml.dom.minidom", + "xml.etree.ElementTree", + "xml.parsers.expat", + "xmlrpc.client", + "xmlrpc.server", + "zipapp", + "zipfile", + "zlib" + ], + "javascript": [ + "assert", + "async_hooks", + "buffer", + "child_process", + "cluster", + "console", + "crypto", + "dgram", + "dns", + "domain", + "events", + "fs", + "fs/promises", + "http", + "http2", + "https", + "inspector", + "module", + "net", + "os", + "path", + "perf_hooks", + "process", + "querystring", + "readline", + "repl", + "stream", + "string_decoder", + "timers", + "timers/promises", + "tls", + "trace_events", + "tty", + "url", + "util", + "v8", + "vm", + "wasi", + "worker_threads", + "zlib" + ] +} \ No newline at end of file diff --git a/src/exploit_iq_commons/data_models/__init__.py b/src/exploit_iq_commons/data_models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/exploit_iq_commons/data_models/checker_status.py b/src/exploit_iq_commons/data_models/checker_status.py new file mode 100644 index 000000000..4047d02ea --- /dev/null +++ b/src/exploit_iq_commons/data_models/checker_status.py @@ -0,0 +1,313 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from enum import Enum +from enum import IntEnum +from pathlib import Path +from typing import Any, Literal + +from pydantic import BaseModel, Field + + +class PackageCheckerStatus(IntEnum): + """Per-CVE status codes produced by the PackageIdentify phase.""" + OK = 0 + ERROR_PKG_IDENT_NO_INTEL = 1 + PKG_IDENT_NOT_VUL = 2 + ERROR_FAILED_TO_DOWNLOAD_SRPM = 3 + PKG_IDENT_CVE_MISMATCH = 4 + PKG_INTEL_LOW_SCORE = 5 + CONFLICTING_INTEL_EVIDENCE = 6 + + +PACKAGE_CHECKER_STATUS_DESCRIPTIONS: dict[PackageCheckerStatus, str] = { + PackageCheckerStatus.OK: + "Package identified and in affected range -- continue investigation", + PackageCheckerStatus.ERROR_PKG_IDENT_NO_INTEL: + "No Intel found for the package", + PackageCheckerStatus.PKG_IDENT_NOT_VUL: + "Identification state concluded from intel that target package is not vulnerable", + PackageCheckerStatus.ERROR_FAILED_TO_DOWNLOAD_SRPM: + "Failed to download the patched SRPM", + PackageCheckerStatus.PKG_IDENT_CVE_MISMATCH: + "CVE does not apply to target package - RHSA does not list this package", + PackageCheckerStatus.PKG_INTEL_LOW_SCORE: + "Intel quality score below threshold - insufficient information for reliable analysis", + PackageCheckerStatus.CONFLICTING_INTEL_EVIDENCE: + "Code analysis conflicts with RHSA 'not affected' assessment - manual review required", +} + +CHECKER_FAILURE_ERROR_TYPES: dict[PackageCheckerStatus, str] = { + PackageCheckerStatus.ERROR_PKG_IDENT_NO_INTEL: "no-intel", + PackageCheckerStatus.ERROR_FAILED_TO_DOWNLOAD_SRPM: "srpm-download-failed", + PackageCheckerStatus.PKG_IDENT_CVE_MISMATCH: "invalid-input", +} + + +class EnumIdentifyResult(str, Enum): + """Result of the PackageIdentify phase for a single CVE.""" + YES = "yes" + NO = "no" + UNKNOWN = "unknown" + +class PackageIdentifyResult(BaseModel): + """Result of the PackageIdentify phase for a single CVE.""" + affected_rpm_list: list[str] = [] + fixed_rpm_list: list[str] = [] + + is_target_package_affected: EnumIdentifyResult = EnumIdentifyResult.UNKNOWN + is_target_package_fixed: EnumIdentifyResult = EnumIdentifyResult.UNKNOWN + + rhsa_fix_state: str | None = Field( + default=None, + description="Raw RHSA fix_state value when is_target_package_affected is based on RHSA assessment" + ) + + conclusion_reason: str = Field( + default="", + description="Detailed explanation of why the package was determined to be vulnerable or not vulnerable" + ) + + + +class AcquiredArtifacts(BaseModel): + """Resolved file locations populated by source_acquisition, consumed by downstream checker nodes.""" + srpm_path: Path | None = None + source_dir: Path | None = None + build_log_path: Path | None = None + binary_rpm_path: Path | None = None + patch_source_dir: Path | None = None + patch_diff_path: Path | None = None + source_url: str | None = None + + +class VulnerabilityIntel(BaseModel): + """Structured intelligence extracted from CVE advisories and patches. + + Used to provide grep-ready patterns and context for L1 agent source searches. + """ + + affected_files: list[str] = Field( + default_factory=list, + description="Source file paths likely to contain vulnerable code" + ) + vulnerable_functions: list[str] = Field( + default_factory=list, + description="Function names that contain or handle the vulnerability" + ) + vulnerable_variables: list[str] = Field( + default_factory=list, + description="Variable names involved in the vulnerability" + ) + vulnerable_patterns: list[str] = Field( + default_factory=list, + description="Code patterns/snippets indicating vulnerable code (from - lines)" + ) + fix_patterns: list[str] = Field( + default_factory=list, + description="Code patterns/snippets indicating fixed code (from + lines)" + ) + root_cause: str = Field( + default="", + description="Technical explanation of why the code is vulnerable" + ) + vulnerability_type: str = Field( + default="", + description="Category: buffer_overflow, integer_overflow, use_after_free, null_deref, etc." + ) + search_keywords: list[str] = Field( + default_factory=list, + description="Recommended grep patterns ordered by specificity (most specific first)" + ) + component_names: list[str] = Field( + default_factory=list, + description="Module or component names explicitly mentioned in CVE intel (e.g., mod_http2, libxml2_sax). " + "Only extract if explicitly named in advisory/description - do NOT infer or guess." + ) + affected_bitness: Literal["32-bit", "64-bit", "both"] = Field( + default="both", + description="Which bitness is affected: 32-bit only, 64-bit only, or both (default)" + ) + affected_architectures: list[str] | None = Field( + default=None, + description="CPU families affected (e.g., ['x86', 'arm']). None means all architectures." + ) + is_downstream_patch_available: bool = Field( + default=False, + description="True if a CVE-specific patch file exists in the downstream package" + ) + is_patch_applied_in_build: bool = Field( + default=False, + description="True if the patch was confirmed applied in build logs" + ) + patch_file_name: str = Field( + default="", + description="Name of the CVE-specific patch file (if available)" + ) + known_mitigations: str = Field( + default="", + description="Vendor-provided mitigations from RHSA or other intel sources (e.g., compiler flags, config changes)" + ) + + # Version range context from Identify phase + affected_version_range: str = Field( + default="", + description="Affected version range from NVD configurations (e.g., '< 2.4.68')" + ) + fixed_version: str = Field( + default="", + description="First fixed version from NVD intel (e.g., '2.4.68')" + ) + target_version_in_vulnerable_range: bool | None = Field( + default=None, + description="True if target package version is within the vulnerable range (from Identify phase)" + ) + target_package_version: str = Field( + default="", + description="Actual version of the target package being analyzed (e.g., '2.4.63')" + ) + + def format_for_prompt(self) -> str: + """Format VulnerabilityIntel for injection into L1 agent runtime prompt. + + Uses UPPERCASE labels so they can be referenced as anchors in thought prompts. + """ + lines = [] + if self.is_downstream_patch_available: + status = "APPLIED" if self.is_patch_applied_in_build else "AVAILABLE" + lines.append(f"DOWNSTREAM_PATCH_STATUS: {status}") + if self.patch_file_name: + lines.append(f"PATCH_FILE: {self.patch_file_name}") + if self.affected_files: + lines.append(f"AFFECTED_FILES: {', '.join(self.affected_files)}") + if self.vulnerable_functions: + lines.append(f"VULNERABLE_FUNCTIONS: {', '.join(self.vulnerable_functions)}") + if self.vulnerable_variables: + lines.append(f"VULNERABLE_VARIABLES: {', '.join(self.vulnerable_variables)}") + if self.vulnerable_patterns: + lines.append("VULNERABLE_PATTERNS:") + for p in self.vulnerable_patterns: + lines.append(f" - {p}") + if self.fix_patterns: + lines.append("FIX_PATTERNS:") + for p in self.fix_patterns: + lines.append(f" - {p}") + if self.search_keywords: + lines.append(f"SEARCH_KEYWORDS: {', '.join(self.search_keywords)}") + if self.component_names: + lines.append(f"COMPONENT_NAMES: {', '.join(self.component_names)}") + if self.root_cause: + lines.append(f"ROOT_CAUSE: {self.root_cause}") + if self.affected_bitness and self.affected_bitness != "both": + lines.append(f"AFFECTED_BITNESS: {self.affected_bitness}") + if self.affected_architectures: + lines.append(f"AFFECTED_ARCHITECTURES: {', '.join(self.affected_architectures)}") + if self.known_mitigations: + lines.append(f"KNOWN_MITIGATIONS: {self.known_mitigations}") + # Version context - target version first for clear comparison + if self.target_package_version: + lines.append(f"TARGET_PACKAGE_VERSION: {self.target_package_version}") + if self.affected_version_range: + lines.append(f"AFFECTED_VERSION_RANGE: {self.affected_version_range}") + if self.fixed_version: + lines.append(f"FIXED_VERSION: {self.fixed_version}") + return "\n".join(lines) + + +class L1InvestigationResult(BaseModel): + """Intermediate result from L1 investigation, input to L2 or report generation.""" + downstream_report: dict[str, Any] | None = Field( + default=None, + description="DownstreamSearchReport from L1 investigation (as dict)", + ) + upstream_report: dict[str, Any] | None = Field( + default=None, + description="UpstreamSearchReport from L1 investigation (as dict)", + ) + git_search_report: dict[str, Any] | None = Field( + default=None, + description="GitSearchReport from git commit search (as dict)", + ) + l1_agent_answer: str | None = Field( + default=None, + description="Final answer from the L1 ReAct agent", + ) + vulnerability_intel: VulnerabilityIntel | None = Field( + default=None, + description="Structured vulnerability intelligence extracted from CVE advisories and patches", + ) + preliminary_verdict: Literal["vulnerable", "protected", "not_present", "uncertain"] = Field( + default="uncertain", + description="L1 verdict before L2 refinement", + ) + confidence: float = Field( + default=0.0, + ge=0.0, + le=1.0, + description="Confidence in the preliminary verdict", + ) + + +class L2BuildResult(BaseModel): + """Result from L2 Build Agent (BuildCompilationCheck + HardeningCheck).""" + compilation_status: Literal["compiled", "not_compiled", "unknown"] = Field( + default="unknown", + description="Whether vulnerable code is compiled into the binary", + ) + compilation_confidence: float = Field( + default=0.0, + ge=0.0, + le=1.0, + description="Confidence in compilation status", + ) + compilation_evidence: str | None = Field( + default=None, + description="Evidence supporting compilation status", + ) + hardening_relevant: bool | None = Field( + default=None, + description="Whether detected hardening flags are relevant to the CVE", + ) + hardening_flags: list[str] = Field( + default_factory=list, + description="Hardening flags detected in build log or binary", + ) + hardening_rationale: str | None = Field( + default=None, + description="Rationale for hardening relevance judgment", + ) + l2_override_verdict: Literal["not_vulnerable", "vulnerable_mitigated", None] = Field( + default=None, + description="L2 verdict override (if any)", + ) + evidence_sources: list[str] = Field( + default_factory=list, + description="Sources used for analysis: 'build_log', 'spec_file', 'build_system_files', 'binary'", + ) + + +class PackageCheckerContext(BaseModel): + """Consolidates all checker-specific state on AgentMorpheusInfo.""" + status: PackageCheckerStatus | None = None + source_key: str | None = None + artifacts: AcquiredArtifacts = Field(default_factory=AcquiredArtifacts) + identify_result: PackageIdentifyResult = Field(default_factory=PackageIdentifyResult) + l1_result: L1InvestigationResult | None = Field( + default=None, + description="Result from L1 Code Agent investigation", + ) + l2_result: L2BuildResult | None = Field( + default=None, + description="Result from L2 Build Agent (optional)", + ) diff --git a/src/vuln_analysis/data_models/common.py b/src/exploit_iq_commons/data_models/common.py similarity index 79% rename from src/vuln_analysis/data_models/common.py rename to src/exploit_iq_commons/data_models/common.py index 39455cf77..db848f4ad 100644 --- a/src/vuln_analysis/data_models/common.py +++ b/src/exploit_iq_commons/data_models/common.py @@ -16,6 +16,7 @@ import sys import typing from hashlib import sha512 +from enum import Enum from pydantic import BaseModel from pydantic import Field @@ -23,6 +24,21 @@ _LT = typing.TypeVar("_LT") +class AnalysisType(str, Enum): + IMAGE = "image" + SOURCE = "source" + + +class PipelineMode(str, Enum): + """ + Controls which investigation path the pipeline takes after process_sbom. + Orthogonal to AnalysisType (input format) -- any combination is valid. + """ + FULL_PIPELINE = "full_pipeline" + PACKAGE_CHECKER = "rpm_package_checker" + + + class HashableModel(BaseModel): """ Subclass of a Pydantic BaseModel that is hashable. Use in objects that need to be hashed for caching purposes. @@ -45,7 +61,15 @@ def __ne__(self, other): def __gt__(self, other): return self.__hash__() > other.__hash__() - +class TargetPackage(HashableModel): + """ + A package to investigate. + """ + name: str + version: str | None = None + release: str | None = None # e.g. "1.el8_2.3" (needed for Brew NVR lookup) + arch: str = "x86_64" # e.g. "x86_64", "aarch64", "s390x", "noarch" + class TypedBaseModel(BaseModel, typing.Generic[_LT]): """ Subclass of Pydantic BaseModel that allows for specifying the object type. Use in Pydantic discriminated unions. diff --git a/src/vuln_analysis/data_models/cve_intel.py b/src/exploit_iq_commons/data_models/cve_intel.py similarity index 74% rename from src/vuln_analysis/data_models/cve_intel.py rename to src/exploit_iq_commons/data_models/cve_intel.py index 43b2ddfe5..6d573c297 100644 --- a/src/vuln_analysis/data_models/cve_intel.py +++ b/src/exploit_iq_commons/data_models/cve_intel.py @@ -15,6 +15,8 @@ import abc import functools +import json +import re import typing from pydantic import BaseModel @@ -108,6 +110,7 @@ class Configuration(BaseModel): cvss_vector: str | None = None cvss_base_score: float | None = None cvss_severity: str | None = None + cwe_id: str | None = None cwe_name: str | None = None cwe_description: str | None = None cwe_extended_description: str | None = None @@ -183,8 +186,8 @@ class CVSSV3(BaseModel): class BaseMetricV3(BaseModel): cvssV3: "CVSSV3" - exploitabilityScore: float - impactScore: float + exploitabilityScore: float | None = None + impactScore: float | None = None class Impact(BaseModel): baseMetricV3: "BaseMetricV3" @@ -195,6 +198,10 @@ class Impact(BaseModel): priority: str | None = None ubuntu_description: str | None = None impact: Impact | None = None + patches: dict[str, list[str]] | None = Field( + default=None, + description="Map of package name to patch refs (e.g., 'upstream: https://github.com/.../commit/...')" + ) @property def description_fields(self): @@ -216,6 +223,60 @@ def description_fields(self): return [] +class CveIntelOsidb(IntelSource): + """ + Information about an OSIDB (Open Source Information Database) flaw entry. + Available only for internal Red Hat users on VPN. + """ + model_config = ConfigDict(extra="allow") + + class UpstreamPurl(BaseModel): + model_config = ConfigDict(extra="allow") + name: str | None = None + purl: str | None = None + ecosystem: str | None = None + + class Affect(BaseModel): + model_config = ConfigDict(extra="allow") + ps_module: str | None = None + ps_product: str | None = None + ps_component: str | None = None + ps_update_stream: str | None = None + resolution: str | None = None + affectedness: str | None = None + purl: str | None = None + + class CvssScore(BaseModel): + model_config = ConfigDict(extra="allow") + issuer: str | None = None + score: float | None = None + vector: str | None = None + cvss_version: str | None = None + + cve_id: str | None = None + impact: str | None = None + title: str | None = None + comment_zero: str | None = None + cve_description: str | None = None + statement: str | None = None + cwe_id: str | None = None + source: str | None = None + mitigation: str | None = None + upstream_purls: list[UpstreamPurl] = [] + affects: list[Affect] = [] + cvss_scores: list[CvssScore] = [] + references: list[str] = [] + + @property + def description_fields(self): + return ['comment_zero', 'cve_description', 'statement'] + + +class IntelPluginData(BaseModel): + label: str + description: str + + class CveIntel(BaseModel): """ Information about a CVE (Common Vulnerabilities and Exposures) entry. @@ -231,6 +292,9 @@ class CveIntel(BaseModel): rhsa: CveIntelRhsa | None = None ubuntu: CveIntelUbuntu | None = None epss: CveIntelEpss | None = None + osidb: CveIntelOsidb | None = None + plugin_data: list[IntelPluginData] = [] + intel_score: int = 0 @computed_field() @property @@ -332,3 +396,30 @@ def get_ghsa_id(self): return self.vuln_id return None + + @property + def plugins_intel_data(self) -> list[str]: + return [f"{pd.label}: {pd.description}" for pd in self.plugin_data] + + def get_intel_score(self) -> int: + return self.intel_score + + def parse_plugin_data(self, plugin_list: list[str]) -> dict: + merged = {} + for plugin_str in plugin_list: + if not isinstance(plugin_str, str): + continue # skip non-strings + + match = re.search(r'{.*}', plugin_str, re.DOTALL) + if match: + try: + plugin_dict = json.loads(match.group()) + for k, v in plugin_dict.items(): + if k in merged: + merged[k] += f" {v}" + else: + merged[k] = str(v) + except json.JSONDecodeError: + continue # skip malformed JSON + + return merged \ No newline at end of file diff --git a/src/vuln_analysis/data_models/dependencies.py b/src/exploit_iq_commons/data_models/dependencies.py similarity index 79% rename from src/vuln_analysis/data_models/dependencies.py rename to src/exploit_iq_commons/data_models/dependencies.py index 1af1325f0..43bcda7b8 100644 --- a/src/vuln_analysis/data_models/dependencies.py +++ b/src/exploit_iq_commons/data_models/dependencies.py @@ -28,6 +28,19 @@ class DependencyPackage(BaseModel): relation: typing.Literal["SELF", "DIRECT", "INDIRECT"] | None = None +class CheckedNotVulnerablePackage(BaseModel): + """ + Information about a package that was checked against CVE intel but determined not vulnerable. + + - name: package name that was checked + - version: installed version of the package + - reason: LLM-generated explanation of why the package is not vulnerable + """ + name: str + version: str + reason: str + + class VulnerableSBOMPackage(BaseModel): """ Information about a vulnerable SBOM package and its related vulnerable dependency package. @@ -53,7 +66,10 @@ class VulnerableDependencies(BaseModel): the vulnerable package/version intel for the vuln_id. - vulnerable_sbom_packages: list of VulnerableSBOMPackage objects, representing the SBOM packages that are vulnerable for a given vuln_id. + - checked_not_vulnerable: list of CheckedNotVulnerablePackage objects, representing packages that were + checked but determined not vulnerable, along with the LLM reasoning. """ vuln_id: str vuln_package_intel_sources: list[str] vulnerable_sbom_packages: list[VulnerableSBOMPackage] + checked_not_vulnerable: list[CheckedNotVulnerablePackage] = [] diff --git a/src/vuln_analysis/data_models/info.py b/src/exploit_iq_commons/data_models/info.py similarity index 89% rename from src/vuln_analysis/data_models/info.py rename to src/exploit_iq_commons/data_models/info.py index e94094350..4f7bd1ef1 100644 --- a/src/vuln_analysis/data_models/info.py +++ b/src/exploit_iq_commons/data_models/info.py @@ -15,8 +15,9 @@ from pydantic import BaseModel -from .cve_intel import CveIntel -from .dependencies import VulnerableDependencies +from exploit_iq_commons.data_models.checker_status import PackageCheckerContext +from exploit_iq_commons.data_models.cve_intel import CveIntel +from exploit_iq_commons.data_models.dependencies import VulnerableDependencies class SBOMPackage(BaseModel): @@ -62,3 +63,4 @@ class SBOMInfo(BaseModel): intel: list[CveIntel] | None = None sbom: SBOMInfo | None = None vulnerable_dependencies: list[VulnerableDependencies] | None = None + checker_context: PackageCheckerContext | None = None diff --git a/src/vuln_analysis/data_models/input.py b/src/exploit_iq_commons/data_models/input.py similarity index 69% rename from src/vuln_analysis/data_models/input.py rename to src/exploit_iq_commons/data_models/input.py index c079ca295..b15e2f09a 100644 --- a/src/vuln_analysis/data_models/input.py +++ b/src/exploit_iq_commons/data_models/input.py @@ -25,13 +25,19 @@ from pydantic import Field from pydantic import Tag from pydantic import field_validator +from pydantic import model_validator -from ..utils.string_utils import is_valid_cve_id -from ..utils.string_utils import is_valid_ghsa_id -from .common import HashableModel -from .common import TypedBaseModel -from .info import AgentMorpheusInfo -from .info import SBOMPackage +from exploit_iq_commons.utils.string_utils import is_valid_cve_id +from exploit_iq_commons.utils.string_utils import is_valid_ghsa_id +from exploit_iq_commons.utils.dep_tree import Ecosystem +from exploit_iq_commons.data_models.common import AnalysisType +from exploit_iq_commons.data_models.common import HashableModel +from exploit_iq_commons.data_models.common import PipelineMode , TargetPackage +from exploit_iq_commons.data_models.common import TypedBaseModel +from exploit_iq_commons.data_models.info import AgentMorpheusInfo +from exploit_iq_commons.data_models.info import SBOMPackage + +DEFAULT_FAILURE_REASON = "No failure reason provided" class SourceDocumentsInfo(HashableModel): @@ -108,8 +114,14 @@ class ScanInfoInput(HashableModel): """ id: str = Field(default_factory=lambda: str(uuid4())) type: str | None = None - started_at: str | None = None - completed_at: str | None = None + started_at: str | None = Field( + default=None, + description="Scan start time as ISO-8601 with UTC offset (e.g. ...+00:00).", + ) + completed_at: str | None = Field( + default=None, + description="Scan completion time as ISO-8601 with UTC offset (e.g. ...+00:00).", + ) vulns: typing.Annotated[list[VulnInfo], Field(min_length=1)] @@ -150,9 +162,34 @@ class ImageInfoInput(HashableModel): digest: str | None = None # i.e. sha256:... platform: str | None = None # i.e. linux/amd64 feed_group: str | None = None # i.e. ubuntu:22.04 + ecosystem: Ecosystem | None = None # i.e. python + manifest_path: str | None = None # i.e. path/to/requirements.txt + + analysis_type: AnalysisType # i.e. image + """ + The type of analysis to perform: + - "image": Analysis of a container image and tag with SBOM data + - "source": Analysis of source code and commitId without SBOM data + """ + + pipeline_mode: PipelineMode = PipelineMode.FULL_PIPELINE + """ + Controls which investigation path the pipeline takes after process_sbom: + - "full_pipeline": Full transitive analysis (verify_vuln_package -> llm_engine) + - "package_checker": Focused package vulnerability checker (package_checker -> checker_output) + """ + target_package: TargetPackage | None = None + + source_info: list[SourceDocumentsInfo] = [] + sbom_info: SBOMInfoInput | None = None - source_info: list[SourceDocumentsInfo] - sbom_info: SBOMInfoInput + @model_validator(mode="after") + def validate_pipeline_requirements(self) -> "ImageInfoInput": + if self.pipeline_mode == PipelineMode.PACKAGE_CHECKER and self.target_package is None: + raise ValueError("target_package is required when pipeline_mode is PACKAGE_CHECKER") + if self.pipeline_mode == PipelineMode.FULL_PIPELINE and not self.source_info: + raise ValueError("source_info is required and must not be empty when pipeline_mode is FULL_PIPELINE") + return self @field_validator('source_info', mode='after') @classmethod @@ -175,6 +212,12 @@ class AgentMorpheusInput(HashableModel): """ scan: ScanInfoInput image: ImageInfoInput + credential_id: str | None = Field( + default=None, + validation_alias=AliasChoices("credential_id", "credentialId"), + ) + code_index_success: bool | None = None + failure_reason: str | None = Field(default=DEFAULT_FAILURE_REASON) class AgentMorpheusEngineInput(BaseModel): diff --git a/src/exploit_iq_commons/logging/__init__.py b/src/exploit_iq_commons/logging/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/exploit_iq_commons/logging/loggers_factory.py b/src/exploit_iq_commons/logging/loggers_factory.py new file mode 100644 index 000000000..008e80aaf --- /dev/null +++ b/src/exploit_iq_commons/logging/loggers_factory.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys +import textwrap +from contextvars import ContextVar + +MULTI_LINE_MESSAGE = "MultiLineMessage" +MULTI_LINE_MESSAGE_TRUE = {MULTI_LINE_MESSAGE: True} +old_factory = logging.getLogRecordFactory() +trace_id: ContextVar[str] = ContextVar('trace_id', default=' ') + + +class AgentStreamHandler(logging.StreamHandler): + + def __init__(self, stream=None): + super().__init__(stream) + + def emit(self, record): + # escape new lines to remove from multi lines message logs end of lines, preserving only the \n character itself + # This one is in order to support multiline string messages to be logged in centralized logging in the same log + # record, withut being truncated after first new line. + # if record.__dict__.get("%s" % MULTI_LINE_MESSAGE, None): + # record.msg = record.msg.replace("\n", "\\n") + try: + msg = self.format(record) + if record.__dict__.get("%s" % MULTI_LINE_MESSAGE, None): + msg = msg.replace("\n", "\\n") + stream = self.stream + stream.write(msg + self.terminator) + self.flush() + except RecursionError: + raise + except Exception: + self.handleError(record) + + +def record_factory(*args, **kwargs): + record = old_factory(*args, **kwargs) + record.traceId = trace_id.get() + if kwargs.get("%s" % MULTI_LINE_MESSAGE, None): + record.message = textwrap.dedent(record.message) + return record + + +class LoggingFactory: + instances = 0 + + def __init__(self): + logging.setLogRecordFactory(record_factory) + LoggingFactory.instances += 1 + + @classmethod + def get_agent_logger(cls, logger_name: str): + if cls.instances == 0: + cls.__init__(cls) + + logger = logging.getLogger(logger_name) + logger.propagate = False + handler = AgentStreamHandler(sys.stdout) + formatter = logging.Formatter('%(asctime)s - threadName=%(threadName)s- traceId=%(traceId)s - %(name)s - %(' + 'levelname)s -' + ' %(message)s') + handler.setFormatter(formatter) + if len(logger.handlers) == 0: + logger.addHandler(handler) + + return logger diff --git a/src/exploit_iq_commons/pyproject.toml b/src/exploit_iq_commons/pyproject.toml new file mode 100644 index 000000000..708bf96f4 --- /dev/null +++ b/src/exploit_iq_commons/pyproject.toml @@ -0,0 +1,25 @@ +[build-system] +build-backend = "setuptools.build_meta" +requires = ["setuptools >= 64", "setuptools-scm>=8"] + +[project] +name = "exploit-iq-commons" +version = "0.1.0" +description = "Common library for ExploitIQ - shared code for vulnerability analysis." +requires-python = ">=3.11,<3.13" + +dependencies = [ + "esprima==4.0.1", + "GitPython==3.1.44", + "langchain~=0.3.0", + "langchain-community~=0.3.0", + "pandas==2.3.1", + "tqdm==4.67.1", + "tree-sitter-languages==1.10.2", + "tree-sitter==0.21.3", + "pydantic>=2.0", +] + +[tool.setuptools.packages.find] +where = [".."] +include = ["exploit_iq_commons*"] diff --git a/src/exploit_iq_commons/utils/__init__.py b/src/exploit_iq_commons/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/exploit_iq_commons/utils/c_segmenter_custom.py b/src/exploit_iq_commons/utils/c_segmenter_custom.py new file mode 100644 index 000000000..6601f2f0f --- /dev/null +++ b/src/exploit_iq_commons/utils/c_segmenter_custom.py @@ -0,0 +1,258 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from langchain_community.document_loaders.parsers.language.c import CSegmenter +from typing import List + +#class extened CSegmenter +class CSegmenterExtended(CSegmenter): + + _DEFINE_FUNC_RE = re.compile( + r'^\s*#\s*define\s+(?P[a-zA-Z_]\w*)\s*\((?P[^)]*)\)' + ) + + def __init__(self, code: str): + self._raw_code = code + code = self.remove_macro_blocks(code) + super().__init__(code) + self.structs_enums: List[str] = [] + + @staticmethod + def remove_comments(code: str) -> str: + # Remove all multi-line comments (/* ... */) + code = re.sub(r'/\*.*?\*/', '', code, flags=re.DOTALL) + # Remove all single-line comments (//...) + code = re.sub(r'//.*', '', code) + return code + + @staticmethod + def remove_macro_blocks(text: str) -> str: + """ + Remove macro blocks of the form: + MACRO_NAME(args) { ... } + or + MACRO_NAME(args) { ... } MACRO_NAME_END(args); + Only matches ALL UPPERCASE macro names. + """ + lines = text.splitlines() + total = len(lines) + indices_to_remove = set() + i = 0 + + macro_header = re.compile(r'^\s*([A-Z_][A-Z0-9_]*)\s*\(.*\)\s*{') + macro_end_template = r'^\s*{}\s*_END\\s*\\(.*\\)\\s*;?' + + while i < total: + line = lines[i] + match = macro_header.match(line) + if match: + macro_name = match.group(1) + start = i + brace_count = line.count('{') - line.count('}') + i += 1 + # Find the end of the code block + while i < total and brace_count > 0: + brace_count += lines[i].count('{') + brace_count -= lines[i].count('}') + i += 1 + # Optionally, look for a matching _END macro + macro_end = re.compile(macro_end_template.format(macro_name)) + if i < total and macro_end.match(lines[i]): + i += 1 + for idx in range(start, i): + indices_to_remove.add(idx) + else: + i += 1 + + return '\n'.join( + line for idx, line in enumerate(lines) + if idx not in indices_to_remove + ) + + @staticmethod + def find_top_level_blocks(c_code: str): + """ + Parses C code to find all top-level brace-enclosed blocks. + + This function uses a simple state machine to correctly handle + braces inside comments (block and line) and string/char literals. + + It returns a list of tuples, where each tuple contains the + start and end line number (1-indexed) of a top-level block. + """ + + # --- States for our machine --- + STATE_CODE = "CODE" + STATE_BLOCK_COMMENT = "BLOCK_COMMENT" + STATE_LINE_COMMENT = "LINE_COMMENT" + STATE_STRING = "STRING" + STATE_CHAR = "CHAR" + + state = STATE_CODE + brace_level = 0 + blocks = [] + + current_line = 1 + block_start_line = -1 + + # We need to iterate with an index to look ahead one character + i = 0 + while i < len(c_code): + char = c_code[i] + next_char = c_code[i + 1] if i + 1 < len(c_code) else None + + # --- State: IN_CODE --- + # This is the default state where we process code + if state == STATE_CODE: + if char == '/' and next_char == '*': + # Start of a block comment + state = STATE_BLOCK_COMMENT + i += 1 # Skip the '*' + elif char == '/' and next_char == '/': + # Start of a line comment + state = STATE_LINE_COMMENT + i += 1 # Skip the '/' + elif char == '"': + # Start of a string + state = STATE_STRING + elif char == "'": + # Start of a char literal + state = STATE_CHAR + elif char == '{': + if brace_level == 0: + # This is the start of a new top-level block + block_start_line = current_line + brace_level += 1 + elif char == '}': + brace_level -= 1 + if brace_level == 0 and block_start_line != -1: + # This is the end of a top-level block + blocks.append((block_start_line, current_line)) + block_start_line = -1 + elif brace_level < 0: + # This indicates a syntax error (extra '}') + # We'll reset to handle malformed code gracefully + brace_level = 0 + + # --- State: IN_BLOCK_COMMENT --- + # We are inside a /* ... */ comment + elif state == STATE_BLOCK_COMMENT: + if char == '*' and next_char == '/': + state = STATE_CODE + i += 1 # Skip the '/' + + # --- State: IN_LINE_COMMENT --- + # We are inside a // ... comment + elif state == STATE_LINE_COMMENT: + if char == '\n': + state = STATE_CODE + + # --- State: IN_STRING --- + # We are inside a "..." string literal + elif state == STATE_STRING: + if char == '\\': + # Skip the next character (escaped quote) + i += 1 + elif char == '"': + state = STATE_CODE + + # --- State: IN_CHAR --- + # We are inside a '...' char literal + elif state == STATE_CHAR: + if char == '\\': + # Skip the next character (escaped quote) + i += 1 + elif char == "'": + state = STATE_CODE + + # --- Update line number --- + if char == '\n': + current_line += 1 + + # Move to the next character + i += 1 + + return blocks + + @classmethod + def extract_define_functions(cls, code: str) -> List[str]: + """ + Extract function-like #define macros whose name contains at least one + lowercase letter and convert them into dummy C function segments. + + Handles multi-line macros joined by backslash continuation. + Strips ``do { ... } while(0)`` wrappers when present. + """ + lines = code.splitlines() + total = len(lines) + results: List[str] = [] + i = 0 + + while i < total: + m = cls._DEFINE_FUNC_RE.match(lines[i]) + if not m or m.group('name').isupper(): + i += 1 + continue + + name = m.group('name') + args = m.group('args').strip() + + macro_lines = [lines[i]] + while macro_lines[-1].rstrip().endswith('\\') and i + 1 < total: + i += 1 + macro_lines.append(lines[i]) + + full_line = ' '.join( + ln.rstrip().rstrip('\\').strip() for ln in macro_lines + ) + + header_re = re.compile( + r'#\s*define\s+' + re.escape(name) + r'\s*\([^)]*\)\s*' + ) + body = header_re.sub('', full_line, count=1).strip() + + do_while_re = re.compile( + r'^do\s*\{(?P.*)\}\s*while\s*\(\s*0\s*\)\s*;?\s*$', + re.DOTALL, + ) + dw = do_while_re.match(body) + if dw: + body = dw.group('inner').strip() + + dummy = f"void {name}({args}) {{ {body} }}" + results.append(dummy) + i += 1 + + return results + + def extract_functions_classes(self) -> List[str]: + segments = super().extract_functions_classes() + for i, seg in enumerate(segments): + segments[i] = self.remove_comments(seg) + hidden_segments = [] + for seg in segments: + blocks = self.find_top_level_blocks(seg) + if len(blocks) > 1: + for _, block2 in blocks: + seg2 = "\n".join(seg.splitlines()[block2:]) + new_seg = seg2.lstrip() + hidden_segments.append(new_seg) + break # support only one segment into 2 segments we only add hidden segments into segments list that might be functions + segments.extend(hidden_segments) + segments.extend(self.extract_define_functions(self._raw_code)) + return segments + + \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/chain_of_calls_retriever.py b/src/exploit_iq_commons/utils/chain_of_calls_retriever.py new file mode 100644 index 000000000..7f61f5a49 --- /dev/null +++ b/src/exploit_iq_commons/utils/chain_of_calls_retriever.py @@ -0,0 +1,727 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import time +from collections import defaultdict, deque +from dataclasses import dataclass, field +from pathlib import Path +from typing import List + +from langchain_core.documents import Document + +from exploit_iq_commons.utils.chain_of_calls_retriever_base import ChainOfCallsRetrieverBase, \ + calculate_hashable_string_for_function +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.dep_tree import ROOT_LEVEL_SENTINEL, DependencyTree, Ecosystem +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers_factory import ( + get_language_function_parser, +) +from exploit_iq_commons.utils.standard_library_cache import StandardLibraryCache + +logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") + + +@dataclass +class _SearchCtx: + """Per-search mutable state, created fresh for each get_relevant_documents call.""" + found_path: bool = False + exclusions: defaultdict = field(default_factory=lambda: defaultdict(list)) + last_visited_parent_package_indexes: dict = field(default_factory=dict) + last_visited: dict = field(default_factory=dict) + tree_additions: dict = field(default_factory=dict) + + +def is_likely_macro_block(segment: str) -> bool: + lines = segment.strip().splitlines() + if not lines: + return True + first_line = lines[0].strip() + # If no '{' at all, not a function + if '{' not in segment: + return True + # If segment is very short (less than 3 lines), likely not a function + if len(lines) < 3: + return True + # Macro assignment: MACRO(...) = { ... } + if re.match(r'^[A-Z_][A-Z0-9_]*\s*\(.*\)\s*=\s*{', first_line): + return True + # Macro end: ..._END(...) + if re.match(r'.*_END\s*\(', first_line): + return True + # Macro invocation: ALLCAPS or _ and ( -- but allow if looks like function signature + # If the first line ends with ')' and the next non-empty line is '{', it's a function signature + if re.match(r'^[A-Z_][A-Z0-9_]*\s*\(', first_line): + # Check if it's a function signature + if first_line.rstrip().endswith(')'): + # Look for '{' on the next non-empty line + for line in lines[1:]: + if line.strip() == '{': + return False # It's a function! + # Or, '{' at end of first line + if first_line.rstrip().endswith('){') or first_line.rstrip().endswith(') {'): + return False + return True + return False + + +class ChainOfCallsRetriever(ChainOfCallsRetrieverBase): + """A ChainOfCall retriever that Knows how to perform a deep search, looking for a function usage, whether it's being + called from the application code base or not. + """ + + def __init__(self, documents: List[Document], ecosystem: Ecosystem, manifest_path: Path): + """ + + :param documents: + List of documents containing the functions/methods and classes/types of the application code + + application dependencies' code + :param ecosystem: Ecosystem + The programming language of the list of documents + :param manifest_path: str + The manifest of the application, defining all the direct and transitive dependencies of the application, being + used here to build the dependency tree for a more efficient lookup and search. + """ + + logger.debug("Creating Chain of Calls Retriever") + logger.debug("Starting building Chain of Calls Retriever") + self.ecosystem = ecosystem + logger.debug("Chain of Calls Retriever - creating dependency tree") + # Build dependency tree based on the parameter programming language/package manager. + self.dependency_tree = DependencyTree(ecosystem=ecosystem) + logger.debug("Chain of Calls Retriever - get language parser") + self.language_parser = get_language_function_parser(ecosystem, self.dependency_tree) + self.manifest_path = manifest_path + # A dependency tree object is a must, because otherwise, the search is much more expensive and not efficient. + if self.dependency_tree.builder is None: + raise RuntimeError("Couldn't continue as dependency tree weren't generated") + + self.tree_dict = dict() + + # Build a dependency tree using the dependency tree builder logic. + tree = self.dependency_tree.builder.build_tree(manifest_path=manifest_path) + for package, parents in tree.items(): + parents.extend([package]) + self.tree_dict[package] = parents + self.supported_packages = list(self.tree_dict.keys()) + logger.debug("Chain of Calls Retriever - populating functions documents") + + allowed_files_extensions = self.language_parser.supported_files_extensions() + # filter out unsupported files extensions. + filtered_documents = [doc for doc in documents + if any([ext for ext in allowed_files_extensions if str(doc.metadata['source']) + .endswith(ext)])] + # Filter out all documents but full source docs. + self.documents_of_full_sources = {doc.metadata.get('source'): doc for doc in filtered_documents + if doc.metadata.get('content_type') == 'simplified_code'} + # filter out full code documents and functions/methods docs, retaining only types/classes docs + self.documents_of_types = [doc for doc in filtered_documents + if self.language_parser.is_doc_type(doc)] + + # Apply macro filtering only for C/C++ languages + if not self.language_parser.is_language_supports_macro(): + no_macro_documents = filtered_documents + logger.debug(f"skipping macro check for {self.ecosystem}, using all filtered documents") + else: + no_macro_documents = [doc for doc in filtered_documents if not is_likely_macro_block(doc.page_content)] + logger.debug(f"no_macro_documents len : {len(no_macro_documents)}") + + if not self.language_parser.is_script_language(): + # filter out types and full code documents, retaining only functions/methods documents in this attribute. + self.documents = [doc for doc in no_macro_documents if self.language_parser.is_function(doc)] + self.documents_of_functions = self.documents + else: + self.documents = filtered_documents + self.documents_of_functions = [doc for doc in self.documents + if self.language_parser.is_function(doc)] + + logger.debug(f"self.documents len : {len(self.documents)}") + logger.debug("Chain of Calls Retriever - retaining only types/classes docs " + "documents_of_types len %d", len(self.documents_of_types)) + logger.debug("Chain of Calls Retriever - after documents_of_full_sources") + + # Constructing a map of types and classes to their attributes/members/fields + self.types_classes_fields_mapping = self.language_parser.parse_all_type_struct_class_to_fields(self.documents_of_types) + # Create a data structure containing dict of key=(function_name@source_file),value = dict of + # local variables names mapped to (types, (values or expressions)) + self.functions_local_variables_index = self.language_parser.create_map_of_local_vars(self.documents_of_functions) + logger.debug("Chain of Calls Retriever - after functions_local_variables_index") + + if not self.language_parser.is_search_algo_dfs(): + self.sort_docs = self.__group_docs_by_pkg() + + def _resolve_tree_key(self, package: str, ctx: _SearchCtx) -> str | None: + """Find the canonical tree_dict key for a package name. + Tries exact match first, then falls back to is_same_package + for ecosystems with name normalization (e.g. Python PEP 503).""" + if package in self.tree_dict or package in ctx.tree_additions: + logger.debug("_resolve_tree_key: '%s' exact match", package) + return package + for tree_key in self.tree_dict: + if self.language_parser.is_tree_key_match(package, tree_key): + logger.debug("_resolve_tree_key: '%s' resolved to '%s' via is_tree_key_match (tree_dict)", package, tree_key) + return tree_key + for tree_key in ctx.tree_additions: + if self.language_parser.is_tree_key_match(package, tree_key): + logger.debug("_resolve_tree_key: '%s' resolved to '%s' via is_tree_key_match (tree_additions)", package, tree_key) + return tree_key + logger.debug("_resolve_tree_key: '%s' NOT found in tree_dict (%d keys) or tree_additions (%d keys)", + package, len(self.tree_dict), len(ctx.tree_additions)) + return None + + def _get_parents(self, package: str, ctx: _SearchCtx) -> list | None: + """Look up parents for a package, checking both the immutable tree_dict + and any packages added during the current search (ctx.tree_additions).""" + resolved = self._resolve_tree_key(package, ctx) + if resolved is None: + logger.debug("_get_parents: '%s' has no resolved key, returning None", package) + return None + parents = self.tree_dict.get(resolved) + if parents is not None: + logger.debug("_get_parents: '%s' resolved='%s' -> tree_dict parents=%s", package, resolved, parents) + return parents + parents = ctx.tree_additions.get(resolved) + logger.debug("_get_parents: '%s' resolved='%s' -> tree_additions parents=%s", package, resolved, parents) + return parents + + def __group_docs_by_pkg(self) -> dict[str, list[Document]]: + """ + sort docs by pkg name + """ + t0 = time.time() + sort_docs = defaultdict(list) + for doc in self.documents: + pkgs = self.language_parser.get_package_names(doc) + pkg_name = pkgs[0] + sort_docs[pkg_name].append(doc) + t1 = time.time() - t0 + logger.debug("PROFILE: sort all docs (%d) elaps %.3f", len(self.documents), t1) + return sort_docs + + def __find_caller_function_dfs(self, document_function: Document, function_package: str, + ctx: _SearchCtx) -> Document: + """ + This method gets function and package as arguments, search and return a caller function of a package, if exists + :param document_function: the document containing the function code and signature + :param function_package: the package name containing the function + :param ctx: per-search mutable state + :return: a single document of a function that is calling document_function, or None if not found + """ + package_names = self.language_parser.get_package_names(document_function) + direct_parents = list() + # gets list of all direct parents of function + for package_name in package_names: + parents = self._get_parents(package_name, ctx) + if parents: + direct_parents.extend(parents) + function_name_to_search = self.language_parser.get_function_name(document_function) + if not function_name_to_search: + return None + if function_name_to_search == self.language_parser.get_constructor_method_name(): + function_name_to_search = self.language_parser.get_class_name_from_class_function(document_function) + function_file_name = document_function.metadata.get('source') + relevant_docs_to_search_in = list() + last_visited_package_index = (ctx.last_visited_parent_package_indexes + .get(calculate_hashable_string_for_function(function_file_name, + function_name_to_search), 0)) + package_exclusions = ctx.exclusions[function_package] + # Search for caller functions only at parents according to dependency tree. + for package in direct_parents[last_visited_package_index:]: + sources_location_packages = True + package_parents = self._get_parents(package, ctx) + if package_parents and package_parents[0] == ROOT_LEVEL_SENTINEL: + sources_location_packages = False + + possible_docs = self.get_possible_docs(function_name_to_search, package, + package_exclusions, + sources_location_packages, + frozenset(), + dict()) + # Collect all potential caller functions + for doc in self.get_functions_for_package(package_name=package, + documents=possible_docs, + sources_location_packages=sources_location_packages, + function_to_search=function_name_to_search, + callee_function_file_name=function_file_name): + relevant_docs_to_search_in.append(doc) + # Perform the search only on the subset of potential caller functions + for doc in relevant_docs_to_search_in: + function_is_being_called = self.language_parser.search_for_called_function(caller_function=doc, + callee_function_name= + function_name_to_search, + callee_function=document_function, + callee_function_package= + function_package, + code_documents= + self.documents_of_full_sources, + type_documents= + self.documents_of_types, + callee_function_file_name= + function_file_name, + fields_of_types= + self.types_classes_fields_mapping, + functions_local_variables_index= + self.functions_local_variables_index, + documents_of_functions= + self.documents_of_functions) + + # If current document is found to be calling document_function in function_package, then return it as a + # match, and add it to exclusions so it will not consider it when backtracking in order to prevent cycles. + if function_is_being_called: + package_exclusions.append(doc) + # update index of last scanned package for backtracking + # hashed_value = calculate_hashable_string_for_function(function_file_name, function_name_to_search) + # self.last_visited_parent_package_indexes[hashed_value] = last_visited_package_index + package_index + return doc + + # If didn't find a matching caller function document, returns None. + return None + + def _is_doc_excluded(self, doc: Document, exclusions: list[Document]) -> bool: + """ + Checks if a document is in the exclusions list based on its + function name, function body and source metadata. + """ + doc_function_content = doc.page_content.strip() + doc_source = doc.metadata.get('source').strip() + + for exclusion_doc in exclusions: + exclusion_function_content = exclusion_doc.page_content.strip() + exclusion_source = exclusion_doc.metadata.get('source').strip() + + if doc_function_content == exclusion_function_content and doc_source == exclusion_source: + return True + return False + + + # This helper method filter out irrelevant function ( that cannot be caller functions), it filter out all + # excluded functions, and all function that their body doesn't contain the target function name to search for. + def get_possible_docs(self, function_name_to_search: str, package: str, exclusions: list[Document], + sources_location_packages: bool, + target_class_names: frozenset[str], + method_exclusions: dict) -> (list[Document], bool): + if sources_location_packages: + filter_1 = [doc for doc in self.documents if package in doc.metadata.get('source') + and self.language_parser.is_function(doc) and + not self._is_doc_excluded(doc, exclusions)] + else: + filter_1 = [doc for doc in self.documents if self.language_parser.is_root_package(doc) and + (self.language_parser.is_function(doc) or self.language_parser.is_script_language()) and + not self._is_doc_excluded(doc, exclusions)] + + if not function_name_to_search: + return [] + return [doc for doc in filter_1 if doc.page_content.__contains__(f"{function_name_to_search}(")] + + def __find_caller_functions_bfs(self, document_function: Document, function_package: str, + ctx: _SearchCtx) -> List[Document]: + """ + This method gets function and package as arguments, search and return a caller function of a package, if exists + :param document_function: the document containing the function code and signature + :param function_package: the package name containing the function + :param ctx: per-search mutable state + :return: a list of documents of functions that are calling document_function + """ + + total_start = time.time() + direct_parents = list() + # gets list of all direct parents of function + + parents = self._get_parents(function_package, ctx) + if parents is not None: + direct_parents.extend(parents) + logger.debug("__find_caller_functions_bfs: function_package='%s' parents=%s direct_parents=%s", + function_package, parents, direct_parents) + # Add same package itself to search path. + # direct_parents.extend([function_package]) + # gets list of documents to search in only from parents of function' package. + function_name_to_search = self.language_parser.get_function_name(document_function) + if not function_name_to_search: + return [] + function_file_name = document_function.metadata.get('source') + relevant_docs_to_search_in = list() + # Search for caller functions only at parents according to dependency tree. + log_entries = [] + loop_start = time.time() + try: + for package in direct_parents: + pkg_docs = self.sort_docs[package] + for doc in pkg_docs: + # for doc in self.documents: + # is_doc_in_pkg = False + # for package in direct_parents: + # if self.language_parser.is_a_package(package, doc): + # is_doc_in_pkg = True + # break + # if not is_doc_in_pkg: + # continue + + file_name = doc.metadata.get('source') + if doc.metadata.get('state') == "invalid": + continue + try: + func_name = self.language_parser.get_function_name(doc) + except ValueError: + continue + if not func_name: + continue + # check for same doc + if (function_name_to_search == func_name) and (file_name == function_file_name): + continue + # same function name different files ? + # if (function_name_to_search == func_name): + # logger.debug(f"same func name {function_name_to_search}") + + if func_name == "main": + file_path = Path(function_file_name) + # Get all the parts of the path + path_parts = file_path.parts + if self.language_parser.dir_name_for_3rd_party_packages() in path_parts: + continue + if doc.page_content.__contains__(f"{function_name_to_search}("): + last_visited = (ctx.last_visited.get( + calculate_hashable_string_for_function(file_name, func_name), 0)) + if last_visited == 0: + found = self.language_parser.search_for_called_function( + caller_function=doc, + callee_function_name=function_name_to_search, + callee_function=document_function, + callee_function_package=function_package, + code_documents=self.documents_of_full_sources, + type_documents=self.documents_of_types, + callee_function_file_name=function_file_name, + fields_of_types=self.types_classes_fields_mapping, + functions_local_variables_index=self.functions_local_variables_index, + documents_of_functions= + self.documents_of_functions) + + if found and self.language_parser.is_call_allowed( pkg_docs, doc, document_function): + log_entries.append((file_name, func_name, function_name_to_search)) + relevant_docs_to_search_in.append(doc) + if self.language_parser.is_root_package(doc): + return relevant_docs_to_search_in + except ValueError as ex: + logger.error("doc %s / %s", doc, ex) + + loop_elapsed = time.time() - loop_start + logger.debug(f"[PROFILE] __find_caller_functions main for-loop took {loop_elapsed:.3f} seconds") + # logger.debug table-style summary + logger.debug("\nFunction Match Summary:") + logger.debug("-" * 100) + logger.debug(f"{'No.':<5} {'File':<60} {'Function':<30} {'Target'}") + logger.debug("-" * 100) + + for idx, (file_name, func_name, function_name_to_search) in enumerate(log_entries, 1): + logger.debug(f"{idx:<5} {file_name:<60} {func_name:<30} {function_name_to_search}") + + logger.debug("-" * 100) + total_elapsed = time.time() - total_start + logger.debug(f"[PROFILE] __find_caller_functions total time took {total_elapsed:.3f} seconds") + + # If didn't find a matching caller function document, returns None. + return relevant_docs_to_search_in + + + def _breadth_first_search(self, matching_documents: List[Document], target_function_doc: Document, + current_package_name: str, ctx: _SearchCtx) -> tuple[List[Document], bool]: + # main loop. + file_counter = 0 + q = deque() + q.append(target_function_doc) + loop_start = time.time() + while q: + target_doc = q.popleft() + + target_pkg = self.language_parser.get_package_names(target_doc)[0] + if target_doc.metadata.get('state') == "invalid": + logger.debug("get_relevant_documents: invalid function %s", target_doc.metadata['source']) + continue + function_name = self.language_parser.get_function_name(target_doc) + if not function_name: + continue + + function_file = target_doc.metadata.get('source') + hashed_value = calculate_hashable_string_for_function(function_file, function_name) + if hashed_value in ctx.last_visited: + continue + + ctx.last_visited[hashed_value] = 1 + logger.debug("%d:file:%s, func_name : %s , pkg:%s queue len %d", + file_counter, target_doc.metadata['source'], function_name, target_pkg, len(q)) + + file_counter += 1 + # Find a caller function and containing package in the dependency tree according to hierarchy + sub_start = time.time() + found_documents = self.__find_caller_functions_bfs(document_function=target_doc, + function_package=target_pkg, + ctx=ctx) + sub_elapsed = time.time() - sub_start + logger.debug(f"[PROFILE] __find_caller_functions took {sub_elapsed:.3f} seconds") + # If found, then add it to path + if found_documents is not None: + for doc_candidate in found_documents: + # If the function is in the application ( root package), then we finished and found such a path. + if self.language_parser.is_root_package(doc_candidate): + matching_documents.append(doc_candidate) + ctx.found_path = True + break + # Otherwise, we continue to search for callers for the current found function, in order to extend + # the chain of calls and potentially find a path from application to the vulnerable + # function in input package + else: + q.append(doc_candidate) + + if ctx.found_path: + break + loop_elapsed = time.time() - loop_start + logger.debug(f"[PROFILE] Main loop in get_relevant_C_documents took {loop_elapsed:.3f} seconds") + # When the loop is finished, return list of documents ( path) and boolean indicating whether a path was + # found or not. + logger.debug("get_relevant_documents: result %s", ctx.found_path) + logger.debug("get_relevant_documents: docs %s", matching_documents) + return matching_documents, ctx.found_path + + def _depth_first_search(self, matching_documents: List[Document], target_function_doc: Document, + current_package_name: str, ctx: _SearchCtx) -> tuple[List[Document], bool]: + """Execute depth-first search with backtracking strategy.""" + end_loop = False + + # main loop. + while not end_loop: + # Find a caller function and containing package in the dependency tree according to hierarchy + found_document = self.__find_caller_function_dfs(document_function=target_function_doc, + function_package=current_package_name, + ctx=ctx) + # If found, then add it to path + if found_document: + matching_documents.append(found_document) + # If the function is in the application ( root package), then we finished and found such a path. + if self.language_parser.is_root_package(found_document): + end_loop = True + ctx.found_path = True + # Otherwise, we continue to search for callers for the current found function, in order to extend + # the chain of calls and potentially find a path from application to the vulnerable + # function in input package + else: + target_function_doc = found_document + # extract package name from function document + current_package_name = self.__determine_doc_package_name(target_function_doc, ctx) + else: + if len(matching_documents) == 1: + end_loop = True + # Backtrack - means that we're going back because current function has no callers anywhere, so we + # need to remove it, and continue with other possible potential called functions from its caller + else: + dead_end_node = matching_documents.pop() + ctx.exclusions[current_package_name].append(dead_end_node) + target_function_doc = matching_documents[-1] + current_package_name = self.__determine_doc_package_name(target_function_doc, ctx) + return matching_documents, ctx.found_path + + # This method is the entry point for the chain_of_calls_retriever transitive search, it gets a query, + # in the form of "package_name, function", and returns a 2-tuple of (list_of_documents_in_path, bool_result). + # if bool_result is True, then list_of_documents_in_path containing a list of documents that is being a chain of + # calls from application to input function in the input package. + def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: + """Sync implementations for retriever.""" + ctx = _SearchCtx() + query = query.splitlines()[0].replace('"', '').replace("'", "").replace("`", "").strip() + (package_name, function) = tuple(query.split(",")) + before, sep, after = function.rpartition('.') + if sep: + class_name = before.rsplit('.', 1)[-1] + function = after + else: + class_name = None + if class_name and '/' in class_name: + package_name = f"{package_name}/{class_name}" + class_name = None + found_package = False + matching_documents = [] + standard_libs_cache = StandardLibraryCache.get_instance() + # If it's a standard library package, then skip checking the package in dependency tree. + if not standard_libs_cache.is_standard_library(package_name, self.ecosystem): + # Check if input package is in dependency tree + for package in self.tree_dict: + if self.language_parser.is_tree_key_match(package_name, package): + package_name = package + found_package = True + break + # If it's , then create a document for it. + if found_package: + target_function_doc = self.__find_initial_function(function, package_name=package_name, + documents=self.documents, + ctx=ctx, + class_name=class_name) + if not target_function_doc and self.language_parser.get_constructor_method_name(): + target_function_doc = self.__find_initial_function(function_name=self.language_parser.get_constructor_method_name(), + package_name=package_name, + documents=self.documents, + ctx=ctx, + class_name=function) + + # If not, there is a chance that the package is some standard library in the ecosystem. + else: + # Try to create dummy package for ecosystem standard library function, in such case, build a document for + # vulnerable function in standard lib package of language. + page_content = self.language_parser.get_dummy_function(function) + if page_content is None: + return matching_documents, ctx.found_path + if class_name: + page_content = page_content + f'\n{self.language_parser.get_comment_line_notation()}(class: {class_name})' + target_function_doc = Document(page_content=page_content + , metadata={"source": package_name, + "ecosystem": self.ecosystem}) + + importing_docs = self.language_parser.document_imports_package(self.documents_of_full_sources, re.escape(package_name)) + + root_package = [key for (key, value) in self.tree_dict.items() if ROOT_LEVEL_SENTINEL in value] + prefix_of_3rd_parties_libs = self.language_parser.dir_name_for_3rd_party_packages() + # find all parents ( all importing packages) of the ibput package so we'll have candidate pkgs to search in. + parents = list({self.language_parser.get_package_names(doc)[-1] for doc in importing_docs if + doc.metadata['source'].startswith( + prefix_of_3rd_parties_libs) and self.language_parser.get_package_names(doc)[-1] + in self.tree_dict.keys()}) + for doc in importing_docs: + if not doc.metadata.get('source').startswith(prefix_of_3rd_parties_libs): + parents.append(root_package[0]) + break + ctx.tree_additions[package_name] = parents + end_loop = False + current_package_name = package_name + # If an initial document (that represents the vulnerable input function in the input package) was created + # then add it to the path and start constructing path that comprise a chain of calls targeting this function. + if target_function_doc: + matching_documents.append(target_function_doc) + # Otherwise, don't even start the process as the target function is not in dependencies tree or not a standard + # library + else: + end_loop = True + logger.error("Cannot find initial function=%s, in package=%s", function, package_name) + if end_loop: + return matching_documents, ctx.found_path + + if self.language_parser.is_search_algo_dfs(): + matching_documents, ctx.found_path = self._depth_first_search( + matching_documents, target_function_doc, current_package_name, ctx) + else: + matching_documents, ctx.found_path = self._breadth_first_search( + matching_documents, target_function_doc, current_package_name, ctx) + + # When the loop is finished, return list of documents ( path) and boolean indicating whether a path was + # found or not. + return matching_documents, ctx.found_path + + def __determine_doc_package_name(self, target_function_doc, ctx: _SearchCtx): + candidates = self.language_parser.get_package_names(target_function_doc) + for package_name in candidates: + if package_name in self.tree_dict or package_name in ctx.tree_additions: + logger.debug("__determine_doc_package_name: candidates=%s, exact match '%s'", candidates, package_name) + return package_name + for package_name in candidates: + resolved = self._resolve_tree_key(package_name, ctx) + if resolved is not None: + logger.debug("__determine_doc_package_name: candidates=%s, '%s' resolved to '%s' via is_same_package", + candidates, package_name, resolved) + return resolved + if not candidates: + logger.warning("__determine_doc_package_name: no candidates returned for doc %s", target_function_doc) + return None + fallback = candidates[0] + logger.debug("__determine_doc_package_name: no match in tree, fallback='%s' from candidates=%s", + fallback, candidates) + return fallback + + def __find_initial_function(self, function_name: str, package_name: str, documents: list[Document], + ctx: _SearchCtx, class_name: str = None) -> Document: + + if self.language_parser.is_search_algo_dfs(): + pkg_docs = documents + else: + pkg_docs = self.sort_docs[package_name] + + relevant_docs = self.language_parser.filter_docs_by_func_pkg_name(function_name, package_name, pkg_docs) + if class_name: + relevant_docs = [doc for doc in relevant_docs if doc.page_content.endswith( + f'{self.language_parser.get_comment_line_notation()}(class: {class_name})')] + + package_exclusions = ctx.exclusions[package_name] + #for index, document in enumerate(get_functions_for_package(package_name, relevant_docs, language_parser)): + from itertools import chain + for document in chain( + self.get_functions_for_package(package_name, relevant_docs, sources_location_packages=True), + self.get_functions_for_package(package_name, relevant_docs, sources_location_packages=False), + ): + doc_func_name = self.language_parser.get_function_name(document) + if not doc_func_name: + logger.warning("Skipping document with empty function name: %s", document.metadata.get('source', '')) + continue + if function_name.lower() == doc_func_name.lower(): + package_exclusions.append(document) + return document + + if self.language_parser.create_dummy_for_standard_lib(package_name): + dummy_function_doc = Document(page_content=f"func {function_name + '()' + '{}'}", + metadata={ + "source": self.language_parser.dir_name_for_3rd_party_packages() + "/" + package_name + "/dummy.c", + "ecosystem": self.ecosystem, + "func_name": function_name, + "pkg_name": package_name}) + return dummy_function_doc + + return None + + # This method prints as a multi-line string the path of chains of calls , from the start ( first caller) to the end + # (target function). + def print_call_hierarchy(self, call_hierarchy_list: list[Document]) -> list[str]: + results = [] + for i, package_function in enumerate(reversed(call_hierarchy_list)): + packages_names = self.language_parser.get_package_names(package_function) + if len(packages_names) > 1: + maximum_length_package = max(len(packages_names[0]), len(packages_names[1])) + if maximum_length_package == len(packages_names[0]): + package_name = packages_names[0] + else: + package_name = packages_names[1] + elif len(packages_names) > 0: + package_name = packages_names[0] + else: + package_name = package_function.metadata['source'] + try: + function_name = self.language_parser.get_function_name(package_function) + if not function_name: + current_level = f"(document={package_name},depth={i})" + else: + current_level = f"(package={package_name},function={function_name},depth={i})" + except ValueError: + current_level = f"(document={package_name},depth={i})" + results.append(current_level) + + return results + + def function_called_from_caller_body(self, document: Document, function_to_search: str) -> bool: + if function_to_search.strip() == "": + return True + function_word = self.language_parser.get_function_reserved_word() + func_header_template = rf"${function_word} (\(.*\))?\s?${function_to_search}" + return (re.search(pattern=function_to_search, string=document.page_content, + flags=re.IGNORECASE | re.MULTILINE) + # verify caller function or method is not the function + and not re.search(pattern=func_header_template, + string=document.page_content, + flags=re.IGNORECASE | re.MULTILINE)) + + def document_belongs_to_package(self, document: Document, package_name: str) -> bool: + return self.language_parser.is_a_package(package_name, document) \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/chain_of_calls_retriever_base.py b/src/exploit_iq_commons/utils/chain_of_calls_retriever_base.py new file mode 100644 index 000000000..9e9555177 --- /dev/null +++ b/src/exploit_iq_commons/utils/chain_of_calls_retriever_base.py @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import abstractmethod, ABC +from pathlib import Path +from typing import List, Optional + +from langchain_core.documents import Document + +from exploit_iq_commons.utils.dep_tree import DependencyTree, Ecosystem +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser + +PARENTS_INDEX = 0 + +EXCLUSIONS_INDEX = 1 + +METHOD_EXCLUSIONS_INDEX = 2 + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + +logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") + +def calculate_hashable_string_for_function(function_file_name: str, function_name_to_search: str) -> str: + return f"{function_file_name};{function_name_to_search}" + +def get_extension_of_file(file_path: str): + extension_start = file_path.rfind(".") + return file_path[extension_start:] + +def is_function_callable(document: Document, language_parser: LanguageFunctionsParser, callee_function_file_name: str, + documents_of_full_sources: dict[str, Document]) -> bool: + return (language_parser.is_exported_function(document, documents_of_full_sources) or + document.metadata['source'].lower() == callee_function_file_name.lower()) + +class ChainOfCallsRetrieverBase(ABC): + documents: Optional[List[Document]] + documents_of_full_sources: Optional[dict] + documents_of_types: Optional[list] + documents_of_functions: list | None + language_parser: Optional[LanguageFunctionsParser] + dependency_tree: Optional[DependencyTree] + tree_dict: Optional[dict] + supported_packages: Optional[list[str]] + ecosystem: Optional[Ecosystem] + manifest_path: Optional[Path] + types_classes_fields_mapping: Optional[dict[tuple, list[tuple]]] + functions_local_variables_index: Optional[dict[str, dict]] + sort_docs: dict[str, list[Document]] + + def get_language_parser(self): + return self.language_parser + + def get_documents(self): + return self.documents + + # This helper method filter out irrelevant function ( that cannot be caller functions), it filter out all + # excluded functions, and all function that their body doesn't contain the target function name to search for. + @abstractmethod + def get_possible_docs(self, function_name_to_search: str, package: str, exclusions: list[Document], + sources_location_packages: bool, + target_class_names: frozenset[str], + method_exclusions: dict) -> (list[Document], bool): + pass + + # This method is the entry point for the chain_of_calls_retriever transitive search, it gets a query, + # in the form of "package_name, function", and returns a 2-tuple of (list_of_documents_in_path, bool_result). + # if bool_result is True, then list_of_documents_in_path containing a list of documents that is being a chain of + # calls from application to input function in the input package. + @abstractmethod + def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: + pass + + # This method prints as a multi-line string the path of chains of calls , from the start ( first caller) to the end + # (target function). + @abstractmethod + def print_call_hierarchy(self, call_hierarchy_list: list[Document]) ->list[str]: + pass + + @abstractmethod + def function_called_from_caller_body(self, document, function_to_search) -> bool: + pass + + def get_functions_for_package(self, package_name: str, documents: list[Document], + callee_function_file_name: str = "", function_to_search: str = "", sources_location_packages=True): + # Retrieve documents of packages only ( functions + code) + if sources_location_packages: + for document in documents: + doc_extension = get_extension_of_file(document.metadata.get('source')) + if (not self.language_parser.is_root_package(document) and + document.metadata.get('content_type') == 'functions_classes' and + self.language_parser.is_function(document) and + is_function_callable(document, self.language_parser, callee_function_file_name, self.documents_of_full_sources) and + self.language_parser.is_supported_file_extensions(doc_extension) and + self.document_belongs_to_package(document, package_name) and + self.function_called_from_caller_body(document, function_to_search)): + yield document + # Retrieve documents of application only ( functions + code) + else: + for document in documents: + doc_extension = get_extension_of_file(document.metadata.get('source')) + if (self.language_parser.is_root_package(document) + and self.language_parser.is_supported_file_extensions(doc_extension) + and self.function_called_from_caller_body(document, function_to_search)): + if (self.language_parser.is_script_language() or + (document.metadata.get('content_type') == 'functions_classes' and + self.language_parser.is_function(document))): + yield document + + @abstractmethod + def document_belongs_to_package(self, document: Document, + package_name: str) -> bool: + pass \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/chain_of_calls_retriever_factory.py b/src/exploit_iq_commons/utils/chain_of_calls_retriever_factory.py new file mode 100644 index 000000000..c80933d38 --- /dev/null +++ b/src/exploit_iq_commons/utils/chain_of_calls_retriever_factory.py @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +from typing import List + +from langchain_core.documents import Document + +from exploit_iq_commons.data_models.input import SourceDocumentsInfo +from exploit_iq_commons.utils.chain_of_calls_retriever import ChainOfCallsRetriever +from exploit_iq_commons.utils.chain_of_calls_retriever_base import ChainOfCallsRetrieverBase +from exploit_iq_commons.utils.dep_tree import Ecosystem +from exploit_iq_commons.utils.java_chain_of_calls_retriever import JavaChainOfCallsRetriever + +def get_chain_of_calls_retriever(ecosystem: Ecosystem, documents: List[Document], manifest_path: Path, query: str, + code_source_info: SourceDocumentsInfo, + uber_jar_file_threshold: int) -> ChainOfCallsRetrieverBase: + """ + Get an ecosystem(e.g programming language) parameter, and returns the right language functions parser associated + with the ecosystem. + :param code_source_info: + :param query: + :param manifest_path: + :param documents: + :param ecosystem: the desired programming language parser. + :return: + The right language functions parser associated to the ecosystem, if not exists, return ABC parent that + doesn't do anything + """ + if ecosystem == Ecosystem.JAVA: + return JavaChainOfCallsRetriever(documents, + ecosystem, + manifest_path, + query, + code_source_info, + uber_jar_file_threshold=uber_jar_file_threshold) + else: + return ChainOfCallsRetriever(documents, ecosystem, manifest_path) \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/credential_client.py b/src/exploit_iq_commons/utils/credential_client.py new file mode 100644 index 000000000..775f418db --- /dev/null +++ b/src/exploit_iq_commons/utils/credential_client.py @@ -0,0 +1,258 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import os +from cryptography.exceptions import InvalidTag +from collections.abc import Generator +from contextlib import contextmanager +from contextvars import ContextVar +from pathlib import Path + +import requests +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + +logger = LoggingFactory.get_agent_logger(__name__) + +_K8S_SERVICE_ACCOUNT_TOKEN_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/token" +AES_256_KEY_SIZE_BYTES = 32 + +_credential_id_ctx: ContextVar[str | None] = ContextVar("credential_id", default=None) + + +@contextmanager +def credential_context(credential_id: str | None) -> Generator[None]: + """ + Context manager that makes credential_id available to any code running + within this scope (including async tasks spawned from it). + + Usage:: + + with credential_context(message.credential_id): + embedder.build_vdbs(source_infos, ...) + """ + token = _credential_id_ctx.set(credential_id) + try: + yield + finally: + _credential_id_ctx.reset(token) + + +def _resolve_jwt_token(jwt_token: str | None) -> str: + """ + Resolve JWT token for authenticating with the credential backend. + + Resolution order: + 1. Provided jwt_token argument (highest priority) + 2. Kubernetes service account token (mounted at well-known path) + 3. CLIENT_JWT_TOKEN environment variable (local development fallback) + + Raises + ------ + RuntimeError + If no JWT token source is available. + """ + if jwt_token: + return jwt_token + + if Path(_K8S_SERVICE_ACCOUNT_TOKEN_PATH).is_file(): + with Path(_K8S_SERVICE_ACCOUNT_TOKEN_PATH).open(encoding="utf-8") as f: + token = f.read().strip() + if token: + logger.debug("Using Kubernetes service account token for credential backend auth") + return token + + env_token = os.environ.get("CLIENT_JWT_TOKEN") + if env_token: + logger.debug("Using CLIENT_JWT_TOKEN env var for credential backend auth") + return env_token + + raise RuntimeError( + "No JWT token available for credential backend authentication. " + "Set CLIENT_JWT_TOKEN env var or run in a Kubernetes pod with a mounted service account token." + ) + + +class CredentialNotFoundError(Exception): + """Raised when credential is not found, expired, or already consumed (HTTP 404).""" + + +class AuthenticationError(Exception): + """Raised when JWT token is invalid or missing (HTTP 401/403).""" + + +class DecryptionError(Exception): + """Raised when AES-256-GCM decryption fails (wrong key or corrupted data).""" + + +class TLSConfigurationError(RuntimeError): + """Raised when CA bundle configuration is invalid or inaccessible.""" + + +def _validate_ca_bundle(ca_bundle_path: str) -> str: + """ + Validate CA bundle file before using it for TLS verification. + + Parameters + ---------- + ca_bundle_path : str + Path to the CA bundle file (PEM format). + + Returns + ------- + str + The validated CA bundle path. + + Raises + ------ + TLSConfigurationError + If the path does not point to a regular file, the file is empty, + or the file cannot be opened due to permission errors. + """ + try: + if not Path(ca_bundle_path).is_file(): + raise TLSConfigurationError( + f"CA bundle is not a regular file: {ca_bundle_path}. " + "Set CLIENT_CA_BUNDLE env var to a valid CA certificate path." + ) + with Path(ca_bundle_path).open("rb") as f: + if not f.read(1): + raise TLSConfigurationError(f"CA bundle is empty: {ca_bundle_path}") + except OSError as exc: + raise TLSConfigurationError( + f"CA bundle not readable: {ca_bundle_path}" + ) from exc + return ca_bundle_path + + +def fetch_and_decrypt_credential( + credential_id: str, + backend_url: str, + encryption_key: str, + jwt_token: str | None = None, +) -> dict: + """ + Fetch encrypted credential from backend and decrypt it in memory. + + Parameters + ---------- + credential_id : str + Unique credential identifier (UUID). + backend_url : str + Base URL of the backend. + encryption_key : str + AES encryption key as UTF-8 string. First 32 bytes are used, + matching the backend's SecretKeySpec(keyBytes, 0, 32, "AES"). + jwt_token : str | None, optional + JWT token for authenticating with the backend. When None, the token + is resolved automatically: first from the Kubernetes service account + token file, then from the CLIENT_JWT_TOKEN environment variable. + + Returns + ------- + dict with keys: + - secret_value (str): plaintext secret (PAT token or SSH private key) + - username (str | None): git username, may be None for SSH keys + - credential_type (str): "PAT" or "SSH_KEY" + - user_id (str): owner identifier + + Raises + ------ + CredentialNotFoundError + Credential not found, expired, or already consumed (single-use). + AuthenticationError + JWT token is invalid or insufficient permissions. + DecryptionError + Decryption failed — wrong key or corrupted payload. + RuntimeError + Unexpected HTTP status or network error. + """ + resolved_token = _resolve_jwt_token(jwt_token) + url = f"{backend_url.rstrip('/')}/api/v1/credentials/{credential_id}" + headers = {"Authorization": f"Bearer {resolved_token}"} + + logger.info("Fetching credential: credential_id=%s", credential_id) + + # NOTE: SSL verification is only applicable for HTTPS endpoints. + # When running the agent locally against a non-TLS backend (e.g. http://localhost:8080), + # SSL verification is skipped automatically. + # + # Required environment variables for local debugging with private Git repositories: + # CLIENT_JWT_TOKEN=dummy_token + # CLIENT_BACKEND_URL=http://localhost:8080 + if url.startswith("https"): + ca_bundle = os.environ.get("CLIENT_CA_BUNDLE", "/app/certs/service-ca.crt") + verify_ssl = _validate_ca_bundle(ca_bundle) + else: + verify_ssl = False + + try: + response = requests.get(url, headers=headers, timeout=10, verify=verify_ssl) + except requests.RequestException as exc: + logger.error("Request failed", exc_info=True) + raise RuntimeError(f"Network error fetching credential {credential_id}") from exc + + if response.status_code == 404: + raise CredentialNotFoundError( + f"Credential not found or already consumed: credential_id={credential_id}" + ) + if response.status_code in (401, 403): + raise AuthenticationError( + f"Authentication failed fetching credential: status={response.status_code}" + ) + if not response.ok: + raise RuntimeError( + f"Unexpected error fetching credential {credential_id}: " + f"status={response.status_code}" + ) + + data = response.json() + + try: + encrypted_bytes = base64.b64decode(data["encryptedSecretValue"]) + iv_bytes = base64.b64decode(data["iv"]) + except (KeyError, ValueError) as exc: + raise DecryptionError("Invalid response payload: missing or malformed fields") from exc + + try: + # Key derivation matches Agent backend: + # new SecretKeySpec(keyBytes, 0, 32, "AES") where keyBytes = key.getBytes(UTF_8) + key = encryption_key.encode("utf-8")[:AES_256_KEY_SIZE_BYTES] + aesgcm = AESGCM(key) + plaintext_bytes = aesgcm.decrypt(iv_bytes, encrypted_bytes, None) + secret_value = plaintext_bytes.decode("utf-8") + except (ValueError, InvalidTag) as exc: + raise DecryptionError("AES-256-GCM decryption failed") from exc + except Exception as exc: + raise DecryptionError("AES-256-GCM unexpected general failure encountered during decrypting credential") from exc + + credential_type = data.get("credentialType", "") + user_id = data.get("userId", "") + + logger.info( + "Credential decrypted successfully: credential_id=%s, type=%s, user_id=%s", + credential_id, + credential_type, + user_id, + ) + + return { + "secret_value": secret_value, + "username": data.get("username"), + "credential_type": credential_type, + "user_id": user_id, + } diff --git a/src/exploit_iq_commons/utils/data_utils.py b/src/exploit_iq_commons/utils/data_utils.py new file mode 100644 index 000000000..dac8ba241 --- /dev/null +++ b/src/exploit_iq_commons/utils/data_utils.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import json +import os +import pickle +import typing + +import pandas as pd +from pandas import DataFrame + +from pydantic import BaseModel as BaseModel +from pydantic.v1 import BaseModel as BaseModelV1 + +from exploit_iq_commons.data_models.cve_intel import CveIntel +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + +logger = LoggingFactory.get_agent_logger(__name__) + +PathLike = typing.Union[str, os.PathLike] + +DEFAULT_PICKLE_CACHE_DIRECTORY: PathLike = "./.cache/am_cache/pickle" +DEFAULT_GIT_DIRECTORY: PathLike = "./.cache/am_cache/git" +VDB_DIRECTORY: PathLike = "./.cache/am_cache/vdb" + +def to_serializable_object(obj: typing.Any): + """ + Utility function for converting an object to a serializable one, useful for passing as a default to json.dumps(). + """ + # Convert Pydantic V1 objects + if isinstance(obj, BaseModelV1): + return obj.dict() + + # Convert Pydantic V2 objects + elif isinstance(obj, BaseModel): + return obj.model_dump() + + # Convert all other objects to string + else: + logger.warning("Serializing object with unsupported data type to string. Object: %s", obj) + return str(obj) + + +def to_json(obj: typing.Any) -> str: + """ + Serialize object to JSON string with json.dumps(), using the to_serializable_object function as default. + """ + return json.dumps(obj, default=to_serializable_object) + + +def safe_getattr(obj, attr_path, default=None): + """ + Safely access deeply nested attributes in an object. + + Parameters + ---------- + obj : object + The object from which to retrieve attributes. + attr_path : str + Dot-separated string representing the path to the attribute. + default : any, optional + The default value to return if the attribute does not exist. + + Returns + ------- + any + The value of the nested attribute or the default value if not found. + """ + attrs = attr_path.split(".") + for attr in attrs: + obj = getattr(obj, attr, default) + if not obj: + obj = default # Handle cases where the attribute exists but is None + if obj == default: + break + return obj + +def merge_intel_and_plugin_data_convert_to_dataframe(intel_list: list[CveIntel]) -> DataFrame: + return pd.json_normalize([ + { + "vuln_id": x.vuln_id, + "ghsa_id": x.get_ghsa_id(), + "cve_id": x.get_cve_id(), + + # Flatten model_data, then merge plugin_data values if key exists + **{ + key: f"{str(model_flat.get(key, ''))} {str(plugin_data.get(key, ''))}".strip() + for key in model_flat + } + } + + for x in intel_list + for model_flat, plugin_data in [ + (pd.json_normalize(x.model_dump(mode="json"), sep="_").to_dict(orient="records")[0], + x.parse_plugin_data(x.plugins_intel_data)) + ] + ], sep="_") + +def _sanitize_path_component(value: str) -> str: + sanitized = value.replace('//', '.').replace('/', '.').replace(':', '').replace('..', '_') + sanitized = os.path.basename(sanitized) + if not sanitized: + sanitized = hashlib.sha256(value.encode()).hexdigest()[:16] + return sanitized + + +def _build_cache_path(base_pickle_dir: str, repo_url: str, repo_ref: str, documents_name: str = "") -> str: + safe_url = repo_url.replace('//', '.').replace('/', '.').replace(':', '') + safe_ref = _sanitize_path_component(repo_ref) + cached_documents_path = f"{base_pickle_dir}/{safe_url}-{safe_ref}" + if documents_name: + cached_documents_path += "-" + _sanitize_path_component(documents_name) + resolved = os.path.realpath(cached_documents_path) + base_resolved = os.path.realpath(base_pickle_dir) + if not resolved.startswith(base_resolved + os.sep): + raise ValueError(f"Cache path escapes base directory: {cached_documents_path}") + return cached_documents_path + + +def save_to_cache(base_pickle_dir: str, repo_url: str, repo_ref: str, documents: typing.Any, documents_name: str = ""): + cached_documents_path = _build_cache_path(base_pickle_dir, repo_url, repo_ref, documents_name) + + logger.debug(f"Saved loaded Documents into Cache => {cached_documents_path}") + with open(cached_documents_path, 'wb') as doc_file: # open a text file + pickle.dump(documents, doc_file) + +def cache_exists(base_pickle_dir, repo_url, repo_ref, documents_name: str = "") -> bool: + """Check if a pickle cache file exists without loading it into memory.""" + cached_documents_path = _build_cache_path(base_pickle_dir, repo_url, repo_ref, documents_name) + return os.path.isfile(cached_documents_path) + + +def retrieve_from_cache(base_pickle_dir, repo_url, repo_ref, documents_name: str = "") -> tuple: + if not os.path.exists(base_pickle_dir): + os.makedirs(base_pickle_dir) + + cached_documents_path = _build_cache_path(base_pickle_dir, repo_url, repo_ref, documents_name) + + if os.path.isfile(cached_documents_path): + with open(cached_documents_path, 'rb') as doc_file: + documents = pickle.load(doc_file) # deserialize using load() + logger.debug(f"Cache Hit on Documents, Loaded existing docs at {cached_documents_path}") + return documents, True + + return [], False \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/dep_tree.py b/src/exploit_iq_commons/utils/dep_tree.py new file mode 100644 index 000000000..e98ed0dc2 --- /dev/null +++ b/src/exploit_iq_commons/utils/dep_tree.py @@ -0,0 +1,2017 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ast +import configparser +import glob as glob_module +import hashlib +import json +import os +import re +import shutil +import subprocess +import tarfile +import tempfile +import threading +import time +import tomllib +import urllib.error +import urllib.request +import zipfile +from abc import ABC +from abc import abstractmethod +from collections import defaultdict +from contextlib import contextmanager +from enum import Enum +from pathlib import Path +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Tuple + +from packaging.specifiers import SpecifierSet +from packaging.version import Version as _PkgVersion +from tqdm import tqdm + +from exploit_iq_commons.utils.functions_parsers.python_functions_parser import PythonLanguageFunctionsParser + +# C_DEP_LIBS_NAME moved here to avoid circular import +C_DEP_LIBS_NAME = "rpm_libs" +from collections import deque + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.java_utils import add_missing_jar_string +from exploit_iq_commons.utils.java_utils import is_maven_gav +from exploit_iq_commons.utils.java_utils import parse_depgraph_line +from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager + +logger = LoggingFactory.get_agent_logger(__name__) + +# Per-repo locking for Go operations that mutate go.mod/go.sum. +# Prevents concurrent 'go mod tidy' from corrupting shared cached repos. +_go_repo_locks: dict[str, threading.Lock] = {} +_go_repo_locks_lock = threading.Lock() + + +def _get_go_repo_lock(manifest_path) -> threading.Lock: + """Get or create a per-repo lock for serializing Go module operations.""" + key = str(manifest_path) + with _go_repo_locks_lock: + if key not in _go_repo_locks: + _go_repo_locks[key] = threading.Lock() + return _go_repo_locks[key] + + +ROOT_LEVEL_SENTINEL = 'root-top-level-agent-morpheus' + +TRANSITIVE_ENV_NAME = 'transitive_env' +INSTALLED_PACKAGES_FILE = 'installed_packages.txt' + +PYPROJECT_TOML = 'pyproject.toml' +SETUP_PY = 'setup.py' +SETUP_CFG = 'setup.cfg' +PIPFILE = 'Pipfile' +UV_LOCK = 'uv.lock' +POETRY_LOCK = 'poetry.lock' +README_MD = 'README.md' + +# Manifest formats tried in priority order when requirements.txt is absent. +# Each entry is the filename; the install strategy is determined in _install_from_best_manifest. +_PYTHON_MANIFEST_FALLBACK_ORDER = [UV_LOCK, POETRY_LOCK, PYPROJECT_TOML, SETUP_PY, SETUP_CFG, PIPFILE] + +_WALK_EXCLUDE_DIRS = frozenset({ + ".venv", + "venv", + "env", + ".env", + "node_modules", + "vendor", + "dist", + "build", + "__pycache__", + ".git", + "tests", + "test", + "fixtures", + "docs", + "site-packages", + ".tox", + ".nox", +}) + + +class Ecosystem(str, Enum): + GO = "go" + PYTHON = "python" + JAVASCRIPT = "javascript" + JAVA = "java" + C_CPP = "c" + + +GOLANG_MANIFEST = "go.mod" +JAVA_MANIFEST = "pom.xml" +JS_MANIFEST = "package.json" +PYTHON_MANIFEST = "requirements.txt" +C_CPLUSPLUS_MANIFEST_1 = "CMakeLists.txt" +C_CPLUSPLUS_MANIFEST_2 = "meson.build" +C_CPLUSPLUS_MANIFEST_3 = "Makefile" +C_CPLUSPLUS_MANIFEST_4 = "configure" +C_CPLUSPLUS_EXTENSIONS = {".c", ".cpp", ".h", ".hpp", ".cc", ".cxx"} +MANIFESTS_TO_ECOSYSTEMS = { + "go.mod": Ecosystem.GO, + "requirements.txt": Ecosystem.PYTHON, + "package.json": Ecosystem.JAVASCRIPT, + "pom.xml": Ecosystem.JAVA, + "CMakeLists.txt": Ecosystem.C_CPP, + "meson.build": Ecosystem.C_CPP, + "Makefile": Ecosystem.C_CPP, + "configure": Ecosystem.C_CPP +} + + +def detect_ecosystem(git_repo_path: Path) -> Ecosystem | None: + """Detect the programming language ecosystem from manifest files in a git repository. + + Checks for manifest files in priority order: Go, Python, JavaScript, Java, C/C++. + For C/C++, also verifies that actual source files exist. + Returns None if no recognized manifest is found. + """ + if os.path.isfile(git_repo_path / GOLANG_MANIFEST): + return MANIFESTS_TO_ECOSYSTEMS[GOLANG_MANIFEST] + if any( + os.path.isfile(git_repo_path / m) + for m in (PYTHON_MANIFEST, PYPROJECT_TOML, SETUP_PY, SETUP_CFG, UV_LOCK, POETRY_LOCK, PIPFILE) + ): + return MANIFESTS_TO_ECOSYSTEMS[PYTHON_MANIFEST] + if os.path.isfile(git_repo_path / JS_MANIFEST): + return MANIFESTS_TO_ECOSYSTEMS[JS_MANIFEST] + if os.path.isfile(git_repo_path / JAVA_MANIFEST): + return MANIFESTS_TO_ECOSYSTEMS[JAVA_MANIFEST] + c_candidates = [ + git_repo_path / C_CPLUSPLUS_MANIFEST_1, + git_repo_path / C_CPLUSPLUS_MANIFEST_2, + git_repo_path / C_CPLUSPLUS_MANIFEST_3, + git_repo_path / C_CPLUSPLUS_MANIFEST_4, + git_repo_path / "auto" / C_CPLUSPLUS_MANIFEST_4, + ] + if any(p.is_file() for p in c_candidates): + for root, dirs, files in os.walk(git_repo_path): + dirs[:] = [d for d in dirs if not d.startswith('.')] + if any(Path(f).suffix in C_CPLUSPLUS_EXTENSIONS for f in files): + return MANIFESTS_TO_ECOSYSTEMS[C_CPLUSPLUS_MANIFEST_1] + return None + + +def run_command(args: list[str], cwd: str | Path | None = None, input_data: str | None = None) -> str | None: + try: + result = subprocess.run(args, capture_output=True, text=True, cwd=cwd, input=input_data) + logger.debug('Successfully ran command: %s', args) + if result.stderr: + logger.debug("Stderr: %s", result.stderr) + return result.stdout + except Exception as e: + logger.error("Failed to run command: %s with error: %s", args, e) + + +class DependencyTreeBuilder(ABC): + """ + An abstract base class that represents an object that knows how to build a + dependency tree for a given ecosystem. Should be subclassed for each + supported ecosystem. + """ + + # Directory name where dependency source files are stored (e.g. "vendor", "node_modules"). + # Each subclass sets this to its ecosystem's convention. + DEP_SOURCE_DIR: str = "" + + @abstractmethod + # Build a sort of "upside down" tree - a dict containing mapping of each + # package to a list of all consuming packages + def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: + pass + + @abstractmethod + # A Method that knows how to install the app dependencies based on the + # manifest path and ecosystem. + def install_dependencies(self, manifest_path: Path): + pass + + +class CCppDependencyTreeBuilder(DependencyTreeBuilder): + DEP_SOURCE_DIR = C_DEP_LIBS_NAME + + # Pre-compiled regex patterns (optimization: compile once, use many times) + INCLUDE_COMBINED_RE = re.compile(r'#include\s*([<"])([^>"]+)[>"]') # Combined pattern + # Common system headers for fast-path resolution + COMMON_SYSTEM_HEADERS = { + 'stdio.h', + 'stdlib.h', + 'string.h', + 'math.h', + 'unistd.h', + 'sys/types.h', + 'sys/stat.h', + 'errno.h', + 'fcntl.h', + 'time.h' + } + #kernel lib - need only for bpf projects + KERNEL_LIBS = ["libbpf"] + KERNEL_LIBS_PATH_MAP = {"libbpf": "tools/lib/bpf/"} + + def __init__(self): + self.include_paths = [] + self.root_dir = None + self.prj_name: str = "dummy_project" + self.exclude_patterns = [ + "test", "tests", "doc", "docs", "example", "examples", "bench", "benchmark", "demo", "sample" + ] + self.C_STANDARD_LIB = "glibc" + self.RPM_LIBS_DIR = self.DEP_SOURCE_DIR + self.output_json_path = None + self.ccp_dep_tree = None + + def add_include_path(self, path: str): + """Add an include path for header resolution""" + self.include_paths.append(Path(path)) + + def should_exclude_directory(self, dir_path: str) -> bool: + """Check if a directory should be excluded""" + path_lower = dir_path.lower() + return any(pattern in path_lower for pattern in self.exclude_patterns) + + def relative_matches_end(self, full_path, rel_path): + # Normalize to avoid redundant slashes or ../ + full_path = os.path.normpath(full_path) + rel_path = os.path.normpath(rel_path) + + full_parts = full_path.split(os.sep) + rel_parts = rel_path.split(os.sep) + + return full_parts[-len(rel_parts):] == rel_parts + + def find_all_files(self, root_dir): + exclude_keywords = ["test", "tests", "doc", "docs", "example", "examples"] + files = [] + for dirpath, _, filenames in os.walk(root_dir): + # Skip any directory that contains excluded keywords + normalized_path = Path(dirpath.lower()) + # Get all the parts of the path + path_parts = normalized_path.parts + if any(kw.lower() in path_parts for kw in exclude_keywords): + continue + kernel_pkg = False + for lib in self.KERNEL_LIBS: + if lib in path_parts: + kernel_pkg = True + rel_path = self.KERNEL_LIBS_PATH_MAP[lib] + # get the files that are part of the package project + if self.relative_matches_end(dirpath, rel_path): + for fname in filenames: + if fname.endswith(('.c', '.h')): + full_path = os.path.join(dirpath, fname) + files.append(full_path) + else: + # we don't need kernel files which are not part of the + # package + for fname in filenames: + if fname.endswith(('.c', '.h')): + full_path = Path(dirpath) / fname + try: + resolved = full_path.resolve() + root_resolved = Path(root_dir).resolve() + if not str(resolved).startswith(str(root_resolved) + os.sep): + logger.warning("Skipping rename outside root: %s", full_path) + continue + except OSError: + continue + new_path = full_path.with_suffix(".k") + try: + full_path.rename(new_path) + except Exception as e: + logger.warning("Rename failed: %s → %s: %s", full_path, new_path, e) + + if not kernel_pkg: + for fname in filenames: + if fname.endswith(('.c', '.h')): + full_path = os.path.join(dirpath, fname) + files.append(full_path) + return files + + def extract_includes_from_file(self, file_path: str) -> list[dict]: + """Extract #include statements from a file (no per-header cache)""" + includes = [] + try: + with open(file_path, 'r', encoding='utf-8', errors='ignore') as f: + for line_num, line in enumerate(f, 1): + line = line.strip() + if not line.startswith('#include'): + continue + match = self.INCLUDE_COMBINED_RE.search(line) + if match: + bracket_type = match.group(1) + header = match.group(2) + include_type = ('local' if bracket_type == '"' else 'system') + if (include_type == 'system' and header in self.COMMON_SYSTEM_HEADERS): + includes.append({'header': header, 'line': line_num, 'type': 'system_common'}) + else: + includes.append({'header': header, 'line': line_num, 'type': include_type}) + except (OSError, UnicodeDecodeError): + pass + return includes + + def extract_includes_from_content(self, content: list[str]) -> list[dict]: + includes = [] + for line_num, line in enumerate(content): + line = line.strip() + if not line.startswith('#include'): + continue + match = self.INCLUDE_COMBINED_RE.search(line) + if match: + bracket_type = match.group(1) + header = match.group(2) + include_type = ('local' if bracket_type == '"' else 'system') + if (include_type == 'system' and header in self.COMMON_SYSTEM_HEADERS): + includes.append({'header': header, 'line': line_num, 'type': 'system_common'}) + else: + includes.append({'header': header, 'line': line_num, 'type': include_type}) + return includes + + def module_from_path(self, path: str) -> str: + """ + Enhanced module detection (no cache) + """ + if not self.root_dir: + return "unknown" + rel_path = Path(os.path.relpath(os.path.dirname(path), self.root_dir)) + parts = rel_path.parts + if self.RPM_LIBS_DIR in parts: + idx = parts.index(self.RPM_LIBS_DIR) + if idx + 1 < len(parts): + module = parts[idx + 1] + return module + module = self.prj_name or "ROOT" + return module + + @contextmanager + def read_file(self, file_path: str): + try: + with open(file_path, 'r', encoding='utf-8', errors='ignore') as f: + yield f + except FileNotFoundError: + yield None + + def build_dependency_tree_enhanced(self, root_dir: Path, max_workers: int = 1) -> tuple[dict[str, list[str]], dict]: + """ + Build a C/C++ dependency tree for a project directory. + + This function scans all C/C++ source and header files in the given root + directory, extracts #include dependencies, and constructs a module-level + dependency tree. It uses a single-threaded approach for file reading and + parsing, which is optimal for single-core environments. The function also + tracks and reports cache statistics for header resolution. + + Args: + root_dir (Path): The root directory of the project to analyze. + max_workers (int): Ignored in this implementation (kept for + compatibility). + + Returns: + tuple: + - dict[str, list[str]]: The dependency tree as a mapping from + module to list of dependencies. + - list[str]: List of all files processed. + - dict: Cache statistics (hits, misses, cache size). + """ + self.root_dir = root_dir + self.prj_name = self.find_project_name(str(root_dir)) + # Find all C/C++ files in the project + files = self.find_all_files(root_dir) + # Map from filename to all paths with that name + filename_to_paths = defaultdict(list) + for f in files: + filename_to_paths[os.path.basename(f)].append(f) + # Map each file to its module (project, rpm, or glibc) + file_to_module = {} + standard_lib_headers = set() + for f in files: + module = self.module_from_path(f) + file_to_module[f] = module + if module == self.C_STANDARD_LIB and f.endswith('.h'): + standard_lib_headers.add(os.path.basename(f)) + # Extract includes from each file (single-threaded) + file_includes_map = {} + all_includes = set() + for i, f in enumerate(files): + lines = [] + includes = [] + with self.read_file(f) as fileobj: + # file_content = fileobj.read() + if fileobj: + for line in fileobj: + lines.append(line) + includes = self.extract_includes_from_content(lines) + + file_includes_map[f] = includes + including_module = file_to_module[f] + for include_info in includes: + header = include_info['header'] + all_includes.add((header, f, including_module)) + # Build header-to-module ambiguity map + batch_header_cache = {} + ambiguous_headers = set() + header_to_modules = defaultdict(set) + for header, _, _ in all_includes: + candidates = filename_to_paths.get(os.path.basename(header), []) + for candidate in candidates: + mod = self.module_from_path(candidate) + header_to_modules[header].add(mod) + for header, mods in header_to_modules.items(): + if len(mods) > 1: + ambiguous_headers.add(header) + # Resolve all unique (header, including_module) pairs in bulk, with + # cache stats + cache_hits = 0 + cache_misses = 0 + for header, _, including_module in all_includes: + cache_key = ((header, including_module) if header in ambiguous_headers else header) + if cache_key in batch_header_cache: + cache_hits += 1 + continue + resolved = self._batch_resolve_header(header, including_module, filename_to_paths) + batch_header_cache[cache_key] = resolved + cache_misses += 1 + # Build the dependency tree using the resolved headers + tree = defaultdict(set) + for f in files: + current_module = file_to_module[f] + includes = file_includes_map[f] + for include_info in includes: + header = include_info['header'] + include_type = include_info['type'] + # Skip system headers and standard library + if include_type == 'system_common': + continue + if (include_type == 'system' and os.path.basename(header) in standard_lib_headers): + continue + cache_key = ((header, current_module) if header in ambiguous_headers else header) + if cache_key in batch_header_cache: + resolved_module = batch_header_cache[cache_key] + else: + resolved_module = self._batch_resolve_header(header, current_module, filename_to_paths) + if (resolved_module and resolved_module != current_module): + if (resolved_module != self.C_STANDARD_LIB and current_module != self.C_STANDARD_LIB): + tree[current_module].add(resolved_module) + tree_as_list = {k: list(v) for k, v in tree.items()} + print(f"[dep_tree] batch_header_cache: {len(batch_header_cache)} " + f"entries, hits: {cache_hits}, misses: {cache_misses}") + cache_stats = { + 'batch_header_cache_size': len(batch_header_cache), 'cache_hits': cache_hits, 'cache_misses': cache_misses + } + return tree_as_list, cache_stats + + def _batch_resolve_header(self, header: str, including_module: str, + filename_to_paths: dict[str, list[str]]) -> Optional[str]: + """ + Header resolution logic for batch/bulk mode (same as before, but no + caching here) Adds debug output for ambiguous headers (limited to 10 + examples). Only prints debug info if header or including_module contains + 'openssl' (case-insensitive). + """ + header_basename = os.path.basename(header) + candidates = filename_to_paths.get(header_basename, []) + if not candidates: + return None + if len(candidates) == 1: + return self.module_from_path(candidates[0]) + + # Try to resolve by module context + for candidate in candidates: + if self.module_from_path(candidate) == including_module: + return including_module + # Try to resolve by path match + if '/' in header: + for candidate in candidates: + if candidate.endswith(header): + return self.module_from_path(candidate) + # Prefer non-glibc + for candidate in candidates: + mod = self.module_from_path(candidate) + if mod != self.C_STANDARD_LIB: + return mod + # All candidates are glibc + return self.C_STANDARD_LIB + + # Build a sort of "upside down" tree - a dict containing mapping of each + # package to a list of all consuming packages + def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: + + source_path_obj = Path(manifest_path) + for subdir in ["include", "src"]: + candidate = source_path_obj / subdir + if candidate.exists(): + self.add_include_path(str(candidate)) + start_time = time.time() + tree, cache_stats = self.build_dependency_tree_enhanced(source_path_obj) + elapsed = time.time() - start_time + logger.info(f"Dependency tree built with {len(tree)} nodes in {elapsed:.2f} seconds.") + for clib in tree[self.prj_name]: + if clib in tree: + tree[clib].append(self.prj_name) + else: + tree[clib] = [self.prj_name] + tree[self.prj_name].append(ROOT_LEVEL_SENTINEL) + + # add standard lib to all projects + tree[self.C_STANDARD_LIB] = [] + for key in tree: + tree[self.C_STANDARD_LIB].append(key) + + return tree + + # Return only package name, without version + def extract_package_name(self, package_name: str) -> str: + return "libDummy.so" # not needed and not called in C env + + def install_dependencies(self, manifest_path: Path): + """ + Install dependencies for C/C++ projects. + Supports two modes: + 1. Traditional RPM download from public repositories + 2. Container source download from registry.redhat.io (when USE_CONTAINER_SOURCES=true) + """ + self.prj_name = self.find_project_name(str(manifest_path)) + self.root_dir = str(manifest_path) + download_path = os.path.join(manifest_path, C_DEP_LIBS_NAME) + rpm_cache_dir = RPMDependencyManager.get_instance().get_rpm_cache_dir() + #check if container image is already downloaded + container_image_file = os.path.join(download_path, "container_image.txt") + if os.path.exists(container_image_file): + with open(container_image_file, "r") as f: + container_image = f.read() + if container_image == RPMDependencyManager.get_instance().container_image: + logger.info(f"Container image {container_image} already downloaded") + return + + # Check if container source download is enabled + use_container_sources = os.getenv('USE_CONTAINER_SOURCES', 'true').lower() == 'true' + if use_container_sources: + logger.info("Container source download mode enabled") + try: + self._install_dependencies_from_container(manifest_path, rpm_cache_dir) + except Exception as e: + logger.warning(f"Container source download failed: {e}. Falling back to traditional RPM download.") + rpm_downloader = RPMDependencyManager.get_instance() + rpm_downloader.threaded_rpm_download(rpm_downloader.sbom, self.prj_name, download_path, max_workers=3) + #write the image name to file to mark rpms as downloaded + file_data = RPMDependencyManager.get_instance().container_image + with open(os.path.join(download_path, "container_image.txt"), "w") as f: + f.write(file_data) + #filter out kernel files from libbpf project by calling find_all_files method + #we want to use this before the creation of documents by the segmenter class + self.find_all_files(self.root_dir) + + def _install_dependencies_from_container(self, manifest_path: Path, download_path: str): + """Install dependencies from container sources using skopeo""" + # Validate environment variables + container_image = RPMDependencyManager.get_instance().container_image + if not container_image: + raise ValueError("CONTAINER_IMAGE must be set when USE_CONTAINER_SOURCES=true") + + username = os.getenv('REGISTRY_REDHAT_USERNAME') + password = os.getenv('REGISTRY_REDHAT_PASSWORD') + if not username or not password: + raise ValueError( + "REGISTRY_REDHAT_USERNAME and REGISTRY_REDHAT_PASSWORD must be set when USE_CONTAINER_SOURCES=true") + + logger.info(f"Downloading container sources for: {container_image}") + + # Setup authentication + self._setup_skopeo_auth(username, password) + + # Download container sources + container_sources_dir = self._download_container_sources(container_image, manifest_path) + + # Extract and organize RPMs + self._extract_and_organize_rpms(container_sources_dir, download_path, container_image) + + # Cleanup temporary files + self._cleanup_temp_files(container_sources_dir) + + def _setup_skopeo_auth(self, username: str, password: str): + """Setup skopeo authentication for registry.redhat.io""" + logger.info("Setting up skopeo authentication...") + + cmd = ['skopeo', 'login', 'registry.redhat.io', '--username', username, '--password-stdin'] + + result = subprocess.run(cmd, capture_output=True, text=True, input=password) + if result.returncode != 0: + raise RuntimeError(f"Skopeo login failed: {result.stderr}") + + logger.info("Skopeo authentication successful") + + def _download_container_sources(self, container_image: str, manifest_path: Path) -> Path: + """Download container source using skopeo with caching support""" + # Create source image name (append -source) + source_image = f"{container_image}-source" + + # Create clean directory name + container_dir = source_image.replace('/', '-').replace(':', '-').replace('@', '-') + container_sources_dir = manifest_path / "container_sources" / container_dir + + # Create directory + container_sources_dir.mkdir(parents=True, exist_ok=True) + + logger.info(f"Downloading container source: {source_image}") + + cmd = ['skopeo', 'copy', f'docker://{source_image}', f'dir:{container_sources_dir}'] + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(f"Skopeo copy failed: {result.stderr}") + + logger.info(f"Container source downloaded to: {container_sources_dir}") + return container_sources_dir + + def _extract_and_organize_rpms(self, container_sources_dir: Path, download_path: str, container_image: str): + """Extract RPMs from container sources and organize them""" + logger.info("Extracting RPMs from container sources...") + + metadata_dir = container_sources_dir / "metadata" + # Create download path if it doesn't exist + os.makedirs(metadata_dir, exist_ok=True) + + # move metadata.json to metadata directory + metadata_file = container_sources_dir / "manifest.json" + shutil.move(metadata_file, metadata_dir / "manifest.json") + version_file = container_sources_dir / "version" + shutil.move(version_file, metadata_dir / "version") + + # Extract all tar files in the container sources directory + for tar_file in container_sources_dir.glob("*"): + if tar_file.is_dir(): + logger.warning(f"Skipping non-file: {tar_file}") + continue + logger.info(f"Extracting: {tar_file.name}") + try: + with tarfile.open(str(tar_file)) as tar: + tar.extractall(str(container_sources_dir), filter='data') + except tarfile.TarError as e: + logger.warning(f"Failed to extract {tar_file}: {e}") + except Exception as e: + logger.warning(f"Error extracting {tar_file}: {e}") + + # Extract app name from container image (same logic as sync_rpms.sh) + app_name = self._extract_app_name_from_container_image(container_image) + logger.info(f"Extracted app name: {app_name}") + + # Find and copy RPMs to the download path + rpm_dir = container_sources_dir / "rpm_dir" + if rpm_dir.exists(): + logger.info(f"Found RPM directory: {rpm_dir}") + + # Copy RPMs to the standard download path, excluding the container app + for rpm_file in rpm_dir.rglob("*.rpm"): + # Skip RPMs that belong to the container app + if rpm_file.name.startswith(f"{app_name}-"): + logger.debug(f"Skipping container app RPM: {rpm_file.name}") + continue + + try: + subprocess.run(['cp', str(rpm_file), str(download_path)], check=True) + logger.debug(f"Copied RPM: {rpm_file.name} -> {download_path}") + except subprocess.CalledProcessError as e: + logger.warning(f"Failed to copy RPM {rpm_file}: {e}") + else: + logger.warning("No rpm_dir found in container sources") + + def _extract_app_name_from_container_image(self, container_image: str) -> str: + """Extract app name from container image (same logic as sync_rpms.sh)""" + # Extract the basename (last part after /) + basename = container_image.split('/')[-1] + # Remove the tag (part after :) + name_without_tag = basename.split(':')[0] + # Extract the first part before the first dash + app_name = name_without_tag.split('-')[0] + return app_name + + def _cleanup_temp_files(self, container_sources_dir: Path): + """Clean up temporary container source files (with cache preservation option)""" + preserve_cache = os.getenv('PRESERVE_CONTAINER_CACHE', 'false').lower() == 'true' + + if preserve_cache: + logger.info("Preserving container source cache (PRESERVE_CONTAINER_CACHE=true)") + return + + logger.info("Cleaning up temporary files...") + try: + # Remove the container sources directory + import shutil + shutil.rmtree(container_sources_dir) + logger.info("Temporary files cleaned up") + except Exception as e: + logger.warning(f"Failed to cleanup temporary files: {e}") + + def clear_container_cache(self, manifest_path: Path): + """Clear all cached container sources for a project""" + container_sources_dir = manifest_path / "container_sources" + if container_sources_dir.exists(): + try: + import shutil + shutil.rmtree(container_sources_dir) + logger.info(f"Cleared container cache: {container_sources_dir}") + except Exception as e: + logger.error(f"Failed to clear container cache: {e}") + else: + logger.info("No container cache found to clear") + + def find_project_name(self, root_dir="."): + cmake_path = os.path.join(root_dir, "CMakeLists.txt") + if os.path.exists(cmake_path): + with open(cmake_path, "r", encoding="utf-8", errors="ignore") as f: + for line in f: + m = re.search(r'project\s*\(\s*([A-Za-z0-9_\-]+)', line, re.IGNORECASE) + if m: + return m.group(1).lower() + meson_path = os.path.join(root_dir, "meson.build") + if os.path.exists(meson_path): + with open(meson_path, "r", encoding="utf-8", errors="ignore") as f: + for line in f: + m = re.search(r'project\s*\(\s*["\']?([A-Za-z0-9_\-]+)', line, re.IGNORECASE) + if m: + return m.group(1).lower() + makefile_path = os.path.join(root_dir, "Makefile") + if os.path.exists(makefile_path): + with open(makefile_path, "r", encoding="utf-8", errors="ignore") as f: + for line in f: + m = re.search(r'^(PROJECT_NAME|PACKAGE_NAME|PROJECT|PACKAGE)\s*[:=]\s*' + r'([A-Za-z0-9_\-]+)', line) + if m: + return m.group(2).lower() + configure_ac_path = os.path.join(root_dir, "configure.ac") + if os.path.exists(configure_ac_path): + with open(configure_ac_path, "r", encoding="utf-8", errors="ignore") as f: + for line in f: + m = re.search(r'AC_INIT\(\[([A-Za-z0-9_\-]+)\]', line) + if m: + return m.group(1).lower() + + configure_path = os.path.join(root_dir, "configure") + if os.path.exists(configure_path): + with open(configure_path, "r", encoding="utf-8", errors="ignore") as f: + for line in f: + m = re.search(r"^PACKAGE_NAME\s*[:=]\s*['\"]?([A-Za-z0-9_\-]+)['\"]?", line) + if m: + return m.group(1).lower() + return "dummy_project" + + +class GoDependencyTreeBuilder(DependencyTreeBuilder): + DEP_SOURCE_DIR = "vendor" + + def install_dependencies(self, manifest_path: Path): + self.download_go_mod_vendor(manifest_path) + + @staticmethod + def _parent_son_separator(line: str) -> list[str]: + return line.split(" ") + + def _get_parent(self, line: str): + parts = self._parent_son_separator(line) + return parts[0] + + def _get_son(self, line: str): + parts = self._parent_son_separator(line) + return parts[1] + + def __init__(self): + pass + + def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: + ## Get go version from manifest + # TODO - download go binary of this version and run the go mod graph + # using it for optimal performance. + go_version = self.determine_go_version(manifest_path) + + go_mod_tree = self.get_go_mod_graph_tree(manifest_path) + lines = go_mod_tree.splitlines() + root_package_name = self._get_parent(lines[0]) + tree = dict() + for line in lines: + parent = self.extract_package_name(self._get_parent(line)) + son = self.extract_package_name(self._get_son(line)) + if tree.get(son, None) is None: + tree[son] = [parent] + else: + tree.get(son).append(parent) + # Mark the top level for + tree[root_package_name] = [ROOT_LEVEL_SENTINEL] + return tree + + @staticmethod + def determine_go_version(manifest_path): + go_version = "" + with open(str(manifest_path) + "/go.mod") as go_mod_file: + for line in go_mod_file: + if line.startswith("go"): + go_version = line.split(" ")[1] + break + return go_version + + @staticmethod + def get_go_mod_graph_tree(manifest_path) -> str: + """ + run the go mod graph command to retrieve all dependencies from go.mod, + and their hierarchy + :param manifest_path: path to the go.mod manifest. + :return: a string representation of the go.mod graph, including + dependencies and their hierarchy. + """ + process_object: subprocess.CompletedProcess + formatted_error_msg: str + env = {**os.environ, 'GOWORK': 'off'} + try: + process_object = subprocess.run( + ["go", "mod", "graph", "-modfile", str(Path(manifest_path) / "go.mod")], + capture_output=True, + text=True, + env=env) + if process_object.returncode > 0: + process_object = subprocess.run( + ["go", "mod", "graph"], + capture_output=True, + text=True, + cwd=str(manifest_path), + env=env) + if process_object.returncode > 0: + formatted_error_msg = (f"Failed to generate dependencies tree from go.mod " + f"manifest at {manifest_path}, error details => " + f"{process_object.stderr}") + raise Exception(formatted_error_msg) + except Exception as e: + error_message_exception = (f"Failed to generate dependencies tree from because go.mod " + f"manifest wasn't found at {manifest_path}, error details => " + f"{repr(e)}") + logger.error(error_message_exception) + raise e + return process_object.stdout + + @staticmethod + def get_resolved_modules(manifest_path) -> dict[str, str]: + """ + Run 'go list -m all' to get MVS-resolved module versions. + + Unlike go.sum (which is a checksum file that may contain multiple versions), + this returns the actual versions selected by Go's Minimal Version Selection + algorithm that will be compiled into the binary. + + Args: + manifest_path: Path to the directory containing go.mod + + Returns: + dict mapping module paths to versions (without 'v' prefix) + + Raises: + RuntimeError: If 'go list -m all' fails + """ + env_with_gowork = {**os.environ, "GOWORK": "off"} + + # Serialize access to the same repo to prevent concurrent 'go mod tidy' + # from corrupting go.mod/go.sum in the shared cache. + repo_lock = _get_go_repo_lock(manifest_path) + with repo_lock: + try: + # First ensure go.mod/go.sum are consistent (required for repos with + # outdated or inconsistent module files in the cache) + tidy_result = subprocess.run( + ["go", "mod", "tidy"], + capture_output=True, + text=True, + cwd=manifest_path, + env=env_with_gowork, + timeout=90, + ) + tidy_failed = tidy_result.returncode != 0 + if tidy_failed: + logger.warning( + "go mod tidy failed at %s (exit %d): %s", + manifest_path, tidy_result.returncode, tidy_result.stderr.strip() + ) + + # Then get the resolved modules + # Use -mod=readonly if tidy succeeded (enforces consistency) + # Use -mod=mod if tidy failed (allows Go to update go.mod, bypasses vendor dir) + mod_flag = "-mod=mod" if tidy_failed else "-mod=readonly" + go_list_cmd = ["go", "list", mod_flag, "-m", "all"] + process_object = subprocess.run( + go_list_cmd, + capture_output=True, + text=True, + cwd=manifest_path, + env=env_with_gowork, + timeout=60, + ) + if process_object.returncode > 0: + raise RuntimeError( + f"'go list -m all' failed at {manifest_path}: {process_object.stderr}") + except subprocess.TimeoutExpired as e: + logger.error("Go command timed out at %s: %s exceeded %ds", manifest_path, e.cmd, e.timeout) + raise RuntimeError(f"Go command timed out at {manifest_path}: command exceeded {e.timeout}s") + except RuntimeError: + raise + except Exception as e: + logger.error(f"Failed to run 'go list -m all' at {manifest_path}: {repr(e)}") + raise + + packages: dict[str, str] = {} + for line in process_object.stdout.splitlines(): + parts = line.split() + if len(parts) >= 2: + module, version = parts[0], parts[1] + packages[module] = version.lstrip('v') + + return packages + + def download_go_mod_vendor(self, manifest_path): + """ + Run the 'go mod vendor' command, in order to download go' app + dependencies, in case vendor directory doesn't already exist in git repo, + it downloads it to git repo root dir where the go.mod file located. + :param manifest_path: path to go.mod manifest in repo directory. + """ + env = {**os.environ, 'GOWORK': 'off'} + subprocess.run(["go", "mod", "vendor"], + cwd=str(manifest_path), + env=env) + + def extract_package_name(self, package_name: str) -> str: + if package_name.__contains__("@"): + version_start = package_name.index("@") + return package_name[:version_start] + else: + return package_name + + +_MAVEN_VERSION_RE = re.compile(r'^\d+\.\d+\.\d+$') + + +def _extract_mvnw_maven_version(manifest_path: Path) -> str | None: + """Extract and validate the Maven version from the wrapper properties file.""" + props = manifest_path / ".mvn" / "wrapper" / "maven-wrapper.properties" + if not props.is_file(): + return None + try: + for line in props.read_text().splitlines(): + if line.startswith("distributionUrl="): + match = re.search(r'apache-maven-(\d+\.\d+\.\d+)', line) + if match and _MAVEN_VERSION_RE.match(match.group(1)): + return match.group(1) + except OSError: + pass + return None + + +def resolve_mvn_command(manifest_path: Path) -> str: + """Return the Maven command to use for *manifest_path*. + + If a ``mvnw`` wrapper exists, regenerate it from a trusted ``mvn`` + binary (using the version declared in ``maven-wrapper.properties``) + to avoid executing an untrusted script. Falls back to ``"mvn"`` + when regeneration fails or the version cannot be determined. + """ + if not (Path(manifest_path) / "mvnw").is_file(): + return "mvn" + version = _extract_mvnw_maven_version(manifest_path) + if not version: + logger.warning("Could not extract valid Maven version from wrapper properties, using system mvn") + return "mvn" + logger.info("Regenerating Maven wrapper with version %s", version) + result = subprocess.run( + ["mvn", "wrapper:wrapper", f"-Dmaven={version}"], + cwd=manifest_path, + capture_output=True, + text=True, + ) + if result.returncode == 0: + return "./mvnw" + logger.warning( + "Failed to regenerate Maven wrapper (exit %d): %s", + result.returncode, + result.stderr or result.stdout, + ) + return "mvn" + + +class JavaDependencyTreeBuilder(DependencyTreeBuilder): + DEP_SOURCE_DIR = "dependencies-sources" + + def __init__(self, query: str): + self._query = query + + def __check_file_exists(self, dir_path: str | Path, filename: str) -> bool: + """ + Return True iff `filename` exists as a regular file directly under `dir_path`. + """ + p = Path(dir_path) / filename + return p.is_file() + + def install_dependencies(self, manifest_path: Path): + mvn_command = resolve_mvn_command(manifest_path) + settings_path = os.getenv('JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH','../../../../kustomize/base/settings.xml') + source_path = self.DEP_SOURCE_DIR + + process_object = subprocess.run([mvn_command, "-s", settings_path, "dependency:copy-dependencies", "-Dclassifier=sources", + "-DincludeScope=runtime", f"-DoutputDirectory={manifest_path.resolve()}/{source_path}"], cwd=manifest_path) + + if process_object.returncode > 0: + # Remove stale node_modules owned by a different UID to prevent + # EPERM when frontend-maven-plugin runs pnpm install during build. + for child in manifest_path.rglob("node_modules"): + if child.is_dir() and child.name == "node_modules": + shutil.rmtree(child, ignore_errors=True) + logger.debug("Removed stale %s", child) + + process_object = subprocess.run([mvn_command, "clean", "install", + "-DskipTests", "-s", settings_path], cwd=manifest_path) + if process_object.returncode > 0: + formatted_error_msg = ( + f"Failed to build project" + f"manifest at {manifest_path}, error details => " + f"{process_object.stderr}" + ) + raise Exception(formatted_error_msg) + + process_object = subprocess.run([mvn_command, "-s", settings_path, + "dependency:copy-dependencies", "-Dclassifier=sources", + "-DincludeScope=runtime", f"-DoutputDirectory={manifest_path.resolve()}/{source_path}"], cwd=manifest_path) + + if process_object.returncode > 0: + formatted_error_msg = ( + f"Failed to install dependencies" + f"manifest at {manifest_path}, error details => " + f"{process_object.stderr}" + ) + raise Exception(formatted_error_msg) + + full_source_path = manifest_path / source_path + for jar in full_source_path.glob("*-sources.jar"): + if jar.stat().st_size > 0: + dest = full_source_path / jar.stem # folder named after jar + + if not dest.exists(): + dest.mkdir(exist_ok=True) + result = subprocess.run(["jar", "xf", str(jar.resolve())], cwd=dest) + if result.returncode != 0: + logger.warning("Failed to extract sources jar: %s (exit code %d)", jar, result.returncode) + else: + logger.warning("Empty sources jar (size=0), possibly corrupt: %s", jar) + + def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: + mvn_command = resolve_mvn_command(manifest_path) + settings_path = os.getenv("JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH", "../../../../kustomize/base/settings.xml") + dependency_file = manifest_path / "dependency_tree.txt" + package_name = self._query.split(",")[0] + + if is_maven_gav(package_name): + subprocess.run( + [ + mvn_command, + "com.github.ferstl:depgraph-maven-plugin:4.0.3:aggregate", + "-s", settings_path, + "-DgraphFormat=text", + "-DshowGroupIds", + "-DshowVersions", + "-DshowTypes", + "-DoutputDirectory=.", + "-DoutputFileName=dependency_tree.txt", + "-DclasspathScope=runtime", + f"-DtargetIncludes={add_missing_jar_string(package_name)}", + ], + cwd=manifest_path, + check=True, + ) + else: + subprocess.run( + [ + mvn_command, + "com.github.ferstl:depgraph-maven-plugin:4.0.3:aggregate", + "-s", settings_path, + "-DgraphFormat=text", + "-DshowGroupIds", + "-DshowVersions", + "-DshowTypes", + "-DoutputDirectory=.", + "-DclasspathScope=runtime", + "-DoutputFileName=dependency_tree.txt", + ], + cwd=manifest_path, + check=True, + ) + + with dependency_file.open("r", encoding="utf-8") as f: + lines = f.readlines() + + parent, graph = self.__build_upside_down_dependency_graph(lines) + + # Mark *all* roots, not just the first one. + # A "root" is any node with no parents in the computed parent-chain list. + # This preserves old single-root behavior and fixes multi-root / multi-parent trees. + roots = [node for node, parents in graph.items() if not parents] + for r in roots: + graph[r] = [ROOT_LEVEL_SENTINEL] + + # Backward-compatible: keep the old behavior too (harmless if already set above) + graph[parent] = [ROOT_LEVEL_SENTINEL] + + return graph + + def __build_upside_down_dependency_graph(self, dependency_lines: List[str]) -> Tuple[str, Dict[str, List[str]]]: + root: str = "" + stack: List[str] = [] + graph_sets: Dict[str, set[str]] = {} # coord -> set of direct parents + + for line in dependency_lines: + depth, coord = self.__parse_dependency_line(line) + if depth is None or coord is None: + continue + + if depth == 0: + # depgraph aggregate can emit multiple top-level roots. Keep the first as "root" + # for backward compatibility, but still record others as separate roots in graph_sets. + if not root: + root = coord + stack = [coord] + graph_sets.setdefault(coord, set()) + continue + + # trim stack so that parent is at depth-1 + while len(stack) > depth: + stack.pop() + + parent = stack[-1] if stack else None + if parent is not None: + graph_sets.setdefault(coord, set()).add(parent) # supports multiple direct parents + graph_sets.setdefault(parent, set()) + else: + graph_sets.setdefault(coord, set()) + + stack.append(coord) + + def build_parent_chain(node: str) -> List[str]: + """ + Return a flat list of all parents reachable via any path, no duplicates. + Deterministic BFS order: nearest parents outward. + """ + result: List[str] = [] + seen: set[str] = set() + + q = deque(sorted(graph_sets.get(node, ()))) + while q: + p = q.popleft() + if p in seen: + continue + seen.add(p) + result.append(p) + + for gp in sorted(graph_sets.get(p, ())): + if gp not in seen: + q.append(gp) + + return result + + graph: Dict[str, List[str]] = {coord: build_parent_chain(coord) for coord in graph_sets.keys()} + return root, graph + + def __parse_dependency_line(self, line: str) -> Tuple[Optional[int], Optional[str]]: + """ + Parse one dependency line from depgraph's graphFormat=text output. + + Expected depgraph token shape (after indentation/branch prefix): + groupId:artifactId:version:type:scope + Example from your file: + org.apache.activemq:artemis-openwire-protocol:2.28.0:bundle:compile + + We return (depth, "groupId:artifactId:version") and ignore type/scope/optional marker. + + Also tolerates Maven log prefixes like "[INFO] " if they appear. + """ + coord = parse_depgraph_line(line) + if coord is None: + return None, None + return (coord.depth, coord.gav) + +class PythonDependencyTreeBuilder(DependencyTreeBuilder): + DEP_SOURCE_DIR = TRANSITIVE_ENV_NAME + + def _ensure_uv_cache_dir(self, manifest_path: Path): + """Set UV_CACHE_DIR if not already configured. + + In OpenShift containers the default ~/.cache/uv is often read-only. + This is normally set during install_dependencies(), but when the pickle + cache is warm that method is skipped entirely. + """ + if "UV_CACHE_DIR" not in os.environ: + fallback_cache = manifest_path / ".uv_cache" + try: + fallback_cache.mkdir(exist_ok=True) + os.environ["UV_CACHE_DIR"] = str(fallback_cache) + logger.info("Set UV_CACHE_DIR=%s (home dir may be read-only)", fallback_cache) + except OSError: + logger.warning("Could not create UV_CACHE_DIR at %s", fallback_cache) + + def _ensure_venv(self, manifest_path: Path) -> str: + """Ensure transitive_env exists with a working python binary. + + Checks if bin/python exists; if not, creates the venv using the + same version-detection logic as install_dependencies. This is + called from both build_tree (where the pickle cache may be warm + and install_dependencies was skipped) and install_dependencies + itself. Using the same determine_python_version ensures both + callers produce venvs with the same Python version, so + site-packages directories are consistent across pods. + """ + venv_python = f'{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/python' + if Path(venv_python).exists(): + return venv_python + logger.warning("Venv python not found at %s — creating venv", venv_python) + self._ensure_uv_cache_dir(manifest_path) + python_version = self.determine_python_version(str(manifest_path)) + if not python_version: + import sys + python_version = f"{sys.version_info.major}.{sys.version_info.minor}" + logger.info("Python version undetermined; using current interpreter %s", python_version) + logger.info("Creating transitive_env with Python %s using uv", python_version) + run_command(['uv', 'venv', TRANSITIVE_ENV_NAME, '--python', python_version], cwd=str(manifest_path)) + return venv_python + + def build_tree(self, manifest_path: Path) -> defaultdict[Any, list]: + venv_python = self._ensure_venv(manifest_path) + run_command(['uv', 'pip', 'install', 'setuptools<81', 'deptree', '--python', venv_python]) + dependencies = run_command([venv_python, '-m', 'deptree']) + if not dependencies or not dependencies.strip(): + logger.error("deptree returned empty output — third-party dependency tree will be incomplete. " + "Check that the virtual environment at %s has packages installed.", manifest_path) + dependencies = self._build_flat_tree_from_manifest(manifest_path) + parent_stack = [] + tree = defaultdict(set) + ROOT_PROJECT = 'root_project' + tree[ROOT_PROJECT] = [ROOT_LEVEL_SENTINEL] + for line in dependencies.split(os.linesep) if dependencies else []: + level = 0 + while line.startswith(' '): + level += 1 + line = line[2:] + package = line.split('==')[0].strip().lower() + if level < len(parent_stack): + parent_stack = parent_stack[:level] + try: + tree[package].add(parent_stack[-1]) + except IndexError: + pass + parent_stack.append(package) + + # Normalize manifest names with PEP 503 for comparison against tree keys + installed_dependencies = set() + with open(manifest_path / PYTHON_MANIFEST, 'r') as manifest: + for line in manifest: + if line.strip() and not PythonLanguageFunctionsParser.is_comment_line(line): + raw_name = re.split(r"[=>< ]", line.strip())[0] + installed_dependencies.add(re.sub(r'[-_.]', '-', raw_name.lower())) + for dependency, parents in tree.items(): + if re.sub(r'[-_.]', '-', dependency) in installed_dependencies: + parents.add(ROOT_PROJECT) + tree[dependency] = list(parents) + + # Re-key from PyPI names to import names using dist-info/top_level.txt + self._import_to_pypi = {} + site_packages = self._find_site_packages(manifest_path) + if site_packages: + for pypi_key in list(tree.keys()): + if pypi_key == ROOT_PROJECT: + continue + import_names = self._find_module_dirs(pypi_key, site_packages) + if import_names: + primary_import = import_names[0].lower() + if primary_import != pypi_key: + # Re-key: move parents from PyPI key to import key + tree[primary_import] = tree.pop(pypi_key) + self._import_to_pypi[primary_import] = pypi_key + # Update parent references in other packages + for other_key, parents in tree.items(): + if pypi_key in parents: + idx = parents.index(pypi_key) + parents[idx] = primary_import + + return tree + + def _build_flat_tree_from_manifest(self, manifest_path: Path) -> str: + """Build a flat deptree-format string from requirements.txt as a fallback + when deptree cannot run (e.g. venv binary missing after pickle cache hit).""" + lines = [] + try: + with open(manifest_path / PYTHON_MANIFEST, 'r') as manifest: + for line in manifest: + line = line.strip() + if line and not PythonLanguageFunctionsParser.is_comment_line(line): + lines.append(line) + except FileNotFoundError: + logger.error("requirements.txt not found at %s — cannot build fallback tree", manifest_path) + return "" + if lines: + logger.warning("Using requirements.txt fallback for dependency tree (%d packages)", len(lines)) + return os.linesep.join(lines) + + def extract_version_from_specifier(self, specifier_str: str) -> str | None: + """Extract the most likely runtime Python version from a PEP 440 specifier string. + + Priority order: exact pin (==) -> Python 2 upper-bound (<3) -> highest >= lower + bound -> compatible release (~=) -> exclusive lower bound (>). + + Args: + specifier_str: A PEP 440 version specifier, e.g. ``">=3.8,<4.0"``. + + Returns: + A ``"major.minor"`` string such as ``"3.9"`` or ``"2.7"``, or ``None`` + when no version can be determined. + """ + if not specifier_str: + return None + try: + specifier_set = SpecifierSet(specifier_str) + + for spec in specifier_set: + if spec.operator in ("==", "==="): + return spec.version + + for spec in specifier_set: + if spec.operator in ("<", "<=") and _PkgVersion(spec.version) <= _PkgVersion("3.0"): + return "2.7" + + lower_bounds = [_PkgVersion(spec.version) for spec in specifier_set if spec.operator == ">="] + if lower_bounds: + return str(max(lower_bounds)) + + compatible = re.search(r"~=(\d+\.\d+(?:\.\d+)?)", specifier_str) + if compatible: + return compatible.group(1) + + greater = re.search(r">(\d+\.\d+)", specifier_str) + if greater: + major, minor = map(int, greater.group(1).split(".")) + return f"{major}.{minor + 1}" + + except Exception as exc: + logger.warning("Could not parse version specifier %r: %s", specifier_str, exc) + return None + + def extract_version_from_readme_hint(self, line: str) -> str | None: + """Extract a Python version from a single free-form text line. + + Matches patterns such as ``Python 3.9``, ``python3.10``, or ``py3.11``. + + Args: + line: A single line of free-form text. + + Returns: + A ``"major.minor"`` string, or ``None`` if no match is found. + """ + match = re.search(r'(?:Python|python|py)\s*(\d+\.\d+)', line, re.IGNORECASE) + return match.group(1) if match else None + + def extract_version_from_pyproject_toml(self, content: str) -> str | None: + """Extract the required Python version from a ``pyproject.toml`` file. + + Reads ``[project].requires-python`` (PEP 621) first, then falls back to + ``[tool.poetry.dependencies].python`` (Poetry). Poetry's caret operator + (``^X.Y``) is normalised to ``>=X.Y`` before parsing. + + Args: + content: Raw text content of a ``pyproject.toml`` file. + + Returns: + A ``"major.minor"`` string, or ``None`` when no Python constraint is + found or the file cannot be parsed. + """ + try: + data = tomllib.loads(content) + except tomllib.TOMLDecodeError: + logger.warning("Failed to parse pyproject.toml as TOML") + return None + + pep621 = data.get("project", {}).get("requires-python") + if pep621: + logger.debug("Found requires-python in pyproject.toml: %r", pep621) + return self.extract_version_from_specifier(pep621) + + poetry = (data.get("tool", {}).get("poetry", {}).get("dependencies", {}).get("python")) + if poetry: + logger.debug("Found tool.poetry.dependencies.python: %r", poetry) + normalized = re.sub( + r"\^(\d+)\.(\d+)", + lambda m: f">={m.group(1)}.{m.group(2)}", + str(poetry), + ) + return self.extract_version_from_specifier(normalized) + + return None + + def extract_version_from_setup_py(self, content: str) -> str | None: + """Extract the required Python version from a ``setup.py`` file. + + Parses the AST to read the ``python_requires`` keyword argument of the + ``setup()`` call. Falls back to the highest ``Programming Language :: + Python :: X.Y`` classifier when ``python_requires`` is absent. + + Args: + content: Raw text content of a ``setup.py`` file. + + Returns: + A ``"major.minor"`` string, or ``None`` when no version information + is found or the file cannot be parsed. + """ + try: + tree = ast.parse(content) + except SyntaxError: + logger.warning("Failed to parse setup.py as Python code") + return None + + for node in ast.walk(tree): + if not (isinstance(node, ast.Call) and isinstance(node.func, (ast.Name, ast.Attribute)) + and getattr(node.func, "id", None) == "setup"): + continue + + for kw in node.keywords: + if (kw.arg == "python_requires" and isinstance(kw.value, ast.Constant) + and isinstance(kw.value.value, str)): + logger.debug("Found python_requires in setup.py: %r", kw.value.value) + return self.extract_version_from_specifier(kw.value.value) + + for kw in node.keywords: + if kw.arg == "classifiers" and isinstance(kw.value, (ast.List, ast.Tuple)): + versions = [] + for elt in kw.value.elts: + if not (isinstance(elt, ast.Constant) and isinstance(elt.value, str)): + continue + m = re.match( + r"Programming Language :: Python :: (\d+\.\d+)$", + elt.value, + ) + if m: + versions.append(m.group(1)) + if versions: + logger.debug("Found Python version classifiers in setup.py: %s", versions) + return str(max(versions, key=lambda v: _PkgVersion(v))) + + return None + + def extract_version_from_readme_md(self, content: str) -> str | None: + """Extract the highest Python version hint from README content. + + Scans every line for patterns matched by + :meth:`extract_version_from_readme_hint` and returns the highest + version found. + + Args: + content: Raw text content of a ``README.md`` or ``README.rst`` file. + + Returns: + A ``"major.minor"`` string, or ``None`` when no hint is found. + """ + best: str | None = None + for line in content.splitlines(): + candidate = self.extract_version_from_readme_hint(line) + if candidate is None: + continue + if best is None or _PkgVersion(candidate) > _PkgVersion(best): + best = candidate + logger.debug("Updated best Python version hint from README: %r", best) + return best + + def extract_version_from_python_version_file(self, content: str) -> str | None: + """Extract the Python version from a ``.python-version`` file (pyenv). + + The pyenv format is a single non-empty line containing ``X.Y`` or + ``X.Y.Z``. The first non-empty line is treated as the authoritative + value; if it does not match the expected pattern the file is considered + invalid and ``None`` is returned immediately. + + Args: + content: Raw text content of a ``.python-version`` file. + + Returns: + A ``"major.minor"`` string such as ``"3.9"``, or ``None``. + """ + for line in content.splitlines(): + line = line.strip() + if not line: + continue + m = re.match(r"^(\d+)\.(\d+)(?:\.\d+)?$", line) + if m: + return f"{m.group(1)}.{m.group(2)}" + return None + return None + + def extract_version_from_setup_cfg(self, content: str) -> str | None: + """Extract the required Python version from a ``setup.cfg`` file. + + Reads ``[options] python_requires`` and delegates version resolution to + :meth:`extract_version_from_specifier`. + + Args: + content: Raw text content of a ``setup.cfg`` file. + + Returns: + A ``"major.minor"`` string, or ``None`` when no ``python_requires`` + key is present or the file cannot be parsed. + """ + parser = configparser.ConfigParser() + try: + parser.read_string(content) + except configparser.Error: + logger.warning("Failed to parse setup.cfg") + return None + specifier = parser.get("options", "python_requires", fallback=None) + if specifier: + logger.debug("Found python_requires in setup.cfg: %r", specifier) + return self.extract_version_from_specifier(specifier.strip()) + return None + + def extract_version_from_pipfile(self, content: str) -> str | None: + """Extract the required Python version from a ``Pipfile``. + + Reads ``[requires] python_full_version`` (preferred) or + ``python_version`` and truncates to ``major.minor``. + + Args: + content: Raw text content of a ``Pipfile``. + + Returns: + A ``"major.minor"`` string, or ``None`` when no version is declared + or the file cannot be parsed. + """ + try: + data = tomllib.loads(content) + except tomllib.TOMLDecodeError: + logger.warning("Failed to parse Pipfile as TOML") + return None + requires = data.get("requires", {}) + full = requires.get("python_full_version") or requires.get("python_version") + if not full: + return None + m = re.match(r"^(\d+\.\d+)", str(full)) + logger.debug("Found python version in Pipfile: %r", full) + return m.group(1) if m else None + + def determine_python_version(self, git_repo_path: str) -> str | None: + """Determine the required Python version for a cloned Git repository. + + Searches for version information using a priority chain of manifest + files. The repository root is checked first; the rest of the tree is + then walked with common non-project directories excluded to avoid false + matches from fixtures or vendored code. + + Priority order (highest to lowest): + + 1. ``.python-version`` (pyenv) + 2. ``pyproject.toml`` — ``[project].requires-python`` then Poetry + 3. ``setup.cfg`` — ``[options] python_requires`` + 4. ``setup.py`` — ``python_requires`` kwarg then classifiers + 5. ``Pipfile`` — ``[requires] python_version`` + 6. ``README.md`` / ``README.rst`` — free-text hints (weakest) + + Args: + git_repo_path: Absolute path to the root of the cloned repository. + + Returns: + A ``"major.minor"`` string such as ``"3.9"`` or ``"2.7"``, or + ``None`` when no authoritative version information is found. + """ + root = Path(git_repo_path) + + priority_chain = [ + (".python-version", self.extract_version_from_python_version_file), + ("pyproject.toml", self.extract_version_from_pyproject_toml), + ("setup.cfg", self.extract_version_from_setup_cfg), + ("setup.py", self.extract_version_from_setup_py), + ("Pipfile", self.extract_version_from_pipfile), + ("README.md", self.extract_version_from_readme_md), + ("README.rst", self.extract_version_from_readme_md), + ] + + def _try_file(path: Path, extractor) -> str | None: + if not path.is_file(): + return None + try: + version = extractor(path.read_text(encoding="utf-8", errors="replace")) + if version: + logger.info("Detected Python %s from %s", version, path) + return version + except OSError as exc: + logger.warning("Could not read %s: %s", path, exc) + return None + + for filename, extractor in priority_chain: + version = _try_file(root / filename, extractor) + if version: + return version + + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [d for d in dirnames if d not in _WALK_EXCLUDE_DIRS and not d.startswith(".")] + if Path(dirpath) == root: + continue + for filename, extractor in priority_chain: + if filename in filenames: + version = _try_file(Path(dirpath) / filename, extractor) + if version: + return version + + return None + + def _ensure_venv(self, manifest_path: Path) -> str: + """Ensure transitive_env exists with a working python binary.""" + venv_python = f'{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/python' + if Path(venv_python).exists(): + return venv_python + logger.warning("Venv python not found at %s — creating venv", venv_python) + python_version = self.determine_python_version(str(manifest_path)) + if not python_version: + import sys + python_version = f"{sys.version_info.major}.{sys.version_info.minor}" + logger.info("Python version undetermined; using current interpreter %s", python_version) + logger.info("Creating transitive_env with Python %s using uv", python_version) + run_command(["uv", "venv" ,TRANSITIVE_ENV_NAME, "--python", python_version] ,cwd=manifest_path) + return venv_python + + def install_dependencies(self, manifest_path: Path): + """Install Python dependencies for the given repository into a virtual environment. + + Calls :meth:`_ensure_venv` to select the interpreter and create the venv. + Installs packages from whichever manifest format is present, trying formats + in this priority order: + + 1. ``requirements.txt``: line-by-line install (original behaviour) + 2. ``uv.lock`` or ``poetry.lock``: ``uv export | uv pip install -r -`` + 3. ``pyproject.toml``, ``setup.py``, or ``setup.cfg``: ``uv pip install .`` + 4. ``Pipfile``: ``pipenv requirements | uv pip install -r -`` + + After installation, writes ``installed_packages.txt`` containing a + freeze-format snapshot of every package in the venv so that Code + Keyword Search can answer "is package X installed?" without source + traversal. + + Args: + manifest_path: Absolute path to the root of the cloned repository. + """ + venv_python = self._ensure_venv(manifest_path) + site_packages = self._find_site_packages(manifest_path) + + installed_via = self._install_from_best_manifest(manifest_path, venv_python, site_packages) + if installed_via: + logger.info("Installed Python dependencies via %s", installed_via) + else: + logger.warning("No supported Python manifest found in %s; transitive_env will be empty", manifest_path) + + self._write_installed_packages(manifest_path) + + def _install_from_best_manifest(self, manifest_path: Path, venv_python: str, + site_packages: Optional[Path]) -> Optional[str]: + """Try each Python manifest format in priority order; return the format name on success.""" + req_txt = manifest_path / PYTHON_MANIFEST + if req_txt.exists(): + self._install_from_requirements_txt(req_txt, manifest_path, site_packages) + return PYTHON_MANIFEST + + # Lock files: export to requirements format then pipe to uv pip install + for lock_file in (UV_LOCK, POETRY_LOCK): + if (manifest_path / lock_file).exists(): + working_dir = manifest_path + create_requirements_command = ["uv","export","--format","requirements-txt","no-dev","2>/dev/null"] + requirements_txt_result = run_command(args=create_requirements_command, cwd=working_dir) + + if requirements_txt_result is not None: + run_command(args=["uv","pip","install","-r","-", "--python", venv_python], + input_data=requirements_txt_result) + + return lock_file + + # Project manifests: uv pip install . resolves and installs all declared deps + for manifest_name in (PYPROJECT_TOML, SETUP_PY, SETUP_CFG): + if (manifest_path / manifest_name).exists(): + run_command([ "uv", "pip", "install", "." , "--python" "venv_python"] , cwd=manifest_path) + return manifest_name + + # Pipfile: requires pipenv; skip silently if not available + if (manifest_path / PIPFILE).exists(): + requirements_txt_result = run_command(["pipenv","requirements","2>/dev/null"], cwd=manifest_path) + if requirements_txt_result is not None: + run_command(["uv","pip","install","-r","-", "--python", venv_python],input_data=requirements_txt_result) + return PIPFILE + + return None + + def _install_from_requirements_txt(self, req_txt: Path, manifest_path: Path, + site_packages: Optional[Path]) -> None: + """Install dependencies line-by-line from requirements.txt (original behaviour).""" + with open(req_txt, 'r') as manifest: + for line in tqdm(manifest): + if line.strip() and not PythonLanguageFunctionsParser.is_comment_line(line): + self.install_dependency(line, manifest_path) + if site_packages: + package_name = re.split(r'[=>< \n]', line.strip())[0] + self._fallback_if_stub_only(package_name, site_packages) + + def _write_installed_packages(self, manifest_path: Path) -> None: + """Write a freeze-format snapshot of the venv to installed_packages.txt.""" + pip_full_path= f"{manifest_path}/{TRANSITIVE_ENV_NAME}/bin/pip" + pip_freeze = run_command([pip_full_path,"list","--format=freeze"]) + if pip_freeze: + (manifest_path / INSTALLED_PACKAGES_FILE).write_text(pip_freeze) + logger.info("Wrote installed packages snapshot to %s/%s", manifest_path, INSTALLED_PACKAGES_FILE) + else: + logger.warning("Could not generate installed packages list for %s", manifest_path) + + def install_dependency(self, dependency, repo_path): + dependency = dependency.strip() + valid_signs = ['==', '>=', '<=', '!='] + if not any(sign in dependency for sign in valid_signs): + dependency = dependency.replace('=', '==') + venv_python = str(Path(repo_path) / TRANSITIVE_ENV_NAME / 'bin' / 'python') + res = run_command(['uv', 'pip', 'install', dependency, '--python', venv_python]) + if not res: + logger.warning('Failed to install dependency %s', dependency) + + def _find_module_dirs(self, package_name: str, site_packages: Path) -> list[str]: + candidates = [] + try: + from importlib_metadata import distribution + dist = distribution(package_name) + top_level = dist.read_text('top_level.txt') + if top_level: + candidates.extend(top_level.strip().split('\n')) + if candidates: + return candidates + except Exception: + pass + + normalized = package_name.replace('-', '_') + for dist_dir in site_packages.glob(f'{normalized}-*.dist-info'): + top_level_file = dist_dir / 'top_level.txt' + if top_level_file.exists(): + content = top_level_file.read_text().strip() + if content: + candidates.extend(content.split('\n')) + if candidates: + return candidates + + if package_name.startswith('types-'): + base = package_name[6:] + candidates = [f'{base}-stubs', f'{base.lower()}-stubs', base, base.lower()] + elif package_name.startswith('mypy-boto3-'): + base = package_name[11:] + candidates = [f'mypy_boto3_{base}'] + elif package_name.endswith('-stubs'): + candidates = [package_name] + else: + candidates = [package_name.replace('-', '_')] + + existing = [c for c in candidates if (site_packages / c).exists()] + return existing if existing else candidates + + def _find_site_packages(self, git_repo_path) -> Optional[Path]: + pattern = str(git_repo_path / TRANSITIVE_ENV_NAME / 'lib' / '*' / 'site-packages') + matches = glob_module.glob(pattern) + if matches: + return Path(matches[0]) + logger.warning("Could not find site-packages directory") + return None + + def _is_stub_only_package(self, package_name: str, site_packages: Path) -> bool: + pkg_dir = site_packages / package_name + if not pkg_dir.exists(): + return False + py_files = glob_module.glob(str(pkg_dir / '**' / '*.py'), recursive=True) + pyi_files = glob_module.glob(str(pkg_dir / '**' / '*.pyi'), recursive=True) + so_files = glob_module.glob(str(pkg_dir / '**' / '*.so'), recursive=True) + return len(py_files) == 0 and (len(pyi_files) > 0 or len(so_files) > 0) + + def _fallback_if_stub_only(self, package_name: str, site_packages: Path): + """ + check package if stub-only, if so download from PyPI + """ + module_dirs = self._find_module_dirs(package_name, site_packages) + + if not module_dirs: + logger.debug(f"Could not find module directories for package '{package_name}'") + return + + for module_name in module_dirs: + if not self._is_stub_only_package(module_name, site_packages): + continue + + logger.info( + f"Package '{package_name}' (module: '{module_name}') is stub-only, fetching source from PyPI...") + + actual_package = package_name + if package_name.startswith('types-'): + actual_package = package_name[6:] + logger.info( + f"Detected stub package '{package_name}', will fetch source for actual package '{actual_package}'") + elif package_name.endswith('-stubs'): + actual_package = package_name[:-6] + logger.info( + f"Detected stub package '{package_name}', will fetch source for actual package '{actual_package}'") + + possible_names = [ + actual_package, + actual_package.replace('-', '_'), + actual_package.replace('_', '-'), + ] + + data = None + for name in possible_names: + try: + url = f'https://pypi.org/pypi/{name}/json' + with urllib.request.urlopen(url) as r: + data = json.loads(r.read()) + logger.info(f"Found package on PyPI as '{name}'") + break + except urllib.error.HTTPError: + continue + + if not data: + logger.warning(f"Package '{package_name}' not found on PyPI with any variant, skipping fallback") + return + + sdist = next((u for u in data['urls'] if u['packagetype'] == 'sdist'), None) + if not sdist: + logger.warning(f"No sdist found on PyPI for '{package_name}', skipping fallback") + return + + sdist_url = sdist['url'] + + try: + with tempfile.TemporaryDirectory() as tmpdir: + tarball_path = Path(tmpdir) / 'sdist.tar.gz' + urllib.request.urlretrieve(sdist_url, tarball_path) + expected_hash = sdist.get('digests', {}).get('sha256') + if expected_hash: + actual_hash = hashlib.sha256(tarball_path.read_bytes()).hexdigest() + if actual_hash != expected_hash: + raise ValueError(f"SHA256 mismatch for {package_name}: expected {expected_hash}, got {actual_hash}") + else: + logger.warning( + f"Expected SHA256 hash is missing for package '{package_name}' " + f"at {sdist_url}. Skipping integrity check." + ) + with tarfile.open(tarball_path, 'r:gz') as tar: + tar.extractall(tmpdir, filter='data') + + extracted = [d for d in Path(tmpdir).iterdir() if d.is_dir() and d.name != '__pycache__'] + if not extracted: + return + + src_root = extracted[0] + + candidates = [ + src_root / 'src' / actual_package, + src_root / actual_package, + src_root / 'src' / actual_package.replace('-', '_'), + src_root / actual_package.replace('-', '_'), + ] + + src_dir = next((c for c in candidates if c.exists()), None) + + if src_dir: + dest = site_packages / module_name + shutil.copytree(str(src_dir), str(dest), dirs_exist_ok=True) + logger.info(f"Fallback source copied for '{package_name}' to {dest}") + else: + logger.warning(f"Could not locate source dir for '{package_name}' in sdist") + except Exception as e: + logger.warning(f"Fallback failed for '{package_name}': {e}") + + +class JavaScriptDependencyTreeBuilder(DependencyTreeBuilder): + DEP_SOURCE_DIR = "node_modules" + + def build_tree(self, manifest_path: Path) -> dict[str, list[str]]: + + tree = defaultdict(set) + ROOT_PROJECT = 'root_project' + tree[ROOT_PROJECT] = [ROOT_LEVEL_SENTINEL] + package_json_path = manifest_path / 'package.json' + if not package_json_path.exists(): + logger.error(f"package.json not found at {package_json_path}") + raise FileNotFoundError(f"package.json not found at {package_json_path}") + + result = run_command(['npm', 'ls', '--json', '--all'], cwd=str(manifest_path)) + try: + npm_tree = json.loads(result) + except json.JSONDecodeError as e: + raise RuntimeError(f"Failed to parse npm ls JSON output: {e}") + self._parse_npm_tree(npm_tree, tree, ROOT_PROJECT) + + return {k: list(v) for k, v in tree.items()} + + def _parse_npm_tree(self, node: dict, tree: dict, parent: str): + """ + Recursively parse npm ls JSON output to build child->parents mapping. + + Args: + node: Current node in the npm tree (has 'name', 'version', 'dependencies') + tree: The dependency tree being built (child -> set of parents) + parent: Name of the parent package + """ + dependencies = node.get('dependencies', {}) + + for dep_name, dep_info in dependencies.items(): + tree[dep_name].add(parent) + if isinstance(dep_info, dict): + self._parse_npm_tree(dep_info, tree, dep_name) + + def install_dependencies(self, manifest_path: Path): + logger.info(f"Installing JavaScript dependencies in {manifest_path}") + res = run_command(['npm', 'install', '--ignore-scripts'], cwd=str(manifest_path)) + if not res: + logger.error(f"Failed to install JavaScript dependencies") + else: + logger.info("JavaScript dependencies installed successfully") + + +def get_dependency_tree_builder(programming_language: Ecosystem, query: str = "") -> DependencyTreeBuilder: + """ + A Factory method function that gets an programming language as argument, + and returns a DependencyTreeBuilder' instance related to the ecosystem. + :param query: The query containing the package name,function name. + :param programming_language: ecosystem to get the dependency tree builder + for. + :return: DependencyTreeBuilder instance for the ecosystem, If not + implemented, raise a NotImplementedError. + """ + if programming_language.value == Ecosystem.GO.value: + return GoDependencyTreeBuilder() + elif programming_language.value == Ecosystem.PYTHON.value: + return PythonDependencyTreeBuilder() + elif programming_language.value == Ecosystem.JAVASCRIPT.value: + return JavaScriptDependencyTreeBuilder() + elif programming_language.value == Ecosystem.JAVA.value: + return JavaDependencyTreeBuilder(query=query) + elif programming_language.value == Ecosystem.C_CPP.value: + return CCppDependencyTreeBuilder() + else: + raise NotImplementedError(f'Unsupported Programming Language for transitive search -> ' + f'{programming_language}') + + +# Maps each ecosystem to its builder class — used to build ECOSYSTEM_DEP_DIRS +# from class-level DEP_SOURCE_DIR without instantiating builders. +_ECOSYSTEM_BUILDER_MAP: dict[Ecosystem, type[DependencyTreeBuilder]] = { + Ecosystem.C_CPP: CCppDependencyTreeBuilder, + Ecosystem.GO: GoDependencyTreeBuilder, + Ecosystem.JAVA: JavaDependencyTreeBuilder, + Ecosystem.PYTHON: PythonDependencyTreeBuilder, + Ecosystem.JAVASCRIPT: JavaScriptDependencyTreeBuilder, +} + +# Dynamic mapping of ecosystem → dependency source directory prefix. +# Built from each builder's DEP_SOURCE_DIR class attribute. +ECOSYSTEM_DEP_DIRS: dict[str, str] = { + eco.value: cls.DEP_SOURCE_DIR + "/" + for eco, cls in _ECOSYSTEM_BUILDER_MAP.items() + if cls.DEP_SOURCE_DIR +} + + +class DependencyTree: + """ + A class that represents a dependency tree to access an appropriate + dependency tree builder, based on ecosystem. + """ + builder: DependencyTreeBuilder + ecosystem: Ecosystem + + def __init__(self, ecosystem: Ecosystem, query: str = ""): + """ + + :param ecosystem: the ecosystem of the dependency tree. + """ + self.ecosystem = ecosystem + try: + # gets a dependency tree builder object based on ecosystem. + self.builder = get_dependency_tree_builder(ecosystem, query) + except ValueError as err: + logger.warning("Unable to build dependency tree - %s", str(err)) + self.builder = None diff --git a/src/vuln_analysis/utils/document_embedding.py b/src/exploit_iq_commons/utils/document_embedding.py similarity index 56% rename from src/vuln_analysis/utils/document_embedding.py rename to src/exploit_iq_commons/utils/document_embedding.py index d06d16f1e..ad805cd1f 100644 --- a/src/vuln_analysis/utils/document_embedding.py +++ b/src/exploit_iq_commons/utils/document_embedding.py @@ -15,9 +15,10 @@ import copy import json -import logging import os import sys +import threading +import weakref import time import typing from hashlib import sha512 @@ -35,17 +36,23 @@ from langchain_community.vectorstores import FAISS from langchain_core.document_loaders.blob_loaders import Blob -from vuln_analysis.data_models.input import SourceDocumentsInfo -from vuln_analysis.utils.js_extended_parser import ExtendedJavaScriptSegmenter -from vuln_analysis.utils.source_code_git_loader import SourceCodeGitLoader +from exploit_iq_commons.data_models.input import SourceDocumentsInfo +from exploit_iq_commons.utils.data_utils import retrieve_from_cache, save_to_cache, DEFAULT_PICKLE_CACHE_DIRECTORY, DEFAULT_GIT_DIRECTORY, \ + VDB_DIRECTORY, PathLike +from exploit_iq_commons.utils.go_segmenters_with_methods import GoSegmenterWithMethods +from exploit_iq_commons.utils.python_segmenters_with_classes_methods import PythonSegmenterWithClassesMethods +from exploit_iq_commons.utils.java_segmenters_with_methods import JavaSegmenterWithMethods +from exploit_iq_commons.utils.javascript_extended_segmenter import ExtendedJavaScriptSegmenter +from exploit_iq_commons.utils.source_code_git_loader import SourceCodeGitLoader +from exploit_iq_commons.utils.git_utils import sanitize_git_url_for_path, get_repo_path_with_ref +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + +from exploit_iq_commons.utils.c_segmenter_custom import CSegmenterExtended if typing.TYPE_CHECKING: from langchain_core.embeddings import Embeddings # pragma: no cover -PathLike = typing.Union[str, os.PathLike] - -logger = logging.getLogger(__name__) - +logger = LoggingFactory.get_agent_logger(__name__) class MultiLanguageRecursiveCharacterTextSplitter(RecursiveCharacterTextSplitter): """ @@ -53,9 +60,9 @@ class MultiLanguageRecursiveCharacterTextSplitter(RecursiveCharacterTextSplitter """ def __init__( - self, - keep_separator: bool = True, - **kwargs, + self, + keep_separator: bool = True, + **kwargs, ) -> None: """Create a new RecursiveCharacterTextSplitter.""" super().__init__(is_separator_regex=True, keep_separator=keep_separator, **kwargs) @@ -100,14 +107,14 @@ class ExtendedLanguageParser(LanguageParser): "hpp": "cpp", } - additional_segmenters: dict[str, type[CodeSegmenter]] = {} - - if os.environ.get('ENABLE_EXTENDED_JS_PARSERS'): - additional_segmenters = { - "javascript": ExtendedJavaScriptSegmenter, - "js": ExtendedJavaScriptSegmenter, - } - + additional_segmenters = { + "go": GoSegmenterWithMethods, + "python": PythonSegmenterWithClassesMethods, + "javascript": ExtendedJavaScriptSegmenter, + "js": ExtendedJavaScriptSegmenter, + "java": JavaSegmenterWithMethods, + "c": CSegmenterExtended +} LANGUAGE_SEGMENTERS: dict[str, type[CodeSegmenter]] = { **LANGUAGE_SEGMENTERS, **additional_segmenters, @@ -117,8 +124,16 @@ def lazy_parse(self, blob: Blob) -> typing.Iterator[Document]: try: code = blob.as_string() except Exception as e: - logger.warning("Failed to read code for '%s'. Ignoring this file. Error: %s", blob.source, e) - return + # Minimal fallback: try decoding raw bytes with UTF-8 replacement, then latin-1 + try: + raw_bytes = blob.as_bytes() + try: + code = raw_bytes.decode('utf-8', errors='replace') + except Exception: + code = raw_bytes.decode('latin-1', errors='replace') + except Exception: + logger.warning("Failed to read code for '%s'. Ignoring this file. Error: %s", blob.source, e) + return language = self.language or (self.LANGUAGE_EXTENSIONS.get(blob.source.rsplit(".", 1)[-1]) if isinstance( blob.source, str) else None) @@ -132,6 +147,10 @@ def lazy_parse(self, blob: Blob) -> typing.Iterator[Document]: ) return + segmenter_cls = self.LANGUAGE_SEGMENTERS.get(language) + if segmenter_cls is not None and hasattr(segmenter_cls, "should_skip") and isinstance(blob.source, str) and segmenter_cls.should_skip(blob.source): + return + if self.parser_threshold >= len(code.splitlines()): yield Document( page_content=code, @@ -142,7 +161,7 @@ def lazy_parse(self, blob: Blob) -> typing.Iterator[Document]: ) return - segmenter = self.LANGUAGE_SEGMENTERS[language](blob.as_string()) + segmenter = self.LANGUAGE_SEGMENTERS[language](code) try: extracted_functions_classes = segmenter.extract_functions_classes() @@ -202,19 +221,77 @@ def lazy_parse(self, blob: Blob) -> typing.Iterator[Document]: ) +class _LoggingEmbeddingProxy: + """Wraps an Embeddings instance to log per-batch progress during VDB creation. + + FAISS calls embed_documents once with all texts; the NIM SDK loops + internally in batches of max_batch_size calling _embed per batch. + This proxy intercepts embed_documents and does the batching itself + so it can log progress between batches. + + NOTE: Tightly coupled with langchain_nvidia_ai_endpoints.NVIDIAEmbeddings. + Calls the private _embed(texts, model_type="passage") method directly. + Other Embeddings implementations (langchain ABC) don't expose _embed, + so this proxy will break if the embedding type changes or if NVIDIAEmbeddings + renames/removes _embed in a future version. + """ + + def __init__(self, embedding, total_chunks: int, start_time: float): + self._embedding = embedding + self._total_chunks = total_chunks + self._start_time = start_time + self._embedded = 0 + + def embed_documents(self, texts): + batch_size = getattr(self._embedding, "max_batch_size", 128) + all_embeddings = [] + for i in range(0, len(texts), batch_size): + batch = texts[i:i + batch_size] + batch_start = time.time() + all_embeddings.extend(self._embedding._embed(batch, model_type="passage")) + self._embedded += len(batch) + elapsed = time.time() - self._start_time + rate = self._embedded / elapsed if elapsed > 0 else 0 + remaining_min = ((self._total_chunks - self._embedded) / rate / 60) if rate > 0 else 0 + logger.info("Embedding progress: %d / %d chunks (%.1f%%) - batch took %.2fs - ETA %.1f min", + self._embedded, self._total_chunks, + self._embedded / self._total_chunks * 100, + time.time() - batch_start, + remaining_min) + return all_embeddings + + def embed_query(self, text): + return self._embedding.embed_query(text) + + def __getattr__(self, name): + return getattr(self._embedding, name) + + class DocumentEmbedding: """ A class to create a FAISS database from a list of source documents. The source documents are collected from git repositories and chunked into smaller pieces for embedding. """ - def __init__(self, - *, - embedding: "Embeddings", - vdb_directory: PathLike = "./.cache/am_cache/vdb", - git_directory: PathLike = "./.cache/am_cache/git", - chunk_size: int = 800, - chunk_overlap: int = 160): + # Per-repo lock for serializing collect_documents calls on the same (git_repo, ref). + # Prevents concurrent git clone, install_dependencies, and document scanning + # when multiple CVEs target the same repo (e.g., from cve_generate_vdbs and cve_agent_executor). + _repo_locks: weakref.WeakValueDictionary[tuple, threading.Lock] = weakref.WeakValueDictionary() + _repo_locks_guard = threading.Lock() + + @classmethod + def _get_repo_lock(cls, git_repo: str, ref: str) -> threading.Lock: + key = (git_repo, ref) + with cls._repo_locks_guard: + lock = cls._repo_locks.get(key) + if lock is None: + lock = threading.Lock() + cls._repo_locks[key] = lock + return lock + + def __init__(self, *, embedding: "Embeddings", vdb_directory: PathLike = VDB_DIRECTORY, + git_directory: PathLike = DEFAULT_GIT_DIRECTORY, chunk_size: int = 800, chunk_overlap: int = 160, + pickle_cache_directory: PathLike = DEFAULT_PICKLE_CACHE_DIRECTORY): """ Create a new DocumentEmbedding instance. @@ -231,6 +308,7 @@ def __init__(self, Maximum size of a single chunk, by default 1000 chunk_overlap : int, optional Overlap between chunks, by default 200 + :param pickle_cache_directory: """ self._embedding = embedding @@ -238,6 +316,7 @@ def __init__(self, self._git_directory = Path(git_directory) self._chunk_size = chunk_size self._chunk_overlap = chunk_overlap + self._pickle_cache_directory = Path(pickle_cache_directory) @property def embedding(self): @@ -306,6 +385,9 @@ def _chunk_documents(self, documents: list[Document]) -> list[Document]: def get_repo_path(self, source_info: SourceDocumentsInfo): """ Returns the path to the git repository for a source document info. + + The path is ref-scoped to prevent conflicts when multiple threads work on + different refs of the same repository concurrently. Parameters ---------- @@ -315,15 +397,59 @@ def get_repo_path(self, source_info: SourceDocumentsInfo): Returns ------- Path - Returns the path to the git repository. + Returns the path to the git repository (ref-scoped). + + Example + ------- + For git_repo='https://github.com/org/repo' and ref='v1.0': + Returns: {git_directory}/https.github.com.org.repo_v1.0 """ - return self._git_directory / PurePath(source_info.git_repo) + return get_repo_path_with_ref(self._git_directory, source_info.git_repo, source_info.ref) - def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]: + def clone_and_install_dependencies(self, source_info: SourceDocumentsInfo) -> Path: """ - Collect documents from a source document info. This will clone the git repository and collect files from the - repository based on the include and exclude patterns. Each file is then parsed and segmented based on its - language into Documents. + Clone the repository and install dependencies without document parsing. + + This is the first phase of the split pipeline: clone repo, install deps, + return repo path. Document parsing/segmentation happens later via + collect_documents_from_cloned(). + + Thread-safe: Uses per-(git_repo, ref) lock to serialize operations. + + Parameters + ---------- + source_info : SourceDocumentsInfo + The source document info containing git_repo, ref, etc. + + Returns + ------- + Path + Path to the cloned repository (ref-scoped). + """ + repo_path = self.get_repo_path(source_info) + repo_lock = self._get_repo_lock(source_info.git_repo, source_info.ref) + + with repo_lock: + loader = SourceCodeGitLoader( + repo_path=repo_path, + clone_url=source_info.git_repo, + ref=source_info.ref, + include=source_info.include, + exclude=source_info.exclude, + ) + loader.load_repo() + logger.info("clone_and_install_dependencies completed for '%s' ref='%s'", source_info.git_repo, source_info.ref) + + return repo_path + + def collect_documents_from_cloned(self, source_info: SourceDocumentsInfo) -> list[Document]: + """ + Collect and parse documents from an already-cloned repository. + + This is the second phase of the split pipeline: parse/segment files + from a repo that was previously cloned via clone_and_install_dependencies(). + + Thread-safe: Re-acquires per-(git_repo, ref) lock before working. Parameters ---------- @@ -335,24 +461,114 @@ def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]: list[Document] Returns a list of documents collected from the source document info. """ - repo_path = self.get_repo_path(source_info) + cache_name = source_info.type if source_info.type != "code" else "" + + documents, documents_were_in_cache = retrieve_from_cache( + self._pickle_cache_directory, + source_info.git_repo, + source_info.ref, + documents_name=cache_name, + ) + if documents_were_in_cache or len(documents) > 0: + return documents + + repo_lock = self._get_repo_lock(source_info.git_repo, source_info.ref) + with repo_lock: + documents, documents_were_in_cache = retrieve_from_cache( + self._pickle_cache_directory, + source_info.git_repo, + source_info.ref, + documents_name=cache_name, + ) + if documents_were_in_cache or len(documents) > 0: + return documents + + if not repo_path.exists(): + logger.warning( + "collect_documents_from_cloned called but repo not cloned: %s", + repo_path, + ) + return [] + + blob_loader = SourceCodeGitLoader( + repo_path=repo_path, + clone_url=source_info.git_repo, + ref=source_info.ref, + include=source_info.include, + exclude=source_info.exclude, + ) + blob_parser = ExtendedLanguageParser() + + loader = GenericLoader(blob_loader=blob_loader, blob_parser=blob_parser) + documents = loader.load() + + logger.info("Collected documents for '%s', Document count: %d", repo_path, len(documents)) + save_to_cache( + self._pickle_cache_directory, + source_info.git_repo, + source_info.ref, + documents, + documents_name=cache_name, + ) + return documents - blob_loader = SourceCodeGitLoader(repo_path=repo_path, - clone_url=source_info.git_repo, - ref=source_info.ref, - include=source_info.include, - exclude=source_info.exclude) - - blob_parser = ExtendedLanguageParser() - - loader = GenericLoader(blob_loader=blob_loader, blob_parser=blob_parser) + def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]: + """ + Collect documents from a source document info. This will clone the git repository and collect files from the + repository based on the include and exclude patterns. Each file is then parsed and segmented based on its + language into Documents. - documents = loader.load() + If a credential_id is active in the current context (set via + ``credential_context()``), it will be used for private repository + authentication inside SourceCodeGitLoader. - logger.info("Collected documents for '%s', Document count: %d", repo_path, len(documents)) + Parameters + ---------- + source_info : SourceDocumentsInfo + The source document info to collect documents - return documents + Returns + ------- + list[Document] + Returns a list of documents collected from the source document info. + """ + repo_path = self.get_repo_path(source_info) + cache_name = source_info.type if source_info.type != "code" else "" + documents, documents_were_in_cache = retrieve_from_cache(self._pickle_cache_directory, + source_info.git_repo, source_info.ref, + documents_name=cache_name) + if documents_were_in_cache or len(documents) > 0: + return documents + + # Serialize git clone + install_dependencies + document scanning per (git_repo, ref). + # Without this lock, concurrent CVEs targeting the same repo (e.g., two keycloak CVEs) + # would both miss the pickle cache and run install_dependencies simultaneously, + # corrupting the filesystem (e.g., "META-INF: could not create directory"). + repo_lock = self._get_repo_lock(source_info.git_repo, source_info.ref) + with repo_lock: + # Re-check cache — another thread may have populated it while we waited. + documents, documents_were_in_cache = retrieve_from_cache(self._pickle_cache_directory, + source_info.git_repo, source_info.ref, + documents_name=cache_name) + if documents_were_in_cache or len(documents) > 0: + return documents + + # Download 3rd party dependencies to be part of the git repo' dir. + blob_loader = SourceCodeGitLoader(repo_path=repo_path, + clone_url=source_info.git_repo, + ref=source_info.ref, + include=source_info.include, + exclude=source_info.exclude) + blob_parser = ExtendedLanguageParser() + + loader = GenericLoader(blob_loader=blob_loader, blob_parser=blob_parser) + documents = loader.load() + + logger.info("Collected documents for '%s', Document count: %d", repo_path, len(documents)) + save_to_cache(self._pickle_cache_directory, source_info.git_repo, source_info.ref, documents, + documents_name=cache_name) + return documents def create_vdb(self, source_infos: list[SourceDocumentsInfo], output_path: PathLike): """ @@ -379,7 +595,6 @@ def create_vdb(self, source_infos: list[SourceDocumentsInfo], output_path: PathL # Warn if the output path already exists and we will overwrite it if (output_path.exists()): - logger.warning("Vector Database already exists and will be overwritten: %s", output_path) documents = [] @@ -414,8 +629,12 @@ def create_vdb(self, source_infos: list[SourceDocumentsInfo], output_path: PathL embedding_start_time = time.time() + # Wrap embedding in a proxy that logs batch progress + total_chunks = len(chunked_documents) + logging_embedding = _LoggingEmbeddingProxy(self._embedding, total_chunks, embedding_start_time) + # Create the FAISS database - db = FAISS.from_documents(chunked_documents, self._embedding) + db = FAISS.from_documents(chunked_documents, logging_embedding) logger.info("Completed embedding in %.2f seconds for '%s'", time.time() - embedding_start_time, output_path) diff --git a/src/exploit_iq_commons/utils/functions_parsers/__init__.py b/src/exploit_iq_commons/utils/functions_parsers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/exploit_iq_commons/utils/functions_parsers/c_lang_function_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/c_lang_function_parsers.py new file mode 100644 index 000000000..7289e39ca --- /dev/null +++ b/src/exploit_iq_commons/utils/functions_parsers/c_lang_function_parsers.py @@ -0,0 +1,771 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Tuple, List + +from langchain_core.documents import Document +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser +import re +import regex +import os +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + +logger = LoggingFactory.get_agent_logger(__name__) + +# Constants for parsing configuration +REGEX_TIMEOUT_SECONDS = 0.05 # Timeout for regex operations to prevent hanging +MIN_PARTS_FOR_PARSING = 2 # Minimum number of parts required for parsing +STRING_NOT_FOUND = -1 # Return value when string.find() doesn't find the substring + +from exploit_iq_commons.utils.dep_tree import DependencyTree, CCppDependencyTreeBuilder, C_DEP_LIBS_NAME + +def _remove_c_comments(code: str) -> str: + # Remove all multi-line comments (/* ... */) + code = re.sub(r'/\*.*?\*/', '', code, flags=re.DOTALL) + # Remove all single-line comments (//...) + code = re.sub(r'//.*', '', code) + return code + +# Compile the function pattern with recursive param matching +_FUNCTION_PATTERN_REGEX = regex.compile(r''' + ^[ \t]* # Leading indentation + (?P.*?) # Non-greedy return type (can include macros) + \b + (?P[A-Za-z_]\w*) # Function name (C identifier) + \s* + \( + (?P # Named group for parameters + (?: # Repeated nested structure: + [^()]+ # - non-paren characters + | # - or a nested (...) + \((?¶ms)\) # recurse into the same group + )* + ) + \) + \s*(?:const\s*)? # Optional const after params + \s*\{ # Opening brace of function body + ''', regex.VERBOSE | regex.MULTILINE) + + +class CDocumentParser: + """ + Parses a single C code Document and provides methods to extract information + about structs and functions within that document. + """ + + # Greedy patterns to capture full struct definitions + _NAMED_STRUCT_PATTERN = re.compile( + r'^\s*struct\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*\{(.*)\};?\s*$', + re.DOTALL | re.MULTILINE + ) + _TYPEDEF_STRUCT_PATTERN = re.compile( + r'^\s*typedef\s+struct\s+(?:[a-zA-Z_][a-zA-Z0-9_]*)?\s*\{(.*)\}\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*;?\s*$', + re.DOTALL | re.MULTILINE + ) + + + _FUNC_POINTER_PATTERN = re.compile( + r''' + ^\s* + (?P[\w\s\*]+?) + \(\s*\*\s* + (?P[A-Za-z_]\w*(?:\s*\[[^\]]+\])?) + \s*\)\s* + \(\s*(?P[^)]*)\)\s* + ; + $ + ''', + re.VERBOSE + ) + _BITFIELD_PATTERN = re.compile( + r''' + ^\s* + (?P[\w\s\*]+?) + \s+ + (?P[A-Za-z_]\w*) + \s*:\s* + (?P\d+) + \s*; + $ + ''', + re.VERBOSE + ) + + # Pattern for inline struct/union/enum definitions + _INLINE_STRUCT_PATTERN = re.compile( + r'((?:struct|union|enum)\s*\{.*?\})\s*([A-Za-z_][A-Za-z0-9_]*)\s*;', + re.DOTALL + ) + + def __init__(self, doc: Document): + self.c_code = doc.page_content.strip() + self.metadata = doc.metadata + self._parsed_struct_data = None + + def is_doc_struct(self) -> bool: + return bool(self._NAMED_STRUCT_PATTERN.search(self.c_code)) or bool(self._TYPEDEF_STRUCT_PATTERN.search(self.c_code)) + + + + def parse_struct_to_fields(self) -> tuple[tuple, list[tuple]]: + + if self._parsed_struct_data: + return self._parsed_struct_data + # Identify struct name and raw body + m_struct = self._NAMED_STRUCT_PATTERN.search(self.c_code) + m_typedef = self._TYPEDEF_STRUCT_PATTERN.search(self.c_code) + if m_struct: + struct_name = m_struct.group(1).strip() + struct_body = m_struct.group(2) + else: + struct_body = m_typedef.group(1) + struct_name = m_typedef.group(2).strip() + + source_file = self.metadata.get('source_file') + members = [] + + # Handle inline struct/union/enum definitions first + def _collect_inline(match): + raw_block = match.group(1).strip() + # Normalize whitespace to single spaces + normalized = ' '.join(raw_block.split()) + name = match.group(2).strip() + members.append((name, normalized)) + return '' # Remove from body + struct_body = self._INLINE_STRUCT_PATTERN.sub(_collect_inline, struct_body) + + # Process each remaining line + for line in struct_body.splitlines(): + # Strip comments + line = re.sub(r'//.*', '', line) + line = re.sub(r'/\*.*?\*/', '', line) + line = line.strip() + # Only simple declarations ending with ';' + if not line.endswith(';'): + continue + # try to parse function pointers and bit-fields this will work for most of the cases not all of them + if '(*' in line or ':' in line: + functionPointer = self._FUNC_POINTER_PATTERN.match(line) + if functionPointer: + name = functionPointer.group('name').strip() + type_str = f"{functionPointer.group('rettype').strip()} (*)( {functionPointer.group('params').strip()} )" + members.append((name, type_str)) + continue + m_bf = self._BITFIELD_PATTERN.match(line) + if m_bf: + name = m_bf.group('name').strip() + type_str = f"{m_bf.group('type').strip()} :{m_bf.group('bits')}" + members.append((name, type_str)) + continue + #could not parse this should be mark for edge cases + continue + content = line[:-1].strip() + parts = re.split(r'\s+', content) + if len(parts) < MIN_PARTS_FOR_PARSING: + continue + raw_name = parts[-1] + raw_type = ' '.join(parts[:-1]) + # Move pointer stars from name to type + star_match = re.match(r'^(\*+)', raw_name) + star = star_match.group(1) if star_match else '' + name = raw_name[len(star):] + type_str = (raw_type + (' ' + star if star else '')).strip() + members.append((name, type_str)) + + self._parsed_struct_data = ((struct_name, source_file), members) + return self._parsed_struct_data + +PARAMETER = "parameter" + +RETURN_TYPES = "return_types" + +LOCAL_VAR_USAGE = "local_var_usage" + +class CLanguageFunctionsParser(LanguageFunctionsParser): + + def init_CParser(self,depTree :DependencyTree): + self.typedef_pattern = re.compile(r'^\s*typedef\s+', re.MULTILINE) + self.fn_ptr_pattern = re.compile(r'\(\s*\*\s*\w+\s*\)') + self.struct_enum_union_pattern = re.compile(r'^\s*(struct|enum|union)\s+\w+\s*{', re.MULTILINE) + self.function_pattern = re.compile( + r'\b(?:static\s+|extern\s+|inline\s+|const\s+)*' + r'[A-Za-z_][\w\s\*]*?' # Return type (minimal matching) + r'\s+[A-Za-z_]\w*\s*\([^;{]*\)\s*(?:{|;)', # Function name and params + re.MULTILINE + ) + + # Pattern for parsing variable segments in multi-declaration lines + self.var_segment_pattern = re.compile( + r"^\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*(\[.*?\])?(?:\s*=\s*(.*))?$" + ) + + # Pattern for extracting return types from function signatures + self.return_type_pattern = re.compile( + r'((?:static|inline|\s)*[\w_][\w\d_]*(?:\s*\*+\s*|\s*))?\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\(' + ) + + # Pattern for function pointer variable calls + self.fp_var_pattern = re.compile(r'\b([a-zA-Z_][a-zA-Z0-9_]*)\s*\(') + + if isinstance(depTree.builder , CCppDependencyTreeBuilder) : + self.dep_builder_tree = depTree.builder + + self.C_STANDARD_LIB = "glibc" + + + def _optimize_func_key(self, function_name: str, source_path: str) -> str: + """ + Optimize func_key generation by extracting package name and filename from full path. + Uses the dependency tree's module_from_path method for consistent module extraction. + + Examples: + - 'count_trailing_zeros@rpm_libs/gawk/gawk-5.1/gawk-5.1.0/floatcomp.c' + -> 'count_trailing_zeros@gawk/floatcomp.c' + - 'main@src/app/main.c' -> 'main@app/main.c' + - 'process@unknown_source.c' -> 'process@unknown_source.c' + """ + if not source_path or source_path == 'unknown_source.c': + return f"{function_name}@{source_path}" + + # Use the dependency tree's module_from_path method if available + if self.dep_builder_tree: + try: + module_name = self.dep_builder_tree.module_from_path(source_path) + filename = os.path.basename(source_path) + return f"{function_name}@{module_name}/{filename}" + except Exception as e: + logger.debug(f"Failed to get module from path {source_path}: {e}") + # Fall back to original key method + + # Fallback: use original key generation method + return f"{function_name}@{source_path}" + + # This method returns for the given ecosystem , a map of all functions/methods in all packages to a dict containing + # mappings of all local variables defined in these functions/methods, to their type/value/expression. + # Implementation should be consistent with search_for_called_function, as functions_methods_documents is being used + # in this method as functions_local_variables_index + def __parse_function_signature(self,function_signature: str) -> dict: + """ + Parses a C/C++ function signature to extract parameter types and names. + + Returns: + dict mapping each parameter name to {"value": PARAMETER, "type": }. + Supports qualifiers, pointers, arrays, variadic args, and function pointers. + """ + all_vars = {} + + # Extract the parameter substring inside the outermost parentheses + parentheses = re.search(r"\((.*)\)", function_signature) + if not parentheses: + return all_vars + + params_str = parentheses.group(1).strip() + if not params_str or params_str == "void": + return all_vars + + # Split on commas not inside any parentheses (for function-pointer args) + parts = re.split(r',\s*(?![^()]*\))', params_str) + + for param in parts: + param = param.strip() + if not param: + continue + if param == "...": + all_vars["..."] = {"value": PARAMETER, "type": "VARIADIC_ARGS"} + continue + + # 1. Function-pointer: base (*name)(args) + functionPointer = re.match( + r'^(?P.+?)\s*\(\*\s*(?P[A-Za-z_]\w*)\s*\)\s*\((?P.*)\)\s*$', + param + ) + if functionPointer: + base = functionPointer.group('base').strip() + name = functionPointer.group('name').strip() + args = functionPointer.group('args').strip() + p_type = f"{base} (*{name})({args})" + all_vars[name] = {"value": PARAMETER, "type": p_type} + continue + + # 2. Named-anchored regex: separate type and name (with pointers/arrays) + m_name = re.match( + r'^(?P.+?)\s+(?P[\*&]*[A-Za-z_]\w*(?:\[.*?\])?)\s*$', + param + ) + if m_name: + p_type = m_name.group('type').strip() + raw_name = m_name.group('name').strip() + # Move leading pointer/reference symbols onto type + ptr_match = re.match(r'^(?P[\*&]+)(?P[A-Za-z_]\w*(?:\[.*?\])?)$', raw_name) + if ptr_match: + name = ptr_match.group('nm') + p_type = f"{p_type} {ptr_match.group('prefix')}".strip() + else: + # Extract array dimensions from name + arr_match = re.match(r'^(?P[A-Za-z_]\w*)(?P\[.*?\])$', raw_name) + if arr_match: + name = arr_match.group('nm') + p_type = f"{p_type}{arr_match.group('dims')}" + else: + name = raw_name + all_vars[name] = {"value": PARAMETER, "type": p_type} + continue + + # 3. Fallback: last-space split + parts2 = param.rsplit(None, 1) + if len(parts2) == 2: + p_type, name = parts2 + else: + p_type = parts2[0] + name = parts2[0] + all_vars[name] = {"value": PARAMETER, "type": p_type} + + return all_vars + + def create_map_of_local_vars(self, functions_methods_documents: list[Document]) -> dict[str, dict]: + mappings = dict() + + processed_count = 0 + for func_method in functions_methods_documents: + processed_count += 1 + + func_key = self._optimize_func_key( + self.get_function_name(func_method), + func_method.metadata.get('source', 'unknown_source.c') + ) + + + if ("state" in func_method.metadata ) or (func_method.metadata.get("func_name","N/A") == "if"): + continue #means invalid get function name failed + + all_vars = dict() + + # Optimize string operations - avoid multiple .strip() calls + content = func_method.page_content + first_brace_index = content.find('{') + if first_brace_index != STRING_NOT_FOUND: + function_signature = content[:first_brace_index].strip() + function_body = content[first_brace_index:].strip() + else: + function_signature = content.strip() + function_body = None # No function body found + + + # --- Parsing Function Parameters --- + all_vars = self.__parse_function_signature(function_signature) + + + # --- Parsing Return Types --- + # Capture everything before the function name and its opening parenthesis + # This regex is made more specific to ensure it captures the return type accurately. + # It looks for optional leading keywords (static, inline), then the actual type, + # then whitespace, then the function name, then an opening paren. + return_type_match = self.return_type_pattern.match(function_signature) + if return_type_match: + # Group 1 is the potential type including modifiers and pointers (e.g., "static int*", "const char") + # Group 2 is the function name, which we don't need for return type + p_return_type = return_type_match.group(1) + if p_return_type is None: #incase of bad segmentation a case in openssl file quic_wire.c + continue + potential_return_type = p_return_type.strip() + if potential_return_type: + all_vars[RETURN_TYPES] = [potential_return_type] + else: + all_vars[RETURN_TYPES] = [] # Should ideally not happen for valid C functions + else: + all_vars[RETURN_TYPES] = [] + + # --- Parsing Local Variables in Function Body --- + # Skip processing if no function body + if not function_body: + mappings[func_key] = all_vars + continue + + # Pre-filter lines to avoid unnecessary processing + lines = [line.strip() for line in function_body.splitlines() + if line.strip() and not self.is_comment_line(line.strip()) + and not line.strip().startswith("return ")] + + for cleaned_row in lines: + + # Regex to identify a line containing multiple declarations + # Group 1: The base type (e.g., 'int', 'char*') + # Group 2: The rest of the declaration string (e.g., 'i, j = 10') + multi_declaration_line_match = re.match( + r"^\s*([\w_][\w\d_]*(?:\s*[*&]+\s*|\s*))\s*(.+?);\s*$", cleaned_row + ) + + if multi_declaration_line_match: + base_type = multi_declaration_line_match.group(1).strip() + var_list_str = multi_declaration_line_match.group(2).strip() + + # Split the variable list by commas, handling cases like 'var = val' within a segment + # This regex matches: (var_name)( [array_brackets])( = value) + # It's applied to each comma-separated segment. + + for segment in var_list_str.split(','): + segment_match = self.var_segment_pattern.match(segment.strip()) + if segment_match: + var_name = segment_match.group(1).strip() + array_suffix = segment_match.group(2) + initial_value = segment_match.group(3) + + current_var_type = base_type + if array_suffix: + current_var_type += "[]" # Standardize array type representation + + all_vars[var_name] = { + "value": initial_value.strip() if initial_value is not None else "", + "type": current_var_type + } + + else: + # Fallback if a segment doesn't fit the expected var_segment_pattern + # This might indicate a syntax error or a more complex declaration not handled. + pass # Or log a warning + + + else: + # Fallback to single declaration or assignment logic if not a multi-declaration line + # C variable declaration: type var_name [= value]; + declaration_match = re.match(r'^\s*([\w_][\w\d_]*(?:\s*[*&]+\s*|\s*))([a-zA-Z_][a-zA-Z0-9_]*)\s*(\[.*?\])?\s*(?:=\s*(.*?))?\s*;', cleaned_row) + + if declaration_match: + full_type_str = declaration_match.group(1).strip() + var_name = declaration_match.group(2).strip() + array_suffix = declaration_match.group(3) + initial_value = declaration_match.group(4) + + if array_suffix: + full_type_str += "[]" + + all_vars[var_name] = { + "value": initial_value.strip() if initial_value is not None else "", + "type": full_type_str + } + + + # Simple assignments for existing variables (no declaration) + elif '=' in cleaned_row and ';' in cleaned_row: + assignment_match = re.match(r'^\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*([^=;]+);', cleaned_row) + if assignment_match: + var_name = assignment_match.group(1).strip() + value_expr = assignment_match.group(2).strip() + + if var_name in all_vars: + all_vars[var_name]["value"] = value_expr + else: + all_vars[var_name] = {"value": value_expr, "type": LOCAL_VAR_USAGE} + + + # Store the result + mappings[func_key] = all_vars + + return mappings + + + # This method returns (for the given ecosystem) , a mapping of all types/classes in all packages to a dict + # containing mapping of all fields names in the type/class to their defining types. + # a list of + def parse_all_type_struct_class_to_fields(self, types: list[Document], type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]]=None) -> dict[tuple, list[tuple]]: + types_mapping = dict() + for _type in types: + #parse the document + parser = CDocumentParser(_type) + #get the struct name and fields + struct_name, fields = parser.parse_struct_to_fields() + types_mapping[struct_name] = fields + + return types_mapping + + def is_macro_function_invocation(self,seg: str) -> bool: + """ + Detects if a line is a macro function invocation, e.g. IMPLEMENT_ASN1_FUNCTIONS(...) + or IMPLEMENT_ASN1_ENCODE_FUNCTIONS_fname(...) + """ + lines = seg.strip().splitlines() + for line in lines: + res = bool(re.match(r'^[A-Z_][A-Z0-9_]*(?:_[A-Z0-9_]+)*(_fname)?\s*\(.*\)\s*;?$', line.strip())) + if res: + return True + + # Match ALLCAPS or ALLCAPS_with_underscores, optionally with trailing _fname, and then '(' + return False + + def get_function_name(self, function: Document) -> str: + try: + reservred_words =["break", "case", "continue", "default", "do", "else", "for", "goto", "if", "switch", "while"] + if function.metadata.get('func_name') : + return function.metadata.get('func_name') + + seg =function.page_content.strip() + code = seg.replace('\r\n', '\n') + function_pattern = _FUNCTION_PATTERN_REGEX.search(code, timeout=REGEX_TIMEOUT_SECONDS) + if function_pattern: + f_name = function_pattern.group('name') + if f_name: + if f_name in reservred_words: + function.metadata['state'] = "invalid" + return "" + function.metadata['func_name'] = f_name + return f_name + function.metadata['state'] = "invalid" + return "" + except TimeoutError: + logger.warning(f"Function name extraction timed out for file: {function.metadata.get('source', 'unknown')}") + function.metadata['state'] = "invalid" + return "" + + + def search_for_called_function( + self, + caller_function: Document, + callee_function_name: str, + callee_function_package: str, # For C, this is usually the file or library name + code_documents: list[Document], + type_documents: list[Document], + callee_function_file_name: str, + fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict], + documents_of_functions: list[Document] = None, + type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]] = None, + callee_function:Document = None + ) -> bool: + """ + Returns True if caller_function calls callee_function (directly or via struct/function pointer). + """ + # 1. Extract function body (remove comments for easier parsing) + function_body = _remove_c_comments(caller_function.page_content) + # 2. Direct call: callee_function_name( + direct_call_pattern = re.compile(r'\b' + re.escape(callee_function_name) + r'\s*\(') + if direct_call_pattern.search(function_body): + return True + + # 3. Struct member or pointer call: obj->callee_function_name( or obj.callee_function_name( + member_call_pattern = re.compile( + r'([a-zA-Z_][a-zA-Z0-9_]*)\s*(->|\.)\s*' + re.escape(callee_function_name) + r'\s*\(') + for match in member_call_pattern.finditer(function_body): + var_name = match.group(1) + # Look up var_name in local variables and parameters + func_key = self._optimize_func_key( + self.get_function_name(caller_function), + caller_function.metadata.get('source', 'unknown_source.c') + ) + + local_vars = functions_local_variables_index.get(func_key, {}) + var_type = local_vars.get(var_name, {}).get("type", "") + # Check if var_type is a struct with a function pointer named callee_function_name + for (struct_name, _), fields in fields_of_types.items(): + if struct_name == var_type: + for field_name, field_type in fields: + if field_name == callee_function_name and "(*" in field_type: + return True + + # 4. Function pointer variable: foo_ptr( + for match in self.fp_var_pattern.finditer(function_body): + var_name = match.group(1) + if var_name == callee_function_name: + continue # Already checked direct call + # Is this variable a function pointer to callee_function_name? + func_key = self._optimize_func_key( + self.get_function_name(caller_function), + caller_function.metadata.get('source', 'unknown_source.c') + ) + local_vars = functions_local_variables_index.get(func_key, {}) + var_type = local_vars.get(var_name, {}).get("type", "") + # Optionally, check if the function pointer is assigned to callee_function_name somewhere + assignment_pattern = re.compile(rf'{re.escape(var_name)}\s*=\s*&?{re.escape(callee_function_name)}\s*;') + if assignment_pattern.search(function_body): + return True + + # 5. (Optional) Macro expansion, indirect calls, etc. (advanced, skip for now) + + return False + + def get_package_names(self, function: Document) -> list[str]: + pkgs = [] + full_doc_path = str(function.metadata['source']) + if self.dep_builder_tree: + mod = self.dep_builder_tree.module_from_path(full_doc_path) + pkgs.append(mod) + function.metadata['pkg_name']=mod + else: + pkgs.append(os.path.basename(full_doc_path)) + return pkgs + + def get_package_name(self, function: Document, package_name: str) -> str: + return package_name + + def is_root_package(self, function: Document) -> bool: + pkgs = self.get_package_names(function) + if len(pkgs) > 0: + if pkgs[0] == self.dep_builder_tree.prj_name: + return True + return not function.metadata['source'].startswith(self.dir_name_for_3rd_party_packages()) + + def is_comment_line(self, line: str) -> bool: + stripped = line.strip() + return stripped.startswith("//") or stripped.startswith("/*") + + def get_comment_line_notation(self) -> str: + return "//" + + def is_exported_function(self, function: Document, documents_of_full_sources: dict[str, Document]) -> bool: + # In C, functions have external linkage by default unless declared as static + #however internal functions might be called by exported functions + #The analysis needs to understand the complete call graph + #So we return True for all functions + return True + + def is_function(self, function: Document) -> bool: + """ + Check if the code contains a C function declaration/definition using regex. + Focuses on essential function patterns: + 1. Optional modifiers (static, extern, etc) + 2. Any return type (we don't care about specifics) + 3. Function name + 4. Parameter list + 5. Ending with ; or { + + Args: + code: String containing C code + + Returns: + bool: True if code contains a function, False otherwise + """ + # Clean comments first + if function.metadata.get('content_type') == 'functions_classes': + code = _remove_c_comments(function.page_content) + + # Negative checks + if self.typedef_pattern.search(code): + return False + if self.fn_ptr_pattern.search(code): + return False + if self.struct_enum_union_pattern.search(code): + return False + + + return bool(self.function_pattern.search(code)) + + return False + + def dir_name_for_3rd_party_packages(self) -> str: + return C_DEP_LIBS_NAME + + def supported_files_extensions(self) -> list[str]: + return [".c", ".h"] + + def is_supported_file_extensions(self, extension: str) -> bool: + return extension in self.supported_files_extensions() + + def is_searchable_file_name(self, function: Document) -> bool: + return function.metadata.get("file_name", "").endswith((".c", ".h")) + + def get_function_reserved_word(self) -> str: + return "" # C doesn't have a consistent reserved word for functions + + def get_type_reserved_word(self) -> str: + return "struct" + + def is_doc_type(self,doc: Document ) -> bool: + try: + if self.is_function(doc): + return False + parser = CDocumentParser(doc) + res = parser.is_doc_struct() + except Exception as e: + res = False + return res + + def filter_docs_by_func_pkg_name(self,function_name:str ,package_name : str, documents : list[Document] ) ->list[Document] : + + relevant_docs = [doc for doc in documents if doc.page_content.__contains__(function_name)] + + return relevant_docs + + def document_imports_package(self,documents: dict[str, Document],package_name:str) -> list[Document]: + importing_docs = [value for (file, value) in documents.items() + if re.search( + rf"(include {package_name}|include\s*\(\s*[\w\s\/.\"-]*{package_name}[\w\s\/.\"-]*\s*\))" + , value.page_content, flags=re.MULTILINE)] + return importing_docs + + def is_a_package(self,package_name: str,doc:Document) -> bool: + #check for external pkgs + file = doc.metadata.get('source') + b_3party = doc.metadata.get('source').startswith(self.dir_name_for_3rd_party_packages()) + if self.dep_builder_tree and b_3party: + pkg = self.dep_builder_tree.module_from_path(file) + else: + pkg = self.dep_builder_tree.prj_name + return pkg == package_name + + def is_search_algo_dfs(self)-> bool: + return False + + def is_language_supports_macro(self)-> bool: + return True + + def create_dummy_for_standard_lib(self,package_name: str)-> bool: + if package_name == self.C_STANDARD_LIB: + return True + return False + + def get_dummy_function(self, function_name): + return None + + def is_call_allowed(self, pkg_docs: list[Document], caller_function: Document, callee_function: Document) -> bool: + caller_pkg = self.get_package_names(caller_function)[0] + callee_pkg = self.get_package_names(callee_function)[0] + + # check if callee is static + first_line = callee_function.page_content.splitlines()[0] + is_callee_static = bool(re.match(r"^\s*static\b", first_line)) + + if is_callee_static: + # static callee: must be same package and same file + if caller_pkg != callee_pkg: + return False + if caller_function.metadata.get('source') != callee_function.metadata.get('source'): + return False + return True + + # non-static callee + if caller_pkg != callee_pkg: + # collect all functions in caller package + caller_functions = {self.get_function_name(doc): doc for doc in pkg_docs} + + callee_name = self.get_function_name(callee_function) + + if callee_name == "do_shell": + print(f"callee_name: {callee_name}") + + if callee_name in caller_functions: + doc = caller_functions[callee_name] + # if static and in same file → call is allowed + if re.match(r"^\s*static\b", doc.page_content.splitlines()[0]): + if doc.metadata.get('source') != caller_function.metadata.get('source'): + return True + # otherwise (global collision) → not allowed + return False + + return True + + def get_import_search_patterns(self, package_name: str) -> list[re.Pattern]: + escaped = re.escape(package_name) + return [ + re.compile(rf'#include\s*[<"]({escaped}[^>"]*)[>"]', re.IGNORECASE | re.MULTILINE), + ] diff --git a/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py new file mode 100644 index 000000000..c97437807 --- /dev/null +++ b/src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py @@ -0,0 +1,649 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +from typing import Tuple, List + +from langchain_core.documents import Document + +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser + +EMBEDDED_TYPE = "embedded_type" + +PARAMETER = "parameter" + +RETURN_TYPES = "return_types" + +LOCAL_VAR_USAGE = "local_var_usage" +LOCAL_IMPLICIT = "local_implicit" +LOCAL_INDIRECT_TYPES_INDICATIONS = [LOCAL_VAR_USAGE, LOCAL_IMPLICIT] +PRIMITIVE_TYPES = ['uint8', 'uint16', 'uint32', 'uint64', 'int8', 'int16', 'int32', 'int64', 'float32', 'float64', + 'complex64', 'complex128', 'byte', 'rune', 'uint', 'int', 'uintptr', 'string', 'ptr', 'bool'] +MAP_TYPE_REGEX = r"map\[[a-zA-Z0-9]+\][a-zA-Z0-9]+ " +SLICE_TYPES_REGEX = r"\[\][a-zA-Z0-9]+ " + + +def get_package_name_file(function: Document): + content = function.page_content + index_of_package = content.find("package") + index_of_end_of_line = index_of_package + content[index_of_package:].find(os.linesep) + if index_of_package > -1: + return content[index_of_package + 8:index_of_end_of_line].strip() + + +class GoLanguageFunctionsParser(LanguageFunctionsParser): + + def is_same_package(self, package_name_from_input, package_name_from_tree): + return package_name_from_input.lower() in package_name_from_tree.lower() + + def is_tree_key_match(self, package_from_doc: str, tree_key: str) -> bool: + """Boundary-aware matching at '/' separators for Go module paths. + + Only matches when the tree key is equal to, or a child of, the + doc package (tree_key starts with package_from_doc). The reverse + direction (doc package is a sub-path of tree key) is intentionally + excluded — it would map a specific sub-package query like + ``helm.sh/helm/v3/pkg/strvals`` to the entire module + ``helm.sh/helm/v3``, causing false reachability chains.""" + a = package_from_doc.lower() + b = tree_key.lower() + if a == b: + return True + if b.startswith(a + "/"): + return True + return False + + def get_dummy_function(self, function_name): + return f"{self.get_function_reserved_word()} {function_name}() {{}}" + + def __trace_down_package(self, expression: str, code_documents: dict[str, Document], type_documents: list[Document], + callee_package: str, fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict], + caller_function_index: str) -> bool: + variables_mappings = functions_local_variables_index[caller_function_index] + parts = expression.split(".") + + result = False + + (resolved_type, struct_initializer_expression, + value, var_properties) = self.__prepare_package_lookup(parts, variables_mappings, the_part=-1) + + if (var_properties is not None + and (struct_initializer_expression or + resolved_type not in LOCAL_INDIRECT_TYPES_INDICATIONS or value == PARAMETER)): + result = self.__lookup_package(callee_package, resolved_type, struct_initializer_expression, + type_documents, value) + + # Property/member is not in function, check if it's member/property of a type + elif var_properties is None and len(parts) > 1: + (resolved_type, struct_initializer_expression, + value, var_properties) = self.__prepare_package_lookup(parts, variables_mappings, the_part=-2) + if (var_properties is not None + and (resolved_type not in LOCAL_INDIRECT_TYPES_INDICATIONS or value == PARAMETER)): + field_name = parts[-1] + possible_types = {key: value for (key, value) in fields_of_types.items() + if resolved_type in key + and any([field for field in value if field_name in field])} + for the_type, mappings in possible_types.items(): + for mapping in mappings: + if field_name in mapping: + returned_matched_types = self.__get_type_docs_matched_with_callee_package(callee_package, + mapping[1], + type_documents) + if len(returned_matched_types) > 0: + result = True + + elif var_properties is not None: + value = var_properties.get("value", None) + if value is not None and value.strip() != "" and len(parts) > 1: + return self.__trace_down_package(value, code_documents, type_documents, callee_package, + fields_of_types, functions_local_variables_index, + caller_function_index) + + return result + + def __prepare_package_lookup(self, parts, variables_mappings, the_part: int): + var_properties = variables_mappings.get(parts[the_part], None) + if var_properties is not None: + resolved_type = var_properties.get("type") + value = var_properties.get("value") + struct_initializer_expression = re.search(r"(&|\\*)?\w+\s*{", value) + resolved_type = str(resolved_type).replace("&", "").replace("*", "") + return resolved_type, struct_initializer_expression, value, var_properties + else: + return None, None, None, None + + def __lookup_package(self, callee_package, resolved_type, struct_initializer_expression, type_documents, + value) -> bool: + result = False + if not struct_initializer_expression and resolved_type not in PRIMITIVE_TYPES: + docs = self.__get_type_docs_matched_with_callee_package(callee_package, resolved_type, type_documents) + + if len(docs) > 0: + result = True + elif value == PARAMETER: + result = False + + elif struct_initializer_expression: + struct_type = (struct_initializer_expression.group(0).replace("{", "") + .replace("&", "").replace("*", "")) + docs = self.__get_type_docs_matched_with_callee_package(callee_package, struct_type, type_documents) + if len(docs) > 0: + result = True + return result + + def __get_type_docs_matched_with_callee_package(self, callee_package, checked_type, type_documents) -> list[ + Document]: + return [a_type for a_type in type_documents if callee_package in a_type.metadata['source'] and + (self.get_type_name(a_type) == checked_type or self.get_type_name(a_type) in checked_type)] + + def create_map_of_local_vars(self, functions_methods_documents: list[Document]) -> dict[str, dict]: + mappings = dict() + for func_method in functions_methods_documents: + func_key = f"{self.get_function_name(func_method)}@{func_method.metadata['source']}" + all_vars = dict() + for row in func_method.page_content.splitlines(): + if not self.is_comment_line(row): + # Extract arguments and receiver argument of type as parameters + if row.startswith("func"): + match = re.finditer(r"(func|\w+)\s*\([a-zA-Z0-9\s*,.\[\]]+\)" + , func_method.page_content[:func_method.page_content.find("{")] + , flags=re.MULTILINE) + for current_match in match: + current_args = (current_match.group(0).replace("\n\t", "") + .replace("\t", "").replace("\n", "")) + current_params = re.search(r"\(.*\)", current_args) + params = (current_params.group(0).replace("(", "") + .replace(")", "").split(",")) + params_tuple = tuple(params) + param_type: str = "" + for param in reversed(params_tuple): + data = param.strip().split(" ") + param_name = data[0] + if len(data) > 1: + param_type = data[1] + the_value = PARAMETER + all_vars[param_name] = {"value": the_value, "type": param_type} + # Gets return types from function + index_of_start_func = func_method.page_content.find(row) + last_left_curly_bracket = index_of_start_func + func_method.page_content.find("{") + last_right_bracket = func_method.page_content[ + index_of_start_func:last_left_curly_bracket].rfind(")") + return_parameters = func_method.page_content[index_of_start_func + last_right_bracket + 1: + last_left_curly_bracket - 1].strip() + return_parameters = return_parameters.replace(")", "").replace("(", "") + if return_parameters.strip() != "": + all_vars[RETURN_TYPES] = return_parameters.split(",") + else: + all_vars[RETURN_TYPES] = [] + + elif row.strip().startswith("var ") and not re.search(r"var\s*\(", row.strip()): + row_without_var_prefix = row.strip()[3:].strip() + parts = row_without_var_prefix.split() + # variable name + left_side = parts[0] + if len(parts) == 2 and parts[1].__contains__("="): + assignment = row.strip().split("=") + all_vars[(left_side.strip())] = {"value": assignment[1], + "type": assignment[0].replace("*", "")} + else: + if len(parts) == 2: + all_vars[(left_side.strip())] = {"value": "", "type": parts[1].replace("*", "")} + elif row.strip().__contains__(":=") and row.strip().__contains__("if"): + index_of_start_if = func_method.page_content.find(row) + # Go until delimiter of assignment and start of boolean expression + end_of_assignment = func_method.page_content[index_of_start_if:].find(";") + parts = row.strip().split(":=") + left_side = parts[0].replace("if", "") + right_side = func_method.page_content[index_of_start_if + 2: + index_of_start_if + end_of_assignment - 1].strip() + + all_vars[(left_side.strip())] = {"value": right_side.strip().replace + ("\n\t", "").replace("\t", ""), "type": LOCAL_IMPLICIT + } + + elif (row.strip().__contains__(":=") and not row.strip().__contains__("if") + and not row.strip().__contains__("for ") and not row.strip().__contains__("range ") + and not row.strip().__contains__("range ") and not row.strip().__contains__("case ")): + parts = row.strip().split(":=") + left_side = parts[0] + right_side = parts[1] + all_vars[(left_side.strip())] = {"value": right_side.strip(), "type": LOCAL_IMPLICIT} + elif (row.strip().__contains__("=") and not row.strip().__contains__("==") + and not row.strip().__contains__("!=") + and not row.strip().__contains__("if") and not row.strip().__contains__("for ") + and not row.strip().__contains__("range ") and not row.strip().__contains__("case ")): + parts = row.strip().split("=") + left_side = parts[0] + right_side = parts[1] + all_vars[(left_side.strip())] = {"value": right_side.strip(), "type": ("%s" % LOCAL_VAR_USAGE)} + else: + pass + + mappings[func_key] = all_vars + + return mappings + + def __is_struct_or_interface_type(self, doc: Document): + return re.search(r"^type\s+[a-zA-Z0-9_]+\s+(struct|interface)", doc.page_content) + + def __get_type_kind(self, the_type: Document) -> str: + if self.__is_struct_or_interface_type(the_type): + parts_of_type_header = the_type.page_content.split() + return parts_of_type_header[2] if len(parts_of_type_header) > 2 else "" + # not composite type, primitive or wrapper type + else: + names = the_type.page_content.split(sep=" ", maxsplit=2) # if the_type.page_content + # Make sure that aliased types names are cleaned from '=' + return names[2].replace("=", "") + + def get_type_name(self, the_type: Document) -> str: + if self.__is_struct_or_interface_type(the_type): + parts_of_type_header = the_type.page_content.split() + return parts_of_type_header[1] if len(parts_of_type_header) > 1 else "" + else: + parts = the_type.page_content.split(sep=" ", maxsplit=2) # if the_type.page_content + return parts[1] + + def parse_all_type_struct_class_to_fields(self, types: list[Document], type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]]=None) -> dict[tuple, list[tuple]]: + types_mapping = dict() + for the_type in types: + # multiple types in a single block declaration + if re.search(r"^type\s+\(", the_type.page_content): + right_bracket = the_type.page_content.rfind(")") + left_bracket = the_type.page_content.find("(") + all_types = the_type.page_content[left_bracket + 1:right_bracket - 1] + pos = 0 + + current_line_stripped = all_types.lstrip() + size_of_type_block = len(all_types.lstrip()) + while pos < size_of_type_block: + + next_eol = current_line_stripped.find(os.linesep) + next_struct = current_line_stripped.find("struct") + if next_eol > - 1 and (next_struct == -1 or next_struct > next_eol): + if not self.is_comment_line(current_line_stripped[:next_eol + 1]): + declaration_parts = current_line_stripped[:next_eol + 1].split() + # If row inside block contains func, then it's a function type and need to parse it in a + # special way. + if current_line_stripped[:next_eol + 1].__contains__("func"): + declaration_parts = current_line_stripped[:next_eol + 1].split("func") + declaration_parts = [part.strip() for part in declaration_parts] + if len(declaration_parts) == 2: + declaration_parts[1] = f"func {declaration_parts[1]}" + # ignore alias' "equals" notation + if len(declaration_parts) == 3: + [name, _, type_name] = declaration_parts + elif len(declaration_parts) == 2: + [name, type_name] = declaration_parts + if len(declaration_parts) == (2 or 3): + self.parse_one_type(Document(page_content=f"type {name} {type_name}", + metadata={"source": the_type.metadata['source']}), + types_mapping) + pos += next_eol + 1 + current_line_stripped = current_line_stripped[next_eol + 1:].lstrip() + elif -1 < next_struct < next_eol and next_eol > -1: + right_curly_bracket_ind = current_line_stripped.find("}") + if right_curly_bracket_ind == -1: + pos = size_of_type_block + break + self.parse_one_type( + Document(page_content=f"type {current_line_stripped[:right_curly_bracket_ind + 1]}", + metadata={"source": the_type.metadata['source']}), + types_mapping) + pos += right_curly_bracket_ind + current_line_stripped = current_line_stripped[right_curly_bracket_ind + 1:].lstrip() + # End of multiple types declaration + elif next_eol == -1: + pos = size_of_type_block + else: + self.parse_one_type(the_type, types_mapping) + return types_mapping + + def parse_one_type(self, the_type, types_mapping): + the_kind = self.__get_type_kind(the_type) + type_name = self.get_type_name(the_type) + type_key = type_name + if the_kind.lower() == "interface": + type_key = f"interface;{type_name}" + start_index = the_type.page_content.find("{") + 2 + end_index = the_type.page_content.rfind("}") + current_index = start_index + fields_list = list() + if end_index > - 1: + while current_index < end_index: + index_of_eol = the_type.page_content[current_index:].find(os.linesep) + end_of_row = end_index + if index_of_eol > -1: + end_of_row = index_of_eol + current_line = the_type.page_content[current_index:current_index + end_of_row] + if not self.is_comment_line(current_line) and not current_line.strip() == "": + right_bracket_completion = "" + if type_key.startswith("interface"): + parts = current_line.split(sep=")", maxsplit=1) + right_bracket_completion = ")" + else: + parts = current_line.split(sep=" ", maxsplit=1) + if len(parts) == 2: + type_str = parts[1].lstrip() + space_pos = type_str.find(" ") + field_type = type_str[:space_pos] if space_pos != -1 else type_str + fields_list.append((f"{parts[0]}{right_bracket_completion}".strip(), + field_type)) + elif len(parts) == 1: + fields_list.append((EMBEDDED_TYPE, parts[0])) + elif len(parts) > 2: + type_str_raw = parts[1] + space_pos_raw = type_str_raw.find(" ") + field_type_raw = type_str_raw[:space_pos_raw].strip() if space_pos_raw != -1 else type_str_raw.strip() + fields_list.append((f"{parts[0]}{right_bracket_completion}".strip(), + field_type_raw)) + else: + pass + + current_index = current_index + end_of_row + 1 + if type_key.strip() != "" and len(fields_list) != 0: + types_mapping[(type_key, the_type.metadata['source'])] = fields_list + # Primitive type or wrapper of another type + else: + types_mapping[(type_key.strip(), the_type.metadata['source'])] = [(type_name.strip(), the_kind.strip())] + + def get_function_reserved_word(self) -> str: + return "func" + + def get_type_reserved_word(self) -> str: + return "type" + + def is_searchable_file_name(self, function: Document) -> bool: + file_path = str(function.metadata['source']) + return "test" not in file_path[file_path.rfind("/") + 1:].split(".")[0].lower() + + def is_function(self, doc: Document) -> bool: + return doc.page_content.startswith(self.get_function_reserved_word()) + + def supported_files_extensions(self) -> list[str]: + return [".go"] + + @staticmethod + def get_comment_line_notation() -> str: + return "//" + + def dir_name_for_3rd_party_packages(self) -> str: + return "vendor" + + def is_exported_function(self, function: Document, documents_of_full_sources: dict[str, Document]) -> bool: + function_name = self.get_function_name(function) + return re.search("[A-Z][a-z0-9-]*", function_name) + + def get_function_name(self, function: Document) -> str: + try: + index_of_function_opening = function.page_content.index("{") + except ValueError as e: + function_line = function.page_content.find(os.linesep) + # print(f"function {function.page_content[:function_line]} => contains no body ") + return function.page_content[:function_line] + + function_header = function.page_content[:index_of_function_opening] + # function is a method of a type + if function_header.startswith("func ("): + index_of_first_right_bracket = function_header.index(")") + skip_receiver_arg = function_header[index_of_first_right_bracket + 1:] + index_of_first_left_bracket = skip_receiver_arg.index("(") + return skip_receiver_arg[:index_of_first_left_bracket].strip() + # regular function not tied to a certain type + else: + try: + index_of_first_left_bracket = function_header.index("(") + # Go Generic function + except ValueError: + try: + index_of_first_left_bracket = function_header.index("[") + except ValueError: + raise ValueError(f"Invalid function header - {function_header}") + func_with_name = function_header[:index_of_first_left_bracket] + if len(func_with_name.split(" ")) > 1: + return func_with_name.split(" ")[1] + # TODO Try to extract anonymous function var + # else: + + def search_for_called_function(self, caller_function: Document, callee_function_name: str, + callee_function: Document, + callee_function_package: str, code_documents: list[Document], + type_documents: list[Document], + callee_function_file_name: str, fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict], + documents_of_functions: list[Document]=None, + type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]]=None) -> bool: + index_of_function_opening = caller_function.page_content.index("{") + index_of_function_closing = caller_function.page_content.rfind("}") + caller_function_body = str( + caller_function.page_content[index_of_function_opening + 1: index_of_function_closing]) + re.search("", caller_function_body) + escaped_name = re.escape(callee_function_name) + regex = fr'(? bool: + caller_function_file = function.metadata.get('source') + caller_function_name = self.get_function_name(function) + caller_function_index = f"{caller_function_name}@{caller_function_file}" + parts = identifier_function.split(".") + # If there is no qualifier identifier , then check whether callee function and caller + # function are in the same package + if len(parts) == 1: + try: + callee_function = code_documents[callee_function_file_name] + callee_function_package = get_package_name_file(callee_function).strip() + except KeyError as e: + # Standard library function , there is no function code, thus the source name is the package name in + # this case + callee_function_package = callee_function_file_name + + caller_function = code_documents[function.metadata.get('source')] + caller_function_package = get_package_name_file(caller_function).strip() + if (callee_package in self.get_package_names(function) and + callee_function_package == caller_function_package): + return True + # There is a qualifier identifier. + else: + identifier = parts[-2] + if identifier.startswith("return"): + identifier = identifier.replace("return", " ").strip() + if "(" in identifier: + identifier = identifier[identifier.index("(") + 1:] + if ")" in identifier: + identifier = identifier[:identifier.index(")")] + # verify that identifier resolves to the package name. if identifier is imported in same file, and if so , + # if it's the same as callee package name + # TODO - reduce list to only files in package + for doc in code_documents: + if function.metadata.get('source') == code_documents[doc].metadata.get('source'): + code_content = code_documents[doc].page_content + # Check if identifier is the package declaration itself (same-package call). + # e.g. a file in "package time" calling time.Parse() — only valid if + # this file IS in the callee package path. Two different modules can + # share the same Go package name (e.g. sigs.k8s.io/yaml and + # gopkg.in/yaml.v2 both declare "package yaml"), so fall through to + # the import check when the path doesn't match. + pkg_decl = re.search(rf"^package\s+{re.escape(identifier)}\s*$", + code_content, re.MULTILINE) + if pkg_decl: + caller_source = function.metadata.get('source', '') + if callee_package in caller_source: + return True + + identifier_is_imported = self.is_package_imported(code_content, identifier, callee_package) + if identifier_is_imported: + return True + + ## otherwise, the identifier is in the caller function body or in signature/receiver function argument. + + else: + # The identifier was not found as a package import or declaration. + # Check if it's a local variable whose type might resolve to the + # callee package. However, a file that doesn't import the callee + # package cannot use its types, so skip the expensive type lookup + # when the callee package is absent from the file's imports AND + # the file is not itself part of the callee package. + caller_source = function.metadata.get('source', '') + if callee_package not in caller_source: + caller_code = code_documents.get(caller_source) + if caller_code is not None: + has_callee_import = callee_package in caller_code.page_content + if not has_callee_import: + return False + + function_header = function.page_content[:function.page_content.index("{")] + regex_arguments = r"\([a-zA-Z0-9\s*,.]+\)" + match_regex = re.finditer(rf"^(\s*|\n){identifier}\s*(:=|=)\s*[^=]+\n*$", + caller_function_body, flags=re.MULTILINE) + matches = [match.group(0) for match in match_regex] + + if len(matches) > 0 or re.search(regex_arguments, function_header): + receiver_parts = list(parts[:-1]) + if receiver_parts and receiver_parts[0].startswith("return"): + receiver_parts[0] = receiver_parts[0].replace("return", "", 1).strip() + receiver_chain = ".".join(p for p in receiver_parts if p) + return self.__trace_down_package(expression=receiver_chain.strip(), code_documents=code_documents, + type_documents=type_documents, callee_package=callee_package, + fields_of_types=fields_of_types, + functions_local_variables_index=functions_local_variables_index, + caller_function_index=caller_function_index) + + # parameters = [tuple(e.replace(")", "").replace("(", "").split(",")) for e + # in re.findall(regex_arguments, function_header)[:2]] + # return check_types_from_callee_package(parameter=identifier, params=parameters, + # type_documents=type_documents, + # callee_package=callee_package, + # code_documents=code_documents, + # callee_function_file_name=callee_function_file_name + # ) + + return False + + def get_package_names(self, function: Document) -> list[str]: + package_names = list() + full_doc_path = str(function.metadata['source']) + parts = full_doc_path.split("/") + version = "" + if len(parts) > 4: + match = re.search(r"[vV][0-9]{1,2}", parts[4]) + if match and match.group(0): + version = f"/{match.group(0)}" + + if parts[0].startswith(self.dir_name_for_3rd_party_packages()) and len(parts) > 3: + package_names.append(f"{parts[1]}/{parts[2]}{version}") + package_names.append(f"{parts[1]}/{parts[2]}/{parts[3]}{version}") + else: + try: + package_names.append(f"{parts[0]}/{parts[1]}{version}") + package_names.append(f"{parts[0]}/{parts[1]}/{parts[2]}{version}") + # Standard library package + except IndexError as index_excp: + if len(parts) > 1: + package_names.append(f"{parts[0]}/{parts[1]}{version}") + else: + package_names.append(f"{parts[0]}{version}") + + return package_names + + def is_root_package(self, function: Document) -> bool: + return not function.metadata['source'].startswith(self.dir_name_for_3rd_party_packages()) + + def is_comment_line(self, line: str) -> bool: + return line.strip().startswith("//") + + def is_package_imported(self, code_content: str, identifier: str, callee_package: str = "") -> bool: + start_search_for_import = max(code_content.rfind("//import"), code_content.rfind("// import")) + if start_search_for_import > -1: + after_last_import_comment_index = start_search_for_import + 4 + else: + after_last_import_comment_index = 0 + after_last_import_comment = code_content[after_last_import_comment_index:] + first_import = after_last_import_comment.find("import") + last_import = after_last_import_comment.rfind("import") + # return after_last_import_comment, first_import, last_import + if first_import == last_import: + block_of_import = after_last_import_comment[first_import + 6:] + after_left_bracket = block_of_import.find("(") + before_right_bracket = block_of_import.find(")") + inside_block = block_of_import[after_left_bracket + 1: before_right_bracket] + position_of_identifier_in_import_block = inside_block.find(identifier) + if position_of_identifier_in_import_block != -1: + previous_end_of_line = inside_block[:position_of_identifier_in_import_block].rfind( + os.linesep) + alias_package_extended = inside_block[previous_end_of_line:].strip() + index_of_end_of_line = alias_package_extended.find(os.linesep) + row_of_identifier = alias_package_extended[:index_of_end_of_line] + if len(row_of_identifier.split(" ")) > 1: + package_name_of_alias = row_of_identifier.split(" ")[1].strip('"') + else: + package_name_of_alias = row_of_identifier.split(" ")[0].strip('"') + + if (package_name_of_alias.strip().lower() == callee_package.strip().lower() + or callee_package.strip().lower() in package_name_of_alias.strip().lower()): + return True + + # Checks if there is a dedicated import with alias and imported package name + else: + identifier_import_position = code_content.find(f"import {identifier}") + if identifier_import_position != -1: + start_of_package_name = code_content[identifier_import_position + len("import ") + + len(identifier):] + index_of_end_of_line = start_of_package_name.find(os.linesep) + package_name_to_check = start_of_package_name[:index_of_end_of_line] + if package_name_to_check.strip().lower() == callee_package.strip().lower(): + return True + # import without alias, in this case maybe package name contain alias + else: + # re.search(regex, caller_function_body, re.MULTILINE) + matching = re.search(rf"import [\'\"].*{identifier}[\'\"]", code_content) + if matching and matching.group(0): + import_line = code_content[matching.start():] + import_package_line = import_line[:import_line.find(os.linesep)].strip() + package_name = import_package_line.split(r"\s")[1] + if package_name.strip().lower() == callee_package.strip().lower(): + return True + return False + + def get_import_search_patterns(self, package_name: str) -> list[re.Pattern]: + escaped = re.escape(package_name) + return [ + re.compile(rf'^\s*(?:import\s+)?(?:[\w.]+\s+)?"({escaped}[^"]*)"', re.MULTILINE), + ] \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py new file mode 100644 index 000000000..0bd51f3be --- /dev/null +++ b/src/exploit_iq_commons/utils/functions_parsers/java_functions_parsers.py @@ -0,0 +1,3793 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import os +import re +from functools import lru_cache +from typing import Dict, Tuple, List, Optional +from tqdm import tqdm + +from langchain_core.documents import Document + +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser +from exploit_iq_commons.utils.java_utils import extract_jar_name, JAVA_METHOD_PRIM_TYPES, collect_fields_from_types, \ + is_java_type, is_java_method, extract_method_name_with_params, find_function, get_target_class_names, \ + strip_java_generics, JAVA_ANNOTATION_SYMBOL, extract_fqcn +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + +logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") + +PARAMETER = "parameter" + +LOCAL_VAR_USAGE = "local_var_usage" +LOCAL_IMPLICIT = "local_implicit" +LOCAL_INDIRECT_TYPES_INDICATIONS = [LOCAL_VAR_USAGE, LOCAL_IMPLICIT] +MODIFIERS = { + "public","protected","private","static","final","abstract","synchronized", + "native","strictfp","default","transient","volatile","sealed","non-sealed" +} + +TRAILING_ARR_RE = re.compile(r'(?:\s*\[\s*\]\s*)+$') +_ARRAY_SUFFIX_RE = re.compile(r"\s*(\[\])+$") +_WS_RE = re.compile(r"\s+") + +# Supports: +# /* inner-type: Inner */ +# /* inner-type: com.foo.Outer.Inner */ +# // inner-type: Inner +_INNER_TYPE_RE = re.compile( + r""" +(?: +/\*\s*inner-type\s*:\s*(?P.*?)\s*\*/ # block comment marker +| +//\s*inner-type\s*:\s*(?P[^\r\n]*) # line comment marker +) +""", + re.IGNORECASE | re.DOTALL | re.VERBOSE, + ) + +class JavaLanguageFunctionsParser(LanguageFunctionsParser): + def parse_all_type_struct_class_to_fields( + self, + types: list[Document], + inheritance_map: dict[Tuple[str, str], List[Tuple[str, str]]], + ) -> dict[Tuple[str, str], list[tuple[str, str]]]: + """ + Returns (TypeFqcn, doc.metadata['source']) -> [(memberName, memberType), ...] + + - Enums are ignored entirely. + - Interfaces: includes only fields (constants). + - Classes/records: includes fields. + - Also includes inherited fields (from superclasses and implemented interfaces) using BFS. + - Uses only the existing `inheritance_map`. + - Classifies *upstream* (parents/interfaces) via relative position indices to avoid pulling in descendants. + + NOTE: `inheritance_map` keys are now FQCNs and may include inner FQCNs. + """ + from collections import deque + from typing import Dict, Tuple, Any, List + + results: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + + # --------------------------- + # CHANGED: build a fast (source, parsed-name) -> fqcn resolver from inheritance_map keys + # --------------------------- + def _norm_fqcn(fqcn: str) -> str: + # normalize inner separator so suffix matching works regardless of '$' vs '.' + return fqcn.replace("$", ".") + + def _strip_package_prefix(fqcn_dot: str) -> str: + """ + Convert "org.foo.Outer.Inner" -> "Outer.Inner" by dropping package tokens. + Uses the heuristic: first segment starting with uppercase begins the type chain. + """ + parts = fqcn_dot.split(".") + for idx, tok in enumerate(parts): + if tok and tok[0].isupper(): + return ".".join(parts[idx:]) + return fqcn_dot + + def _put_unique(m: dict[str, str | None], k: str, v: str) -> None: + # store unique, mark ambiguous with None + prev = m.get(k) + if prev is None: + # already ambiguous or unset (if unset None isn't distinguishable), handle below: + if k in m: + return + m[k] = v + else: + if prev != v: + m[k] = None + + # Index by file source, because inheritance_map keys include (fqcn, source) + suffix_to_fqcn_by_source: dict[str, dict[str, str | None]] = {} + simple_to_fqcn_by_source: dict[str, dict[str, str | None]] = {} + + for (fqcn, src_path) in inheritance_map.keys(): + fqcn_dot = _norm_fqcn(fqcn) + no_pkg = _strip_package_prefix(fqcn_dot) # e.g. Outer.Inner + simple = no_pkg.rsplit(".", 1)[-1] # e.g. Inner + + suf_map = suffix_to_fqcn_by_source.setdefault(src_path, {}) + sim_map = simple_to_fqcn_by_source.setdefault(src_path, {}) + + _put_unique(suf_map, no_pkg, fqcn) # "Outer.Inner" -> fqcn (may be with '$') + _put_unique(sim_map, simple, fqcn) # "Inner" -> fqcn (if unique) + + # also allow direct normalized fqcn dot form as a key for rare parsers returning fqcn-ish names + _put_unique(suf_map, fqcn_dot, fqcn) + + def _resolve_fqcn_for_parsed_name(src_path: str, parsed_name: str) -> str | None: + """ + Map `parsed_name` produced by collect_fields_from_types() to the FQCN key used by inheritance_map. + + Fast-paths: + - If parsed_name includes dots (likely "Outer.Inner"), resolve via suffix map. + - Else resolve via unique simple-name map. + """ + if not parsed_name: + return None + name = parsed_name.strip() + if not name: + return None + name_dot = name.replace("$", ".") # be tolerant + + suf_map = suffix_to_fqcn_by_source.get(src_path) + sim_map = simple_to_fqcn_by_source.get(src_path) + + if "." in name_dot: + if suf_map: + fq = suf_map.get(name_dot) + return fq if fq else None + return None + + if sim_map: + fq = sim_map.get(name_dot) + return fq if fq else None + + return None + + # --------------------------- + # 1) Collect own fields per document + # CHANGED: per_key_fields now keyed by (fqcn, source) not (simple, source) + # --------------------------- + per_key_fields: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + for doc in tqdm(types, total=len(types), desc="Parsing class/interface/enum/record documents for members..."): + src = doc.page_content + source = doc.metadata["source"] + if not src: + continue + per_file = collect_fields_from_types(src, include_nested=True, include_anonymous=True) + + # Map each parsed type name to the inheritance_map fqcn key for this source file + for parsed_type_name, fields in per_file.items(): + fqcn = _resolve_fqcn_for_parsed_name(source, parsed_type_name) + if not fqcn: + # If we cannot map reliably to an inheritance_map key, skip (prevents wrong merges). + continue + per_key_fields[(fqcn, source)] = fields + + # --------------------------- + # 2) Precompute: row lists and O(1) index lookups for each key + # --------------------------- + rows: Dict[Tuple[str, str], List[Tuple[str, str]]] = inheritance_map + row_index: Dict[Tuple[str, str], Dict[Tuple[str, str], int]] = { + type_key: {neighbor: i for i, neighbor in enumerate(neighbors)} + for type_key, neighbors in rows.items() + } + + # --------------------------- + # 3) Upstream classifier with caching (performance) + # --------------------------- + upstream_cache: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + + def upstream_neighbors(type_key: Tuple[str, str]) -> List[Tuple[str, str]]: + """ + Return the direct upstream types (superclasses and implemented interfaces) for `type_key` + using the relative-index rule over `rows` and `row_index`. + """ + cached = upstream_cache.get(type_key) + if cached is not None: + return cached + + type_row = rows.get(type_key) + if not type_row: + upstream_cache[type_key] = [] + return [] + + type_index_map = row_index.get(type_key, {}) + upstreams: List[Tuple[str, str]] = [] + seen_candidates: set[Tuple[str, str]] = set() + + for candidate in type_row: + if candidate == type_key or candidate in seen_candidates: + continue + + idx_candidate_in_type = type_index_map.get(candidate, 1 << 30) + + candidate_index_map = row_index.get(candidate) + if not candidate_index_map: + continue + idx_type_in_candidate = candidate_index_map.get(type_key) + if idx_type_in_candidate is None: + continue + + if idx_candidate_in_type < idx_type_in_candidate: + upstreams.append(candidate) + seen_candidates.add(candidate) + + upstream_cache[type_key] = upstreams + return upstreams + + # --------------------------- + # 4) Merge own + inherited fields per type using BFS over upstream neighbors. + # --------------------------- + for type_key, own_fields in per_key_fields.items(): + merged: List[Tuple[str, str]] = [] + have_names: set[str] = set() # child-over-parent shadowing + + for fname, ftype in own_fields: + if fname not in have_names: + merged.append((fname, ftype)) + have_names.add(fname) + + q: deque[Tuple[str, str]] = deque(upstream_neighbors(type_key)) + visited: set[Tuple[str, str]] = set() + + while q: + upstream_key = q.popleft() + if upstream_key in visited: + continue + visited.add(upstream_key) + + upstream_fields = per_key_fields.get(upstream_key) + if upstream_fields: + for fname, ftype in upstream_fields: + if fname not in have_names: + merged.append((fname, ftype)) + have_names.add(fname) + + for next_upstream in upstream_neighbors(upstream_key): + if next_upstream not in visited: + q.append(next_upstream) + + results[type_key] = merged + + # --------------------------- + # 5) Include types that had no own fields but may inherit some + # --------------------------- + for type_key in rows.keys(): + if type_key in results: + continue + merged: List[Tuple[str, str]] = [] + have_names: set[str] = set() + + q: deque[Tuple[str, str]] = deque(upstream_neighbors(type_key)) + visited: set[Tuple[str, str]] = set() + + while q: + upstream_key = q.popleft() + if upstream_key in visited: + continue + visited.add(upstream_key) + + upstream_fields = per_key_fields.get(upstream_key) + if upstream_fields: + for fname, ftype in upstream_fields: + if fname not in have_names: + merged.append((fname, ftype)) + have_names.add(fname) + + for next_upstream in upstream_neighbors(upstream_key): + if next_upstream not in visited: + q.append(next_upstream) + + if merged: + results[type_key] = merged + + return results + + def get_dummy_function(self, function_name): + return f"public void {function_name + '()' + '{}'}" + + def __get_optional_inner_type(self, method_source: str) -> Optional[str]: + """ + Extract the optional inner type marker from a method's extracted source. + Returns: + - the inner type string (trimmed) if a marker exists + - None otherwise + """ + m = _INNER_TYPE_RE.search(method_source) + if not m: + return None + + inner = m.group("block") if m.group("block") is not None else m.group("line") + inner = (inner or "").strip() + return inner or None + + def get_class_name_from_class_function(self, func: Document) -> str: + """ + Return the Java FQCN (fully-qualified class name) inferred from func.metadata['source']. + + Assumptions (per your note): + - Either: + dependencies-sources/--sources//.java + or: + .../src/main/java//.java + - Package path is the directory structure after the marker, followed by the class filename. + """ + source = (func.metadata.get("source") or "").strip() + if not source: + return "" + + fqcn = extract_fqcn(source) + + inner_type = self.__get_optional_inner_type(func.page_content) + if inner_type: + return fqcn + "." + inner_type + + return fqcn + + def get_function_reserved_word(self) -> str: + return "" + + def get_type_reserved_word(self) -> str: + pass + + def is_searchable_file_name(self, function: Document) -> bool: + pass + + def is_function(self, function: Document) -> bool: + return is_java_method(function.page_content) + + def is_supported_file_extensions(self, extension: str) -> bool: + return extension in self.supported_files_extensions() + + def supported_files_extensions(self) -> list[str]: + return [".java"] + + def get_comment_line_notation(self) -> str: + return "//" + + def dir_name_for_3rd_party_packages(self) -> str: + return "dependencies-sources" + + _uber_jar_dirs: frozenset[str] = frozenset() + + def set_uber_jar_dirs(self, dirs: frozenset[str]) -> None: + self._uber_jar_dirs = dirs + + def _is_same_artifact(self, source_a: str, source_b: str) -> bool: + """Check if two source paths are from the same third-party JAR artifact. + + Returns False for uber-JARs (detected by file count threshold) since + shaded libraries within an uber-JAR are unrelated artifacts. + """ + prefix = self.dir_name_for_3rd_party_packages() + "/" + if not (source_a and source_b and source_a.startswith(prefix) and source_b.startswith(prefix)): + return False + jar_a = source_a[len(prefix):].split("/", 1)[0] + jar_b = source_b[len(prefix):].split("/", 1)[0] + if jar_a != jar_b: + return False + return jar_a not in self._uber_jar_dirs + + def is_exported_function(self, function: Document, documents_of_full_sources: dict[str, Document]) -> bool: + return True + + def get_function_name(self, function: Document) -> str: + # --- constants & tiny char helpers (scoped) --- + JAVA_CTRL = {"if", "for", "while", "switch", "catch", "synchronized", "try"} + # Disallow these as the *first* token of a "type" to avoid false positives (e.g., record header) + DISALLOW_TYPE_HEAD = {"class","interface","enum","record","new","return"} + + # Method modifiers we may see before a return type / constructor name + MODIFIERS = { + "public","protected","private","static","final","abstract","synchronized", + "native","strictfp","default","transient","volatile","sealed" + # 'non-sealed' is two tokens; irrelevant for methods here + } + + WS = " \t\r\n\f\v" + DIG = "0123456789" + IDF_EXTRA = "_$" + + def is_id_start(ch: str) -> bool: + return ('A' <= ch <= 'Z') or ('a' <= ch <= 'z') or (ch in IDF_EXTRA) + + def is_id_part(ch: str) -> bool: + return is_id_start(ch) or (ch in DIG) + + def prev_non_ws(s: str, idx: int): + i = idx + while i >= 0 and s[i] in WS: + i -= 1 + return i + + # --- pass 1: strip //, /* */, and string/char literals (keeps layout lightweight) --- + s = function.page_content + n = len(s) + out = [] + i = 0 + while i < n: + c = s[i] + if c == '/' and i + 1 < n and s[i+1] == '/': + i += 2 + while i < n and s[i] != '\n': + i += 1 + out.append('\n' if i < n else '') + if i < n: i += 1 + elif c == '/' and i + 1 < n and s[i+1] == '*': + i += 2 + while i + 1 < n and not (s[i] == '*' and s[i+1] == '/'): + i += 1 + i = min(i + 2, n) + out.append(' ') + elif c in ("'", '"'): + q = c + i += 1 + while i < n: + if s[i] == '\\': + i += 2 + elif s[i] == q: + i += 1 + break + else: + i += 1 + out.append(' ') + else: + out.append(c) + i += 1 + s = ''.join(out) + n = len(s) + + # --- tiny scanners on cleaned source --- + def skip_ws(idx: int) -> int: + while idx < n and s[idx] in WS: + idx += 1 + return idx + + def skip_ws_back(idx: int) -> int: + while idx >= 0 and s[idx] in WS: + idx -= 1 + return idx + + def match_paren_close(start: int) -> int: + """start at '(' and find matching ')' with nesting inside the parameter list.""" + depth, i = 0, start + while i < n: + ch = s[i] + if ch == '(': + depth += 1 + elif ch == ')': + depth -= 1 + if depth == 0: + return i + i += 1 + return -1 + + def skip_angle(i: int) -> int: + """Skip a balanced <...> block (handles nesting); return index just after the matching '>'.""" + depth, j = 0, i + while j < n: + ch = s[j] + if ch == '<': + depth += 1 + elif ch == '>': + depth -= 1 + if depth == 0: + return j + 1 + j += 1 + return i # malformed + + def skip_annotation(i: int) -> int: + """Skip @Anno and optional (args).""" + j = i + 1 + if j < n and is_id_start(s[j]): + j += 1 + while j < n and is_id_part(s[j]): + j += 1 + j = skip_ws(j) + if j < n and s[j] == '(': + end = match_paren_close(j) + if end == -1: + return i + j = end + 1 + return j + + def begins_with_lambda() -> bool: + """Return True iff the snippet itself starts with a lambda: '(...) ->' or 'x ->'.""" + i0 = skip_ws(0) + if i0 >= n: + return False + if s[i0] == '(': + end = match_paren_close(i0) + if end != -1: + j = skip_ws(end + 1) + return j + 1 < n and s[j] == '-' and s[j + 1] == '>' + return False + if is_id_start(s[i0]): + j = i0 + 1 + while j < n and is_id_part(s[j]): + j += 1 + k = skip_ws(j) + return k + 1 < n and s[k] == '-' and s[k + 1] == '>' + return False + + def parse_return_type(hstart: int, name_start: int) -> int: + """ + Try to parse a return type in [hstart, name_start). + Return index right after the type (including any array suffixes), + or -1 if no valid type. Ensures there is at least one char (ws or []), + and *only* ws/[] between type end and name (guards against "int x = foo("). + """ + i = skip_ws(hstart) + # annotations + modifiers + while True: + i = skip_ws(i) + if i < n and s[i] == JAVA_ANNOTATION_SYMBOL: + i = skip_annotation(i); continue + j = i + if j < n and is_id_start(s[j]): + j += 1 + while j < n and is_id_part(s[j]): + j += 1 + if s[i:j] in MODIFIERS: + i = j + continue + break + + i = skip_ws(i) + # leading method type-parameters: + if i < n and s[i] == '<': + new_i = skip_angle(i) + if new_i == i: # malformed + return -1 + i = new_i + i = skip_ws(i) + + # type-use annotations before the return type + while i < n and s[i] == JAVA_ANNOTATION_SYMBOL: + i = skip_annotation(i) + i = skip_ws(i) + + # parse 'void' or qualified identifier (with optional generic args) + j = i + if j + 4 <= n and s[j:j+4] == 'void' and (j + 4 >= n or not is_id_part(s[j+4])): + j += 4 + else: + if j < n and is_id_start(s[j]): + t0 = j + j += 1 + while j < n and is_id_part(s[j]): + j += 1 + # The first identifier cannot be one of these keywords + if s[t0:j] in DISALLOW_TYPE_HEAD: + return -1 + # dotted qualifiers + while True: + jj = skip_ws(j) + if jj < n and s[jj] == '.': + j = jj + 1 + if j < n and is_id_start(s[j]): + j += 1 + while j < n and is_id_part(s[j]): + j += 1 + continue + return -1 + break + else: + return -1 + # optional generic type arguments on the return type + jj = skip_ws(j) + if jj < n and s[jj] == '<': + nx = skip_angle(jj) + if nx == jj: + return -1 + j = nx + + # Optional array suffixes on return type, with arbitrary spacing inside []. + k = j + while True: + kk = skip_ws(k) + if kk + 1 < n and s[kk] == '[': + kk += 1 + kk = skip_ws(kk) + if kk < n and s[kk] == ']': + kk += 1 + k = kk + continue + break + + # Must have a (non-empty) gap between type end and name (constructor guard) + if k >= name_start: + return -1 + + # Gap must contain only whitespace or [] decorations + scan = k + while scan < name_start: + if s[scan] in WS: + scan += 1 + elif s[scan] == '[': + scan += 1 + scan = skip_ws(scan) + if scan < n and s[scan] == ']': + scan += 1 + else: + return -1 + else: + return -1 + return k + + def is_constructor_header(hstart: int, name_start: int) -> bool: + """ + Return True iff after skipping annotations, modifiers, and leading method type-parameters, + the next non-ws token starts exactly at 'name_start' (i.e., no return type present). + """ + i = skip_ws(hstart) + # skip annotations + modifiers + while True: + i = skip_ws(i) + if i < n and s[i] == JAVA_ANNOTATION_SYMBOL: + i = skip_annotation(i); continue + j = i + if j < n and is_id_start(s[j]): + j += 1 + while j < n and is_id_part(s[j]): + j += 1 + if s[i:j] in MODIFIERS: + i = j + continue + break + + i = skip_ws(i) + # leading method type-parameters: + if i < n and s[i] == '<': + ni = skip_angle(i) + if ni == i: + return False + i = ni + i = skip_ws(i) + + # type-use annotations (rare before constructor name, but tolerate) + while i < n and s[i] == JAVA_ANNOTATION_SYMBOL: + i = skip_annotation(i) + i = skip_ws(i) + + # Now, we should be exactly at the constructor simple name if it's a ctor + return i == name_start + + # --- lambda-at-beginning guard (do NOT classify methods that merely contain lambdas) --- + if begins_with_lambda(): + return "lambda" # TODO handle lambda correctly + + # --- main scan: seek '(' and validate the candidate as a declaration --- + i = 0 + while True: + open_idx = s.find('(', i) + if open_idx == -1: + break # no more candidates + + # candidate name is the identifier immediately before '(' + j = skip_ws_back(open_idx - 1) + if j < 0 or not is_id_part(s[j]): + i = open_idx + 1 + continue + + end = j + 1 + start = j + while start - 1 >= 0 and is_id_part(s[start - 1]): + start -= 1 + name = s[start:end] + + # skip control keywords (if(), for(), etc.) + if name in JAVA_CTRL: + i = open_idx + 1 + continue + + # skip chained calls / annotations like obj.m( or @Anno( + prev = prev_non_ws(s, start - 1) + if prev is not None and s[prev] in ('.', JAVA_ANNOTATION_SYMBOL): + i = open_idx + 1 + continue + + # find matching ')' + close_idx = match_paren_close(open_idx) + if close_idx == -1: + i = open_idx + 1 + continue + + # after ')': optional ws + optional 'throws ...' + then '{' or ';' + p = skip_ws(close_idx + 1) + # lambda guard: "() ->" pattern directly after params + if p + 1 < n and s[p] == '-' and s[p + 1] == '>': + # it's a lambda parameter list, not a method decl + i = open_idx + 1 + continue + + if p + 6 <= n and s[p:p+6] == 'throws' and (p + 6 >= n or s[p+6] in WS): + p = p + 6 + while p < n: + p = skip_ws(p) + ch = s[p] if p < n else '' + if ch in '{;': + break + if ch == '<': + p = skip_angle(p) + elif is_id_start(ch): + # qualified exception with optional generic args and commas + p += 1 + while p < n and is_id_part(s[p]): + p += 1 + while True: + q = skip_ws(p) + if q < n and s[q] == '.': + p = q + 1 + if p < n and is_id_start(s[p]): + p += 1 + while p < n and is_id_part(s[p]): + p += 1 + continue + break + q = skip_ws(p) + if q < n and s[q] == '<': + p = skip_angle(q) + else: + p = q + q = skip_ws(p) + if q < n and s[q] == ',': + p = q + 1 + else: + p = q + else: + p += 1 + p = skip_ws(p) + + if p >= n or s[p] not in '{;': + # likely a call or lambda, not a declaration + i = open_idx + 1 + continue + + # Find header start (after previous ';' or '{' or '}' before the name) + h = start - 1 + while h >= 0 and s[h] not in ';{}': + h -= 1 + h_start = h + 1 + + # Either a regular method (has a return type) OR a constructor (no return type) + if parse_return_type(h_start, start) != -1 or is_constructor_header(h_start, start): + return name + + # otherwise, keep scanning + i = open_idx + 1 + + # No declaration found, and snippet didn't begin with a lambda + return "" + + def search_for_called_function( + self, + caller_function: Document, + callee_function_name: str, + callee_function: Document, + callee_function_package: str, + code_documents: dict[str, Document], + type_documents: list[Document], + callee_function_file_name: str, + fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict], + documents_of_functions: list[Document], + type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]], + ) -> bool: + """ + Scan `caller_function` for occurrences that may call `callee_function_name` (or a constructor) + and confirm via `__check_identifier_resolved_to_callee_function_package`. Supports: + + - Regular calls: `foo(...)`, `obj.foo(...)`, `Type.staticFoo(...)`, chains, casts, etc. + - Constructor calls: `new Type(...)` (when the callee is a constructor). + - Method references: `left-hand-side::method` → synthesized as `left-hand-side.method(` for resolution. + - Constructor references: `left-hand-side::new` → synthesized as `new left-hand-side(` for resolution. + """ + def _find_matching_paren(s: str, open_idx: int) -> int: + """Find matching ')' for the '(' at open_idx. Ignores strings/char literals.""" + depth = 0 + i = open_idx + n = len(s) + in_str = in_chr = False + esc = False + while i < n: + ch = s[i] + if in_str: + if esc: + esc = False + elif ch == '\\': + esc = True + elif ch == '"': + in_str = False + i += 1 + continue + if in_chr: + if esc: + esc = False + elif ch == '\\': + esc = True + elif ch == "'": + in_chr = False + i += 1 + continue + if ch == '"': + in_str = True + i += 1 + continue + if ch == "'": + in_chr = True + i += 1 + continue + if ch == '(': + depth += 1 + elif ch == ')': + depth -= 1 + if depth == 0: + return i + i += 1 + return -1 + + def _next_token_after(s: str, idx: int) -> str: + """Return the next non-space token after idx: '{', 'throws', or first char/empty.""" + n = len(s) + i = idx + 1 + while i < n and s[i].isspace(): + i += 1 + if s.startswith('throws', i) and (i + 6 == n or (not s[i + 6].isalnum() and s[i + 6] != '_')): + return 'throws' + return s[i] if i < n else '' + + # Precise left-scan to the start of the expression that owns this call. + BOUNDARY_CHARS = set(';,=+-*/%!?::&|^<>{}\n') + + def _expr_start_left(s: str, pos: int, max_back: int = 512) -> int: + i = pos - 1 + in_str = in_chr = False + esc = False + dp = db = dbr = 0 # depths for (), [], {} + limit = max(0, pos - max_back) + while i >= limit: + ch = s[i] + if in_str: + if esc: + esc = False + elif ch == '\\': + esc = True + elif ch == '"': + in_str = False + i -= 1 + continue + if in_chr: + if esc: + esc = False + elif ch == '\\': + esc = True + elif ch == "'": + in_chr = False + i -= 1 + continue + if ch == '"': + in_str = True + i -= 1 + continue + if ch == "'": + in_chr = True + i -= 1 + continue + + if ch == ')': + dp += 1 + i -= 1 + continue + if ch == '(': + dp = max(0, dp - 1) + i -= 1 + continue + if ch == ']': + db += 1 + i -= 1 + continue + if ch == '[': + db = max(0, db - 1) + i -= 1 + continue + if ch == '}': + dbr += 1 + i -= 1 + continue + if ch == '{': + dbr = max(0, dbr - 1) + i -= 1 + continue + + if dp == db == dbr == 0 and ch in BOUNDARY_CHARS: + return i + 1 + + i -= 1 + return limit + + # Method-ref left-hand-side extractor: find minimal "lhs" immediately before '::' + def _method_ref_lhs_start(s: str, dc_idx: int, max_back: int = 512) -> int: + """ + Return the start index of the LHS expression immediately preceding '::' at dc_idx. + Stops at top-level delimiters that commonly precede a method reference in args + (notably '(' and ',') so we don't capture an outer call prefix. + """ + i = dc_idx - 1 + while i >= 0 and s[i].isspace(): + i -= 1 + + in_str = in_chr = False + esc = False + dp = db = dbr = da = 0 # (), [], {}, <> (generics) + limit = max(0, dc_idx - max_back) + + REF_BOUNDARY = set(';,=+-*/%!?&|^:\n,(') + + while i >= limit: + ch = s[i] + + if in_str: + if esc: + esc = False + elif ch == '\\': + esc = True + elif ch == '"': + in_str = False + i -= 1 + continue + if in_chr: + if esc: + esc = False + elif ch == '\\': + esc = True + elif ch == "'": + in_chr = False + i -= 1 + continue + + if ch == '"': + in_str = True + i -= 1 + continue + if ch == "'": + in_chr = True + i -= 1 + continue + + if ch == ')': + dp += 1 + i -= 1 + continue + if ch == '(': + if dp > 0: + dp -= 1 + i -= 1 + continue + if db == dbr == da == 0: + return i + 1 + i -= 1 + continue + + if ch == ']': + db += 1 + i -= 1 + continue + if ch == '[': + if db > 0: + db -= 1 + i -= 1 + continue + if dp == dbr == da == 0: + return i + 1 + i -= 1 + continue + + if ch == '}': + dbr += 1 + i -= 1 + continue + if ch == '{': + if dbr > 0: + dbr -= 1 + i -= 1 + continue + if dp == db == da == 0: + return i + 1 + i -= 1 + continue + + if ch == '>': + da += 1 + i -= 1 + continue + if ch == '<': + if da > 0: + da -= 1 + i -= 1 + continue + if dp == db == dbr == 0: + return i + 1 + i -= 1 + continue + + if dp == db == dbr == da == 0 and ch in REF_BOUNDARY: + return i + 1 + + i -= 1 + + return limit + + # --------------------------- + # Extract caller function body + # --------------------------- + src = caller_function.page_content + try: + lo = src.index("{") + hi = src.rfind("}") + caller_function_body = src[lo + 1:hi] + except ValueError: + caller_function_body = src + + # --------------------------- + # Patterns + # --------------------------- + method_pat = re.compile( + rf'(?:\breturn\b\s*\(?[^;]*?)?(?:[\w$()\[\].]*?\.?)?\b{re.escape(callee_function_name)}\s*\(', + re.MULTILINE, + ) + + callee_function_source = callee_function.metadata['source'] + + # CHANGED: get_class_name_from_class_function now returns FQCN (possibly inner). + # Use it as the declaring FQCN for inheritance lookup AND for constructor name derivation. + declaring_fqcn = self.get_class_name_from_class_function(callee_function) + + # CHANGED: normalize inner separator to Java syntax ('.') for matching source text. + declaring_fqcn_dot = declaring_fqcn.replace('$', '.') + declaring_simple = declaring_fqcn_dot.rsplit('.', 1)[-1] + + # Early exit: if the caller's source file cannot reference the declaring + # class, skip all pattern matching and type resolution work. + # Checks three ways the declaring class could be visible: + # 1. Simple class name appears in the source text (direct usage or import) + # 2. Wildcard import of the declaring class's package (import pkg.*) + # 3. Caller is in the same package as the declaring class + # 4. Caller and callee are in the same JAR (intra-library polymorphic + # calls through interfaces don't require the concrete class name) + # In large repos, ~90%+ of source files fail all checks, avoiding + # expensive regex/type resolution for each one. + if declaring_fqcn: + _caller_src = caller_function.metadata.get('source') + _caller_full_doc = code_documents.get(_caller_src) + if _caller_full_doc: + _full_text = _caller_full_doc.page_content + if declaring_simple not in _full_text: + _declaring_pkg = declaring_fqcn_dot.rsplit('.', 1)[0] + if f"import {_declaring_pkg}.*" not in _full_text: + _pkg_m = re.search(r'^\s*package\s+([\w.]+)\s*;', _full_text, re.MULTILINE) + if not _pkg_m or _pkg_m.group(1) != _declaring_pkg: + if not self._is_same_artifact(_caller_src, callee_function_file_name): + return False + + # CHANGED: support ctor targeting by simple name, no-package inner name, or fqcn. + def _strip_package_prefix(fqcn_dot: str) -> str: + parts = fqcn_dot.split('.') + for idx, tok in enumerate(parts): + if tok and tok[0].isupper(): + return '.'.join(parts[idx:]) + return fqcn_dot # fallback + + declaring_no_pkg = _strip_package_prefix(declaring_fqcn_dot) + + # Treat as constructor-target if caller asked for the declaring type name (simple/no-pkg/fqcn). + is_ctor_target = callee_function_name in { + declaring_simple, + declaring_no_pkg, + declaring_fqcn_dot, + declaring_fqcn, # just in case callers still use '$' form + } + + # CHANGED: ctor_pat must work when declaring_simple is derived from an FQCN/inner FQCN. + # We match "new <...>(..." where <...> ends with: + # - Simple class name (Inner) + # - No-package inner chain (Outer.Inner) + # - Fully-qualified name (pkg.Outer.Inner) + ctor_pat = None + if is_ctor_target: + ctor_variants = [] + # Prefer longest first to reduce backtracking. + for v in (declaring_fqcn_dot, declaring_no_pkg, declaring_simple): + if v and v not in ctor_variants: + ctor_variants.append(v) + ctor_variants.sort(key=len, reverse=True) + ctor_alt = "|".join(re.escape(v) for v in ctor_variants) + ctor_pat = re.compile( + rf'\bnew\s+((?:[A-Za-z_$][\w$]*\.)*(?:{ctor_alt}))\s*\(' + ) + + # Method reference patterns (allow optional method type args: Type::m) + methodref_pat = re.compile( + rf'::\s*(?:<[^>]*>\s*)?{re.escape(callee_function_name)}\b' + ) + ctorref_pat = re.compile(r'::\s*new\b') if is_ctor_target else None + + seen_slices: set[tuple[int, int]] = set() + + # --------------------------- + # Target class names (inheritance) + # --------------------------- + # CHANGED: key uses declaring_fqcn (inner-aware), not extract_fqcn(file_path). + key = (declaring_fqcn, callee_function_source) + if "dummy" not in callee_function_file_name: + target_class_names = get_target_class_names(type_inheritance[key]) + else: + target_class_names = frozenset([declaring_fqcn]) + + # --------------------------- + # Helpers + # --------------------------- + def _process_call(start_idx: int, open_paren_pos: int) -> bool: + close_paren_pos = _find_matching_paren(caller_function_body, open_paren_pos) + if close_paren_pos == -1: + return False + + start_ctx = _expr_start_left(caller_function_body, start_idx, max_back=512) + end_ctx = close_paren_pos + 1 + slice_key = (start_ctx, end_ctx) + if slice_key in seen_slices: + return False + seen_slices.add(slice_key) + + ident_snippet = caller_function_body[start_ctx:open_paren_pos + 1] + + if self.__check_identifier_resolved_to_callee_function_package( + function=caller_function, + identifier_function=ident_snippet, + callee_package=callee_function_package, + code_documents=code_documents, + type_documents=type_documents, + callee_function_file_name=callee_function_file_name, + fields_of_types=fields_of_types, + functions_local_variables_index=functions_local_variables_index, + target_class_names=target_class_names, + documents_of_functions=documents_of_functions, + callee_function_name=callee_function_name, + type_inheritance=type_inheritance, + callee_declaring_fqcn=declaring_fqcn, + ): + logger.debug( + "__check_identifier_resolved_to_callee_function_package resolved successfully - " + f"callee_function_name={callee_function_name}, identifier_function={ident_snippet}, " + f"target_class_names={target_class_names}, \ncaller_function_source={caller_function.metadata['source']}" + f", \ncaller_function={caller_function.page_content}" + ) + return True + + logger.debug( + "__check_identifier_resolved_to_callee_function_package resolved unsuccessfully - " + f"callee_function_name={callee_function_name}, identifier_function={ident_snippet}, " + f"target_class_names={target_class_names}, \ncaller_function_source={caller_function.metadata['source']}" + f", \ncaller_function={caller_function.page_content}" + ) + return False + + def _process_method_ref(dc_idx: int, ref_len: int, make_ctor: bool) -> bool: + """ + dc_idx: index of the first ':' in the '::' token + ref_len: length of the matched '::...' + """ + lhs_start = _method_ref_lhs_start(caller_function_body, dc_idx, max_back=512) + lhs = caller_function_body[lhs_start:dc_idx].strip() + if not lhs: + return False + + slice_key = (lhs_start, dc_idx + ref_len) + if slice_key in seen_slices: + return False + seen_slices.add(slice_key) + + ident_snippet = (f"new {lhs}(") if make_ctor else (f"{lhs}.{callee_function_name}(") + + return self.__check_identifier_resolved_to_callee_function_package( + function=caller_function, + identifier_function=ident_snippet, + callee_package=callee_function_package, + code_documents=code_documents, + type_documents=type_documents, + callee_function_file_name=callee_function_file_name, + fields_of_types=fields_of_types, + functions_local_variables_index=functions_local_variables_index, + target_class_names=target_class_names, + documents_of_functions=documents_of_functions, + callee_function_name=callee_function_name, + type_inheritance=type_inheritance, + callee_declaring_fqcn=declaring_fqcn, + ) + + # --------------------------- + # 1) Constructor matches (only when target is a ctor) + # --------------------------- + if is_ctor_target and ctor_pat: + for m in ctor_pat.finditer(caller_function_body): + open_paren_pos = m.end(0) - 1 + if _process_call(m.start(), open_paren_pos): + return True + else: + # --------------------------- + # 2) Regular method matches + # --------------------------- + for m in method_pat.finditer(caller_function_body): + open_paren_pos = m.end(0) - 1 + close_paren_pos = _find_matching_paren(caller_function_body, open_paren_pos) + if close_paren_pos == -1: + continue + + nxt = _next_token_after(caller_function_body, close_paren_pos) + if nxt == '{' or nxt == 'throws': + continue + + if _process_call(m.start(), open_paren_pos): + return True + + # --------------------------- + # 3) Method reference matches + # --------------------------- + for m in methodref_pat.finditer(caller_function_body): + if _process_method_ref(m.start(), m.end() - m.start(), make_ctor=False): + return True + + if ctorref_pat: + for m in ctorref_pat.finditer(caller_function_body): + if _process_method_ref(m.start(), m.end() - m.start(), make_ctor=True): + return True + + return False + + def __check_identifier_resolved_to_callee_function_package( + self, + function: Document, + identifier_function: str, + callee_package: str, + code_documents: dict[str, Document], + type_documents: list[Document], + callee_function_file_name: str, + fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict], + target_class_names: frozenset[str], + documents_of_functions: list[Document], + callee_function_name: str, + type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]], + callee_declaring_fqcn: str = "", + ) -> bool: + """ + Decide if the found call expression (`identifier_function`) in `function` actually targets + the method/constructor in `callee_package`, supporting: + - Unqualified calls inside same package. + - Static calls: ClassName.method(…) or pkg.ClassName.method(…) + - Constructor receivers: new Type(...).method(…) + - Direct constructors: new pkg.Type(...) + - Arbitrary-depth chains: A.staticB(...).c().d(...).targetMethod(…) + - Receiver that is a method call: foo().bar(…) → resolve foo()'s return type. + - Unqualified helper call in same class: helper(…) → resolve helper() return type. + - Receiver that is a cast: ((Type) x).target(…) → use Type as a strong hint. + - SUPER handling uses `target_class_names` (parents/children set) — no file header parsing. + + NOTE (CHANGED): `target_class_names` now contains FQCNs (not simple names), so all calls into + `_type_token_matches_callee(...)` are now made with FQCN candidates only. + """ + def _strip_return_and_ws(s: str) -> str: + s = s.strip() + if s.startswith("return"): + s = s[6:].strip() + return s + + def _strip_leading_this(s: str) -> str: + return s.replace("this.", "", 1) if s.startswith("this.") else s + + def _is_upper_camel(token: str) -> bool: + return bool(token) and token[0].isalpha() and token[0].upper() == token[0] + + def _split_top_level_dots(expr: str) -> list[str]: + """Split by dots ignoring dots inside (), [], {}, <>, and strings/chars.""" + parts, buf = [], [] + dp = db = dbr = da = 0 + in_str = in_chr = False + i, n = 0, len(expr) + while i < n: + ch = expr[i] + if in_str: + buf.append(ch) + if ch == '\\' and i + 1 < n: + buf.append(expr[i + 1]); i += 1 + elif ch == '"': + in_str = False + i += 1; continue + if in_chr: + buf.append(ch) + if ch == '\\' and i + 1 < n: + buf.append(expr[i + 1]); i += 1 + elif ch == "'": + in_chr = False + i += 1; continue + + if ch == '"': + in_str = True; buf.append(ch); i += 1; continue + if ch == "'": + in_chr = True; buf.append(ch); i += 1; continue + if ch == '(': + dp += 1; buf.append(ch); i += 1; continue + if ch == ')': + dp = max(0, dp - 1); buf.append(ch); i += 1; continue + if ch == '[': + db += 1; buf.append(ch); i += 1; continue + if ch == ']': + db = max(0, db - 1); buf.append(ch); i += 1; continue + if ch == '{': + dbr += 1; buf.append(ch); i += 1; continue + if ch == '}': + dbr = max(0, dbr - 1); buf.append(ch); i += 1; continue + if ch == '<': + da += 1; buf.append(ch); i += 1; continue + if ch == '>': + if da > 0: da -= 1 + buf.append(ch); i += 1; continue + + if ch == '.' and dp == db == dbr == da == 0: + parts.append(''.join(buf).strip()); buf = [] + i += 1; continue + + buf.append(ch); i += 1 + parts.append(''.join(buf).strip()) + return [p for p in parts if p] + + def _strip_call_parens(s: str) -> str: + """Remove a single trailing '(...)' from the end if balanced.""" + s2 = s.rstrip() + if not s2.endswith(')'): + return s + depth = 0 + in_str = in_chr = False + i = len(s2) - 1 + while i >= 0: + ch = s2[i] + if in_str: + if ch == '\\': i -= 2; continue + if ch == '"': in_str = False; i -= 1; continue + i -= 1; continue + if in_chr: + if ch == '\\': i -= 2; continue + if ch == "'": in_chr = False; i -= 1; continue + i -= 1; continue + if ch == '"': in_str = True; i -= 1; continue + if ch == "'": in_chr = True; i -= 1; continue + if ch == ')': depth += 1; i -= 1; continue + if ch == '(': + depth -= 1 + if depth == 0: + return s2[:i].rstrip() + i -= 1; continue + i -= 1 + return s + + def _is_fqcn_like(token: str) -> bool: + t = token.replace('$', '.') + if '.' in t: + pkg, _, cls = t.rpartition('.') + return bool(pkg) and _is_upper_camel(cls[:1] + cls[1:]) + return _is_upper_camel(token) + + def _find_method_return_type_in_file(source_text: str, method_name: str) -> str | None: + sig = re.compile( + rf""" + (?\[\]?]*?) + \s+{re.escape(method_name)}\s*\( + """, + re.VERBOSE, + ) + m = sig.search(source_text) + if not m: + return None + ret = m.group(1).strip() + ret = re.sub(r'<[^<>]*>', '', ret) + ret = re.sub(r'\s*(\[\])+$', '', ret) + return ret or None + + _CAST_RE = re.compile( + r'^\(\s*([A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*(?:<[^()<>]*>)?(?:\s*(?:\[\]))*)\s*\)\s*(.*)\Z' + ) + + def _peel_leading_cast(expr: str) -> tuple[str | None, str]: + s = expr.strip() + changed = True + while changed and s.startswith('(') and s.endswith(')'): + changed = False + depth = 0 + for i, ch in enumerate(s): + if ch == '(': + depth += 1 + elif ch == ')': + depth -= 1 + if depth == 0 and i != len(s) - 1: + break + else: + s = s[1:-1].strip() + changed = True + last_type = None + rest = s + while True: + m = _CAST_RE.match(rest) + if not m: + break + last_type = m.group(1) + rest = m.group(2).strip() + return last_type, rest + + def _extract_recv_and_method(snippet: str) -> tuple[str | None, str | None]: + s = snippet.strip() + m = re.search(r'([A-Za-z_$][\w$]*)\s*\($', s) + if not m: + last = None + for mm in re.finditer(r'([A-Za-z_$][\w$]*)\s*\(', s): + last = mm + m = last + if not m: + return None, None + + method = m.group(1) + method_start = m.start(1) + + i = method_start - 1 + while i >= 0 and s[i].isspace(): + i -= 1 + if i < 0 or s[i] != '.': + return None, method + + dot_idx = i + + def _match_paren_reverse(text: str, close_idx: int) -> int: + depth = 1 + i2 = close_idx - 1 + in_str = in_chr = False + esc = False + while i2 >= 0: + ch = text[i2] + if in_str: + if esc: esc = False + elif ch == '\\': esc = True + elif ch == '"': in_str = False + i2 -= 1; continue + if in_chr: + if esc: esc = False + elif ch == '\\': esc = True + elif ch == "'": in_chr = False + i2 -= 1; continue + if ch == '"': in_str = True; i2 -= 1; continue + if ch == "'": in_chr = True; i2 -= 1; continue + if ch == ')': depth += 1 + elif ch == '(': + depth -= 1 + if depth == 0: + return i2 + i2 -= 1 + return -1 + + j = dot_idx - 1 + while j >= 0 and s[j].isspace(): + j -= 1 + + if j >= 0 and s[j] == ')': + open_idx = _match_paren_reverse(s, j) + if open_idx == -1: + start = j + while start >= 0 and (s[start].isalnum() or s[start] in '_$.'): + start -= 1 + recv = s[start + 1:dot_idx].strip() + return (recv if recv else None), method + + k = open_idx - 1 + while k >= 0 and s[k].isspace(): + k -= 1 + if k >= 0 and (s[k].isalnum() or s[k] in '_$.'): + start = k + while start >= 0 and (s[start].isalnum() or s[start] in '_$.'): + start -= 1 + recv = s[start + 1:dot_idx].strip() + else: + recv = s[open_idx:dot_idx].strip() + else: + start = j + while start >= 0 and (s[start].isalnum() or s[start] in '_$.'): + start -= 1 + recv = s[start + 1:dot_idx].strip() + + return (recv if recv else None), method + + caller_src = function.metadata.get('source') + caller_doc = code_documents.get(caller_src) + + caller_text = (caller_doc.page_content if caller_doc else "") # safe fallback + caller_fqcn = self.get_class_name_from_class_function(function) + + # -------------------- FQCN-only type matching -------------------- + # Build quick index: simple-name -> list of target FQCNs (for fast disambiguation) + _target_by_simple: dict[str, list[str]] = {} + for fq in target_class_names: + last = fq.rsplit('.', 1)[-1] + _target_by_simple.setdefault(last, []).append(fq) + if '$' in last: + _target_by_simple.setdefault(last.split('$')[-1], []).append(fq) + + # Parse header imports once (bounded), used only for disambiguation (still cheap) + _hdr = caller_text + _PKG_RE = re.compile(r'^\s*package\s+([\w.]+)\s*;\s*$', re.MULTILINE) + _IMP_RE = re.compile(r'^\s*import\s+(?:static\s+)?([\w$.]+)\s*;\s*$', re.MULTILINE) + + m_pkg = _PKG_RE.search(_hdr) + _caller_pkg = (m_pkg.group(1) if m_pkg else "") + _explicit_imports: dict[str, str] = {} # simple -> fqcn (CHANGED) + _wild_import_pkgs: list[str] = [] # "a.b" for "import a.b.*" (CHANGED) + for imp in _IMP_RE.findall(_hdr): + if imp.endswith(".*"): + _wild_import_pkgs.append(imp[:-2]) + else: + _explicit_imports[imp.rsplit('.', 1)[-1]] = imp + + def _is_concrete_or_paired(candidate_fqcn: str) -> bool: + """Guard against false call chains from ancestor interface/abstract casts. + + When a caller file imports a broad interface (e.g. Map, Transformer) + that appears in the callee's type hierarchy (target_class_names), a + naive match would treat the caller as invoking the callee — even + though the caller never references the callee's declaring class. + + This function accepts a candidate only if it is: + - "concrete": the candidate FQCN IS the callee's declaring class, OR + - "paired": the callee's declaring class is explicitly imported in + the same file, meaning the caller is aware of the concrete type, OR + - "same-artifact": caller and callee are in the same JAR/artifact. + Within the same artifact, classes freely call each other through + interfaces without needing explicit imports of concrete types. + """ + cand_dot = candidate_fqcn.replace('$', '.') + decl_dot = callee_declaring_fqcn.replace('$', '.') + if cand_dot == decl_dot: + return True + callee_simple = decl_dot.rsplit('.', 1)[-1] + if callee_simple in _explicit_imports: + return True + if self._is_same_artifact(caller_src or "", callee_function_file_name or ""): + return True + return False + + def _strip_type_syntax(token: str) -> str: + """Strip generics, arrays, and wildcard bounds from a type-ish token (CHANGED).""" + t = (token or "").strip() + if not t: + return "" + # Drop keywords after the type (e.g. "? extends Foo") + for kw in ("extends", "super"): + if kw in t: + t = t.split(kw, 1)[0].strip() + t = t.replace("?", "").strip() + + # Strip generics with nesting + out = [] + depth = 0 + for ch in t: + if ch == '<': + depth += 1 + continue + if ch == '>': + if depth > 0: + depth -= 1 + continue + if depth == 0: + out.append(ch) + t = ''.join(out).strip() + + # Strip trailing arrays + while t.endswith("]") and "[]" in t: + t = re.sub(r'\s*(\[\])\s*$', '', t).strip() + + return t + + def _is_package_qualified(name: str) -> bool: + """Heuristic: package-qualified names typically start with a lower-case segment (CHANGED).""" + first = name.split('.', 1)[0] + return bool(first) and first[0].islower() + + def _iter_fqcn_candidates(raw_type_token: str): + """ + Yield FQCN candidates for a raw type token. Ensures that downstream + `_type_token_matches_callee(...)` is invoked with FQCNs only (CHANGED). + """ + t = _strip_type_syntax(raw_type_token) + if not t: + return + # Already looks like a FQCN: pkg.Type or pkg.Outer.Inner + if '.' in t and _is_package_qualified(t): + yield t + return + + # Nested type without package: Outer.Inner -> prefix caller package if known + if '.' in t and not _is_package_qualified(t): + if _caller_pkg: + yield f"{_caller_pkg}.{t}" + else: + # best-effort; still a "fqcn-ish" candidate + yield t + return + + # Explicit import is the definitive binding for this simple name. + # Yield only if it resolves to a target type; either way, block + # the _target_by_simple fallthrough. + # _is_concrete_or_paired rejects ancestor interfaces (e.g. Transformer) + # that match target_class_names but aren't the callee's declaring class. + imp = _explicit_imports.get(t) + if imp: + if imp in target_class_names: + if not callee_declaring_fqcn or _is_concrete_or_paired(imp): + yield imp + return + + # Target allow-list by simple name (only reached when no explicit import). + # Same _is_concrete_or_paired guard: a simple-name match on a broad + # interface (Map, Collection) must not create a false edge to the callee. + cands = _target_by_simple.get(t) + if cands: + for fq in cands: + if not callee_declaring_fqcn or _is_concrete_or_paired(fq): + yield fq + + # Wildcard imports: only usable if we can cheaply construct candidates + # (we only build candidates that are already in target_class_names to avoid work) + if _wild_import_pkgs and cands: + # already covered by `cands`; do nothing + pass + + # Same-package fallback + if _caller_pkg: + yield f"{_caller_pkg}.{t}" + + def _type_token_matches_callee(type_fqcn: str) -> bool: + """ + (CHANGED) This function now assumes it receives a FQCN token only. + All callers must pass FQCNs (or FQCN candidates). + """ + if not type_fqcn: + return False + + fq = _strip_type_syntax(type_fqcn) # defensive: keep it cheap + if not fq: + return False + + # IMPORTANT (CHANGED): target_class_names now contains FQCNs; membership is exact. + if target_class_names and fq not in target_class_names: + return False + + if _is_fqcn_like(fq): + if self.is_package_imported(caller_text, fq): + return True + + matches = self.__get_type_docs_matched_with_callee_type( + callee_package, fq, type_documents, target_class_names + ) + return len(matches) > 0 + + def _type_matches_callee(raw_type_token: str) -> bool: + """Resolve raw token -> FQCN candidates, then call `_type_token_matches_callee` (CHANGED).""" + seen = set() + for fq in _iter_fqcn_candidates(raw_type_token): + if fq in seen: + continue + seen.add(fq) + if _type_token_matches_callee(fq): # fqcn-only + return True + return False + + def _caller_key() -> str: + content = function.page_content + md5hex = hashlib.md5(content.strip().encode("utf-8")).hexdigest() + return f"{extract_method_name_with_params(content)}@{md5hex}@{caller_src}" + + expr = _strip_return_and_ws(_strip_leading_this(identifier_function)) + expr = re.sub(r'\s+', ' ', expr).strip() + + def _extract_ctor_type(expr: str) -> str: + i = 4 # skip 'new ' + n = len(expr) + while i < n and expr[i].isspace(): + i += 1 + buf = [] + angle = 0 + while i < n: + ch = expr[i] + if ch == '<': + angle += 1 + buf.append(ch); i += 1; continue + if ch == '>': + if angle > 0: + angle -= 1 + buf.append(ch); i += 1; continue + if (ch == '(' or ch == '{') and angle == 0: + break + buf.append(ch); i += 1 + return ''.join(buf).strip() + + # ====== EARLY: direct constructor snippet 'new Type(...' ====== + if expr.startswith('new '): + ctor_type_token = _extract_ctor_type(expr) + return _type_matches_callee(ctor_type_token) # resolve -> FQCN(s) + + recv_norm, method_norm = _extract_recv_and_method(expr) + if method_norm: + expr = (f"{recv_norm}.{method_norm}(" if recv_norm else f"{method_norm}(").strip() + + is_unqualified = (recv_norm is None) + + # ======== SUPER / THIS handling via target_class_names ======== + if recv_norm is not None: + recv_trim = recv_norm.strip() + + if recv_trim.endswith('.super'): + iface_type = recv_trim[:-6].strip() + if _type_matches_callee(iface_type): + return True + + if recv_trim == 'super' and callee_declaring_fqcn: + # Don't match super calls from root-package functions. + # super delegates to the parent's own implementation which is + # self-contained. Allowing it at the app→dependency boundary + # lets the CCA's backtracking route through the entire + # dependency hierarchy via polymorphic dispatch, producing + # false-positive chains (e.g. handler.handle() hops). + if self.is_root_package(function): + pass # skip – super from app code is not a valid entry point + else: + try: + caller_inheritance = type_inheritance[(caller_fqcn, caller_src)] + except KeyError: + caller_inheritance = [] + # super resolves to the direct parent class only; match only if the + # direct parent IS the callee's declaring class (exact match). + for cand_fqcn, _cand_src in caller_inheritance: + if cand_fqcn == caller_fqcn: + continue + if cand_fqcn == callee_declaring_fqcn or cand_fqcn == callee_declaring_fqcn.replace('$', '.'): + return True + break + + if recv_trim == 'this': + if caller_fqcn and _type_token_matches_callee(caller_fqcn): + return True + + if is_unqualified: + base_before_paren = method_norm + if base_before_paren and callee_function_name != base_before_paren: + fn = find_function(caller_src, base_before_paren, documents_of_functions) + if fn: + ret_type = _find_method_return_type_in_file(fn.page_content, base_before_paren) + if ret_type and _type_matches_callee(ret_type): + return True + + try: + callee_doc = code_documents[callee_function_file_name] + callee_pkg_decl = (self.get_package_name_file(callee_doc) or "").strip() + except KeyError: + callee_pkg_decl = callee_function_file_name + + caller_pkg_decl = (self.get_package_name_file(caller_doc) or "").strip() if caller_doc else "" + jar_name = extract_jar_name(caller_src or "") + return ( + callee_package.__contains__(jar_name) + and callee_pkg_decl + and callee_pkg_decl == caller_pkg_decl + and (caller_fqcn in target_class_names if caller_fqcn else False) + ) + + chain = _split_top_level_dots(expr) + if not chain: + return False + + recv_raw = (chain[-2] if len(chain) >= 2 else "").strip() + + cast_type, post_cast_expr = _peel_leading_cast(recv_raw) + if cast_type: + if _type_matches_callee(cast_type): + return True + m_id_cast = re.match(r'\s*([A-Za-z_$][\w$]*)', post_cast_expr) + if m_id_cast: + traced_cast = self.__trace_down_package( + expression=m_id_cast.group(1), + type_documents=type_documents, + callee_package=callee_package, + fields_of_types=fields_of_types, + functions_local_variables_index=functions_local_variables_index, + caller_function_index=_caller_key(), + target_class_names=target_class_names, + function=function, + code_documents=code_documents, + caller_explicit_imports=_explicit_imports, + caller_package=_caller_pkg, + ) + if traced_cast: + return True + + if recv_raw.endswith('.super'): + iface_type = recv_raw[:-6].strip() + if _type_matches_callee(iface_type): + return True + + if recv_raw == 'super' and callee_declaring_fqcn: + # Don't match super calls from root-package functions (see early-path comment). + if self.is_root_package(function): + pass # skip + else: + try: + caller_inheritance = type_inheritance[(caller_fqcn, caller_src)] + except KeyError: + caller_inheritance = [] + # super resolves to the direct parent class only; match only if the + # direct parent IS the callee's declaring class (exact match). + for cand_fqcn, _cand_src in caller_inheritance: + if cand_fqcn == caller_fqcn: + continue + if cand_fqcn == callee_declaring_fqcn or cand_fqcn == callee_declaring_fqcn.replace('$', '.'): + return True + break + elif recv_raw == 'this': + if caller_fqcn and _type_token_matches_callee(caller_fqcn): + return True + + # Case A: static-style class root in the chain (includes 'new Type(...)' receivers) + for seg in chain[:-1]: + seg_core = _strip_call_parens(seg).strip() + if seg_core.startswith("new "): + typ = _strip_call_parens(seg_core[4:].strip()) + if _type_matches_callee(typ): + return True + continue + if _is_fqcn_like(seg_core) and _type_matches_callee(seg_core): + return True + + # Case B: identifier/expression → dataflow OR rightmost receiver-call return type + recv_for_ident = _strip_call_parens(recv_raw).strip() if not cast_type else post_cast_expr.strip() + if recv_for_ident: + caller_key = _caller_key() + + m_id = re.match(r'\s*([A-Za-z_$][\w$]*)', recv_for_ident) + left_ident = m_id.group(1) if m_id else None + if left_ident and left_ident not in ('this', 'super'): + traced = self.__trace_down_package( + expression=left_ident.strip(), + type_documents=type_documents, + callee_package=callee_package, + fields_of_types=fields_of_types, + functions_local_variables_index=functions_local_variables_index, + caller_function_index=caller_key, + target_class_names=target_class_names, + function=function, + code_documents=code_documents, + caller_explicit_imports=_explicit_imports, + caller_package=_caller_pkg, + ) + if traced: + return True + + if recv_for_ident.endswith(')'): + segs = _split_top_level_dots(recv_for_ident) + recv_call_name = None + for seg in reversed(segs): + seg = seg.strip() + if seg.endswith(')'): + mtok = re.match(r'\s*([A-Za-z_$][\w$]*)\s*\(', seg) + if mtok: + recv_call_name = mtok.group(1) + break + if recv_call_name: + ret_type = _find_method_return_type_in_file(caller_text, recv_call_name) + if ret_type and _type_matches_callee(ret_type): + return True + + # Case C: final receiver is a direct 'new Type(...)' + if recv_raw.startswith("new "): + typ = _strip_call_parens(recv_raw[4:].strip()) + if _type_matches_callee(typ): + return True + + # Case D: immediate receiver is a (pkg.)Class token + if _is_fqcn_like(recv_raw) and _type_matches_callee(recv_raw): + return True + + return False + + def __trace_down_package( + self, + expression: str, + type_documents: list[Document], + callee_package: str, + fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict], + caller_function_index: str, + target_class_names: frozenset[str], + function: Document, + code_documents: dict[str, Document], + caller_explicit_imports: dict[str, str] | None = None, + caller_package: str = "", + ) -> bool: + variables_mappings = functions_local_variables_index.get(caller_function_index, {}) # CHANGED: safe fallback + parts = expression.split(".") + result = False + + def _normalize_type_token(t: str) -> str: + """Strip generics + array suffixes; keep dots/$ for FQCNs.""" + if not t: + return "" + t = t.strip() + t = strip_java_generics(t).strip() + t = re.sub(r"\s*(\[\])+$", "", t).strip() + return t + + def _fqcn_candidates_from_token(type_token: str) -> list[str]: + """ + Return FQCN candidates derived from `type_token`, restricted to `target_class_names`. + - If token is already an exact member of target_class_names, return it. + - Else map by simple-name suffix against target_class_names (handles Inner via '$'). + """ + t = _normalize_type_token(type_token) + if not t: + return [] + + # Exact match (already FQCN in allow-list) + if t in target_class_names: + return [t] + + # Derive simple name and match against allow-list by suffix + simple = t.rsplit(".", 1)[-1] + + if caller_explicit_imports: + imported_fqcn = caller_explicit_imports.get(simple) + if imported_fqcn and imported_fqcn not in target_class_names: + return [] + + if (not caller_explicit_imports or simple not in caller_explicit_imports) and caller_package: + same_pkg_fqcn = f"{caller_package}.{simple}" + if same_pkg_fqcn not in target_class_names: + for td in type_documents: + td_src = td.metadata.get('source', '') + if simple in td_src and td_src.endswith(f"/{simple}.java"): + pkg_path = caller_package.replace('.', '/') + if pkg_path in td_src: + return [] + + out: list[str] = [] + dot_suffix = "." + simple + dollar_suffix = "$" + simple + for fq in target_class_names: + # fq is FQCN; match on end + if fq.endswith(dot_suffix) or fq.endswith(dollar_suffix): + out.append(fq) + + return out + + def _has_matching_type(type_token: str) -> bool: + """Call __get_type_docs_matched_with_callee_type with FQCNs only.""" + for fq in _fqcn_candidates_from_token(type_token): + if self.__get_type_docs_matched_with_callee_type( + callee_package, fq, type_documents, target_class_names + ): + return True + return False + + (resolved_type, struct_initializer_expression, + value_list, var_properties) = self.__prepare_package_lookup(parts, variables_mappings, the_part=-1) + + if ( + var_properties is not None + and ( + struct_initializer_expression + or resolved_type not in LOCAL_INDIRECT_TYPES_INDICATIONS + or PARAMETER in value_list + ) + ): + result = self.__lookup_package( + callee_package=callee_package, + resolved_type=resolved_type, + struct_initializer_expression=struct_initializer_expression, + type_documents=type_documents, + value_list=value_list, + target_class_names=target_class_names, + caller_explicit_imports=caller_explicit_imports, + caller_package=caller_package, + ) + + # Property/member is not in function, check if it's member/property of a type + elif var_properties is None and (expression and (expression[0].islower() or expression[0] == '_')): + fqcn = self.get_class_name_from_class_function(function) + possible_types = { + key: value for (key, value) in fields_of_types.items() + if key == (fqcn, function.metadata['source']) + } + for mappings in possible_types.values(): + for mapping in mappings: + if expression in mapping: + # CHANGED: mapping[1] may be simple; resolve to FQCN candidates first + if _has_matching_type(mapping[1]): + result = True + break + if result: + break + + elif var_properties is not None: + value = var_properties.get("value", None) + if value is not None: + inferred = self._infer_type_from_var_initializer( + initializer=value_list[-1], + variables_mappings=variables_mappings, + type_documents=type_documents, + caller_src=function.metadata['source'], + code_documents=code_documents + ) + if inferred: + # CHANGED: inferred may be simple; resolve to FQCN candidates first + if _has_matching_type(inferred): + result = True + + return result + + def document_imports_package(self, documents: dict[str, Document], package_name: str) -> list[Document]: + importing_docs = [value for (file, value) in documents.items() + if self.is_package_imported(value.page_content, package_name) or package_name.rsplit('.', 1)[-1] in value.page_content] + return importing_docs + + def is_package_imported(self, code_content: str, identifier: str, callee_package: str = "") -> bool: + """ + returns Match if `java_src` contains: + - import ; + - import .*; + Static imports are excluded. + `identifier` is a fully qualified class name (may include nested classes, e.g., java.util.Map.Entry). + """ + # Derive the base *package* (up to the first Type segment, usually starting with uppercase) + parts = identifier.split('.') + pkg_end = None + for i, seg in enumerate(parts): + if seg and seg[0].isalpha() and seg[0].upper() == seg[0]: # heuristically treat Uppercase as type segment + pkg_end = i + break + if pkg_end is None: + # Fallback: assume last segment is the type + pkg_end = max(0, len(parts) - 1) + + base_pkg = '.'.join(parts[:pkg_end]) # e.g., "java.util" for "java.util.Map.Entry" + + # Build the regex: match non-static imports only + if base_pkg: + pattern = rf'(?m)^\s*import\s+(?!static\s)(?:{re.escape(identifier)}|{re.escape(base_pkg)}\.\*)\s*;' + else: + # No sensible package (edge case) -> only match the direct import + pattern = rf'(?m)^\s*import\s+(?!static\s){re.escape(identifier)}\s*;' + + return re.search(pattern, code_content, flags=re.MULTILINE) is not None + + def get_package_name_file(self, function: Document): + content = function.page_content + index_of_package = content.find("package") + index_of_end_of_line = index_of_package + content[index_of_package:].find(os.linesep) + if index_of_package > -1: + return content[index_of_package + 8:index_of_end_of_line].strip() + + def get_package_names(self, function: Document) -> list[str]: + result, acc = [], [] + full_doc_path = str(function.metadata['source']) + parts = full_doc_path.split("/") + + if parts[0].startswith(self.dir_name_for_3rd_party_packages()): + for piece in parts[2:-1]: + acc.append(piece) + result.append('.'.join(acc)) + else: + for piece in parts[0:-1]: + acc.append(piece) + result.append('.'.join(acc)) + + return result + + def is_root_package(self, function: Document) -> bool: + return not function.metadata['source'].startswith(self.dir_name_for_3rd_party_packages()) + + def is_comment_line(self, line: str) -> bool: + pass + + def is_doc_type(self, doc: Document) -> bool: + return is_java_type(doc.page_content) + + def create_map_of_local_vars(self, functions_methods_documents: List[Document]) -> Dict[str, Dict]: + """ + Parse each Document (containing exactly one Java method OR one lambda) and return: + "(sig)@@" for methods, or "lambda@@" for lambdas -> { + : {"value": , "type": }, + return_types: [] # empty for void/constructor/lambda + } + + Notes: + - Catch parameters (e.g., `catch (IOException ex)`) are returned as parameters. + - Catch parameters are **not** included in the signature part of the key. + - For locals, "value" is a list of all assignment RHS expressions to that local in the body + (declaration initializers + later reassignments, including augmented ops like +=, <<=, etc.). + """ + + # --- constants for output shape --- + RETURN_TYPES = "return_types" + + CONTROL = { + "if", "else", "while", "do", "for", "switch", "case", "default", "return", + "break", "continue", "throw", "throws", "try", "catch", "finally", "new", + "assert", "yield" + } + + NON_TYPE_STARTS = { + "return", "throw", "break", "continue", "assert", "yield", + "if", "else", "for", "while", "switch", "case", "default", + "try", "catch", "finally", "synchronized", "new", + "this", "super" + } + + # --- hot-path compiled/aliased helpers for perf --- + ANN_RE = re.compile(r'@\w+(?:\s*\([^()]*\))?') + WS_RE = re.compile(r'\s+') + VAR_END_RE = re.compile(r'([A-Za-z_$][\w$]*)\s*((?:\[\s*\])*)\s*$') + + ANN_SUB = ANN_RE.sub + WS_SUB = WS_RE.sub + RE_SEARCH = re.search + + # ------------------------- + # low-level scanners + # ------------------------- + def strip_comments_keep_strings(s: str) -> str: + if "/*" not in s and "//" not in s: + return s + out = [] + i, n = 0, len(s) + sl = ml = False + in_str = in_chr = False + esc = False + while i < n: + ch = s[i] + nx = s[i+1] if i+1 < n else '' + if sl: + if ch == '\n': + sl = False; out.append('\n') + else: + out.append(' ') + i += 1; continue + if ml: + if ch == '*' and nx == '/': + ml = False; out.extend((' ', ' ')); i += 2 + else: + out.append('\n' if ch == '\n' else ' '); i += 1 + continue + if in_str: + out.append(ch) + if esc: esc = False + else: + if ch == '\\': esc = True + elif ch == '"': in_str = False + i += 1; continue + if in_chr: + out.append(ch) + if esc: esc = False + else: + if ch == '\\': esc = True + elif ch == "'": in_chr = False + i += 1; continue + + if ch == '/' and nx == '/': + sl = True; out.extend((' ', ' ')); i += 2; continue + if ch == '/' and nx == '*': + ml = True; out.extend((' ', ' ')); i += 2; continue + if ch == '"': + in_str = True; out.append(ch); i += 1; continue + if ch == "'": + in_chr = True; out.append(ch); i += 1; continue + + out.append(ch); i += 1 + return ''.join(out) + + def prev_non_ws(s: str, i: int) -> int: + while i >= 0 and s[i].isspace(): i -= 1 + return i + + def scan_back_to_boundary(s: str, pos: int) -> int: + i = pos - 1 + while i >= 0 and s[i] not in "{;\n": + i -= 1 + return i + 1 + + def balanced_region(s: str, open_pos: int, open_ch: str, close_ch: str) -> Tuple[Optional[str], Optional[int]]: + if open_pos < 0 or open_pos >= len(s) or s[open_pos] != open_ch: + return None, None + i = open_pos + depth = 0 + in_str = in_chr = False + esc = False + n = len(s) + while i < n: + ch = s[i] + if in_str: + if esc: esc = False + else: + if ch == '\\': esc = True + elif ch == '"': in_str = False + i += 1; continue + if in_chr: + if esc: esc = False + else: + if ch == '\\': esc = True + elif ch == "'": in_chr = False + i += 1; continue + if ch == '"': in_str = True; i += 1; continue + if ch == "'": in_chr = True; i += 1; continue + if ch == open_ch: depth += 1 + elif ch == close_ch: + depth -= 1 + if depth == 0: + return s[open_pos+1:i], i+1 + i += 1 + return None, None + + def balanced_region_reverse(s: str, close_pos: int, close_ch: str, open_ch: str) -> Tuple[Optional[str], Optional[int]]: + if close_pos < 0 or close_pos >= len(s) or s[close_pos] != close_ch: + return None, None + i = close_pos + depth = 0 + in_str = in_chr = False + esc = False + while i >= 0: + ch = s[i] + if in_str: + if esc: esc = False + else: + if ch == '\\': esc = True + elif ch == '"': in_str = False + i -= 1; continue + if in_chr: + if esc: esc = False + else: + if ch == '\\': esc = True + elif ch == "'": in_chr = False + i -= 1; continue + if ch == '"': in_str = True; i -= 1; continue + if ch == "'": in_chr = True; i -= 1; continue + if ch == close_ch: depth += 1 + elif ch == open_ch: + depth -= 1 + if depth == 0: + return s[i+1:close_pos], i + i -= 1 + return None, None + + def split_top_level(s: str, delim: str) -> List[str]: + out = [] + depth_p = depth_b = depth_br = depth_a = 0 + in_str = in_chr = False + start = 0 + for idx, ch in enumerate(s): + if in_str: + if ch == '\\': + continue + elif ch == '"': + in_str = False + continue + if in_chr: + if ch == '\\': + continue + elif ch == "'": + in_chr = False + continue + if ch == '"': in_str = True; continue + if ch == "'": in_chr = True; continue + if ch == '(': depth_p += 1 + elif ch == ')': depth_p -= 1 + elif ch == '[': depth_b += 1 + elif ch == ']': depth_b -= 1 + elif ch == '{': depth_br += 1 + elif ch == '}': depth_br -= 1 + elif ch == '<': depth_a += 1 + elif ch == '>' and depth_a > 0: depth_a -= 1 + if ch == delim and depth_p == 0 and depth_b == 0 and depth_br == 0 and depth_a == 0 and not (in_str or in_chr): + out.append(s[start:idx]); start = idx + 1 + out.append(s[start:]); return out + + def remove_modifiers(s: str) -> str: + tokens = WS_RE.split(s.strip()) + tokens = [t for t in tokens if t and t not in MODIFIERS] + return ' '.join(tokens).strip() + + def canon_ws(s: Optional[str]) -> str: + return WS_SUB(' ', s or '').strip() + + def count_array_pairs(text: str) -> int: + i = 0; n = len(text); cnt = 0 + while i < n: + if text[i] == '[': + j = i + 1 + while j < n and text[j].isspace(): j += 1 + if j < n and text[j] == ']': + cnt += 1 + i = j + 1 + continue + i += 1 + return cnt + + def scan_identifier(s: str, i: int): + n = len(s) + if i >= n or not (s[i] == '_' or s[i] == '$' or s[i].isalpha()): + return None, i + j = i + 1 + while j < n and (s[j] == '_' or s[j] == '$' or s[j].isalnum()): + j += 1 + return s[i:j], j + + def skip_ws(s: str, i: int) -> int: + n = len(s) + while i < n and s[i].isspace(): i += 1 + return i + + def scan_type_prefix(s: str): + i = skip_ws(s, 0) + if i >= len(s): + return None, None + if s[i:i+3] == 'var' and (i+3 == len(s) or not (s[i+3] == '_' or s[i+3].isalnum() or s[i+3] == '$')): + j = skip_ws(s, i+3) + k = j + while True: + kk = skip_ws(s, k) + if kk < len(s) and s[kk] == '[': + kk = skip_ws(s, kk+1) + if kk < len(s) and s[kk] == ']': + k = kk + 1; continue + break + return s[i:k], k + + start = i + name, j = scan_identifier(s, i) + if not name: return None, None + while True: + jj = skip_ws(s, j) + if jj < len(s) and s[jj] == '.': + jj = skip_ws(s, jj+1) + name2, j2 = scan_identifier(s, jj) + if not name2: break + j = j2 + else: + break + + j = skip_ws(s, j) + if j < len(s) and s[j] == '<': + depth = 0; k = j; n = len(s) + while k < n: + ch = s[k] + if ch == '<': depth += 1 + elif ch == '>': + depth -= 1 + if depth == 0: + k += 1; break + elif ch == '"': + _, end = balanced_region(s, k, '"', '"') + k = (end or (k+1)); continue + k += 1 + j = k + + k = j + while True: + kk = skip_ws(s, k) + if kk < len(s) and s[kk] == '[': + kk = skip_ws(s, kk+1) + if kk < len(s) and s[kk] == ']': + k = kk + 1; continue + break + return s[start:k], k + + def count_and_strip_trailing_array_dims(type_part: str) -> tuple[str, int, bool]: + """ + Count only trailing [] pairs at the END of the type, strip them, and + return (type_without_trailing_arrays, trailing_dims_count, space_before_arrays). + + Examples: + "Object[][]" -> ("Object", 2, False) + "List" -> ("List", 0, False) + "Map[]" + -> ("Map", 1, False) + "String []" -> ("String", 1, True) + "String [][][]" -> ("String", 3, True) # any whitespace before the first '[' counts + """ + # Matches trailing sequences like: " []", "[][]", " [] [] ", etc. + # It's ok if your code already defines TRAILING_ARR_RE; otherwise: + # TRAILING_ARR_RE = re.compile(r'(?:\s*\[\s*\]\s*)+$') + m = TRAILING_ARR_RE.search(type_part) + if not m: + return type_part, 0, False + + suffix = m.group(0) + dims = suffix.count('[') + + # If suffix starts with whitespace, then there was a space before the first '[' + space_before = bool(suffix) and suffix[0].isspace() + + # Strip the whole suffix from the type + return type_part[:m.start()].rstrip(), dims, space_before + + def parse_param_list(text: str) -> list: + t = (text or "").strip() + if not t: return [] + parts = split_top_level(t, ',') + res = [] + for raw in parts: + item = ANN_SUB(' ', raw.strip()) + item = remove_modifiers(item) + item = WS_SUB(' ', item).strip() + if not item: continue + + varargs = item.endswith('...') + if varargs: item = item[:-3].rstrip() + + m = VAR_END_RE.search(item) + if not m: + # untyped parameter (e.g., lambda param) + res.append({"name": item, "type": None, "varargs": varargs, "array_dims": 0}) + continue + + name = m.group(1) + # [] after the variable name + arr_after = count_array_pairs(m.group(2) or "") + + # Type part may have generics, and may END with [] (which belong to this param) + type_part = (item[:m.start()].strip()) + + # ✅ Only count/strip [] at the END of the type, not inside generics + type_part, trailing_dims, space_before = count_and_strip_trailing_array_dims(type_part) + + type_clean = WS_SUB(' ', type_part).strip() if type_part else None + res.append({ + "name": name, + "type": type_clean if type_clean else None, + "varargs": varargs, + "array_dims": arr_after + trailing_dims, + # Preserve whether there was whitespace before type-trailing []. + # Only meaningful if trailing_dims > 0 (i.e., [] were part of the type). + "space_before_arrays": bool(space_before and trailing_dims > 0) + }) + return res + + def render_param_type(p: dict) -> str: + """ + Render a parameter type for the method signature. + + Rules: + - If varargs: "T..." (arrays ignored if present). + - If array_dims > 0: preserve whether there was a space before the type-trailing []. + * If the original had "String []": render "String []" + * If it was "String[]": render "String[]" + * [] moved from variable name (e.g., "String a[]") are aggregated into the type; we render with no extra space by default, + unless type-trailing [] originally had whitespace (captured in 'space_before_arrays'). + - If type is missing (e.g., lambda), use "var". + """ + t = canon_ws(p.get("type") or "") + array_dims = int(p.get("array_dims") or 0) + + if p.get("varargs"): + base = t or "var" + return base + "..." + + if array_dims > 0: + base = t or "var" + # Honor preserved whitespace only when the original trailing [] were part of the type + sep = " " if p.get("space_before_arrays") else "" + return base + sep + "[]" * array_dims + + return t + + def render_param_decl(p: dict) -> str: + t = render_param_type(p) + return (f"{t} {p['name']}".strip() if t else p["name"]) + + def find_top_eq(s: str): + depth_p = depth_b = depth_br = depth_a = 0 + in_str = in_chr = False + for i, ch in enumerate(s): + if in_str: + if ch == '\\': continue + elif ch == '"': in_str = False; continue + continue + if in_chr: + if ch == '\\': continue + elif ch == "'": in_chr = False; continue + continue + if ch == '"': in_str = True; continue + if ch == "'": in_chr = True; continue + if ch == '(': depth_p += 1 + elif ch == ')': depth_p -= 1 + elif ch == '[': depth_b += 1 + elif ch == ']': depth_b -= 1 + elif ch == '{': depth_br += 1 + elif ch == '}': depth_br -= 1 + elif ch == '<': depth_a += 1 + elif ch == '>' and depth_a > 0: depth_a -= 1 + elif ch == '=' and depth_p == 0 and depth_b == 0 and depth_br == 0 and depth_a == 0: + return i + return None + + def _has_comp_ops_before_assign(text: str) -> bool: + depth_p = depth_b = depth_br = depth_a = 0 + in_str = in_chr = False + i = 0; n = len(text) + while i < n: + ch = text[i] + if in_str: + if ch == '\\': i += 2; continue + if ch == '"': in_str = False + i += 1; continue + if in_chr: + if ch == '\\': i += 2; continue + if ch == "'": in_chr = False + i += 1; continue + if ch == '"': in_str = True; i += 1; continue + if ch == "'": in_chr = True; i += 1; continue + if ch == '(': depth_p += 1; i += 1; continue + if ch == ')': depth_p -= 1; i += 1; continue + if ch == '[': depth_b += 1; i += 1; continue + if ch == ']': depth_b -= 1; i += 1; continue + if ch == '{': depth_br += 1; i += 1; continue + if ch == '}': depth_br -= 1; i += 1; continue + if ch == '<': depth_a += 1; i += 1; continue + if ch == '>' and depth_a > 0: depth_a -= 1; i += 1; continue + + if depth_p==depth_b==depth_br==depth_a==0: + if i+1 < n and text[i:i+2] in ('==','!=','&&','||','<=','>='): + return True + if ch == '?': + return True + if ch in '=#,': + return False + i += 1 + return False + + def parse_decl_with_inits(stmt: str): + s = stmt.strip().rstrip(';').strip() + if not s: return None + s1 = s + if JAVA_ANNOTATION_SYMBOL in s1: + while True: + m = ANN_RE.match(s1) + if not m: break + s1 = s1[m.end():].lstrip() + s1 = re.sub( + r'^(?:(?:public|protected|private|static|final|abstract|synchronized|native|strictfp|transient|volatile)\s+)+', + '', s1).lstrip() + + if _has_comp_ops_before_assign(s1): + return None + i0 = skip_ws(s1, 0) + tok0, _ = scan_identifier(s1, i0) + if tok0 and tok0 in NON_TYPE_STARTS: + return None + + type_str, end_idx = scan_type_prefix(s1) + if not type_str: + return None + rest = s1[end_idx:].lstrip() + if not rest: + return None + + id0, _ = scan_identifier(rest, 0) + if not id0: + return None + + decls = split_top_level(rest, ',') + out = [] + for dec in decls: + d = dec.strip() + if not d: continue + eq = find_top_eq(d) + left = d[:eq].rstrip() if eq is not None else d + init = d[eq+1:].strip() if eq is not None else "" + m2 = VAR_END_RE.match(left) + if not m2: continue + name = m2.group(1) + dims = count_array_pairs(m2.group(2) or "") + out.append({"name": name, "dims": dims, "init": init}) + if not out: + return None + return (canon_ws(type_str), out) + + def strip_anon_and_local_class_bodies(s: str) -> str: + """ + Remove bodies of *real* class/interface/enum/record declarations and 'new T(...) { ... }' + anonymous bodies, but DO NOT touch '.class' literals (e.g., 'Foo.class'). + Single pass, preserves string/char literals, keeps statement layout with spaces. + """ + if ('class' not in s and 'interface' not in s and 'enum' not in s and 'record' not in s + and ' new ' not in s and 'new ' not in s): + return s + + out = [] + i = 0 + n = len(s) + in_str = in_chr = False + + def skip_balanced_at(open_pos: int) -> int: + content, endp = balanced_region(s, open_pos, '{', '}') + if endp is None: + # unmatched '{' – be conservative + out.extend(' ') + return open_pos + 1 + # blank out the whole balanced region + out.extend(' ' * (endp - open_pos)) + return endp + + def scan_type_from_index(idx: int): + type_str, end_idx = scan_type_prefix(s[idx:]) + if not type_str: + return None, idx + return type_str, idx + end_idx + + def prev_non_ws_idx(k: int) -> int: + k -= 1 + while k >= 0 and s[k].isspace(): + k -= 1 + return k + + while i < n: + ch = s[i] + + # string/char handling + if in_str: + out.append(ch) + if ch == '"': + in_str = False + i += 1 + continue + if in_chr: + out.append(ch) + if ch == "'": + in_chr = False + i += 1 + continue + if ch == '"': + in_str = True + out.append(ch) + i += 1 + continue + if ch == "'": + in_chr = True + out.append(ch) + i += 1 + continue + + # identifiers + if ch.isalpha() or ch == '_': + j = i + 1 + while j < n and (s[j].isalnum() or s[j] == '_'): + j += 1 + word = s[i:j] + + # ---- real type declarations (NOT '.class' literals) ---- + if word in ("class", "interface", "enum", "record"): + # If previous non-space is '.', this is likely a '.class' literal → keep it. + p = prev_non_ws_idx(i) + if p >= 0 and s[p] == '.': + out.append(s[i:j]) # copy the literal 'class' + i = j + continue + + # Otherwise, treat as a declaration keyword and blank until its body '{...}' + k = j + # blank everything up to the next '{' (header/implements/extends) + while k < n and s[k] != '{': + out.append(' ') + k += 1 + if k < n and s[k] == '{': + i = skip_balanced_at(k) # blanks the body + continue + # No '{' found (malformed). Just copy the token and continue. + out.append(s[i:j]) + i = j + continue + + # ---- 'new T(...) { ... }' anonymous body stripping ---- + if word == "new": + k = j + while k < n and s[k].isspace(): + k += 1 + _t, k2 = scan_type_from_index(k) + if k2 != k: + k = k2 + while k < n and s[k].isspace(): + k += 1 + if k < n and s[k] == '(': + _, endp = balanced_region(s, k, '(', ')') + if endp: + k = endp + while k < n and s[k].isspace(): + k += 1 + # If a '{' follows, it's an anonymous body → blank it. + if k < n and s[k] == '{': + i = skip_balanced_at(k) + continue + + # default: just copy the identifier + out.append(s[i:j]) + i = j + continue + + # default: copy char + out.append(ch) + i += 1 + + return ''.join(out) + + # Canonicalize initializer/assignment RHS expressions + def canon_init(expr: str) -> str: + if not expr: + return "" + e = expr.strip() + e = re.sub(r'\s*\.\s*', '.', e) + e = re.sub(r'\s*\(\s*', '(', e) + e = re.sub(r'\s*\)\s*', ')', e) + e = re.sub(r'\s*,\s*', ', ', e) + e = re.sub(r'\s*<\s*', '<', e) + e = re.sub(r'\s*>\s*', '>', e) + e = re.sub(r'\s+\)', ')', e) + e = re.sub(r'\(\s+', '(', e) + e = re.sub(r'\s+', ' ', e) + return e + + # --------- collect catch parameters from body ---------- + def collect_catch_params(body: str) -> List[dict]: + if not body or "catch" not in body: + return [] + s = body + params = [] + i = 0 + n = len(s) + depth_p = depth_b = 0 + in_str = in_chr = False + while i < n: + ch = s[i] + if in_str: + if ch == '"': in_str = False + i += 1; continue + if in_chr: + if ch == "'": in_chr = False + i += 1; continue + if ch == '"': in_str = True; i += 1; continue + if ch == "'": in_chr = True; i += 1; continue + if ch == '(': + depth_p += 1; i += 1; continue + if ch == ')': + depth_p -= 1; i += 1; continue + if ch == '[': + depth_b += 1; i += 1; continue + if ch == ']': + depth_b -= 1; i += 1; continue + + if depth_p == 0 and depth_b == 0 and (ch.isalpha() or ch == '_'): + j = i + 1 + while j < n and (s[j].isalnum() or s[j] == '_'): j += 1 + token = s[i:j] + k = j + while k < n and s[k].isspace(): k += 1 + if token == "catch" and k < n and s[k] == '(': + content, endp = balanced_region(s, k, '(', ')') + if endp is None: + i = k + 1 + continue + c = ANN_SUB(' ', content or "") + c = remove_modifiers(c) + c = WS_SUB(' ', c).strip() + name_match = RE_SEARCH(r'([A-Za-z_$][\w$]*)\s*$', c) + if name_match: + name = name_match.group(1) + type_part = c[:name_match.start()].strip() + params.append({"name": name, "type": type_part or None, "varargs": False, "array_dims": 0, "is_catch": True}) + i = endp + continue + i = j + continue + i += 1 + return params + + # --- assignment parsing helpers --- + def _split_top_level_assignment(stmt: str) -> Optional[Tuple[str, str, str]]: + """ + Return (lhs, op, rhs) for the first top-level assignment in `stmt`, or None. + Supports =, +=, -=, *=, /=, &=, |=, ^=, %=, <<=, >>=, >>>=. + Ignores ==, !=, <=, >=, &&, || and anything inside (), [], {}, <> or strings/chars. + """ + depth_p = depth_b = depth_br = depth_a = 0 + in_str = in_chr = False + i = 0 + n = len(stmt) + + while i < n: + ch = stmt[i] + + if in_str: + if ch == '\\': i += 2; continue + if ch == '"': in_str = False; i += 1; continue + i += 1; continue + + if in_chr: + if ch == '\\': i += 2; continue + if ch == "'": in_chr = False; i += 1; continue + i += 1; continue + + if ch == '"': in_str = True; i += 1; continue + if ch == "'": in_chr = True; i += 1; continue + + if ch == '(': depth_p += 1; i += 1; continue + if ch == ')': depth_p -= 1; i += 1; continue + if ch == '[': depth_b += 1; i += 1; continue + if ch == ']': depth_b -= 1; i += 1; continue + if ch == '{': depth_br += 1; i += 1; continue + if ch == '}': depth_br -= 1; i += 1; continue + if ch == '<': depth_a += 1; i += 1; continue + if ch == '>' and depth_a > 0: depth_a -= 1; i += 1; continue + + if ch == '=' and depth_p == depth_b == depth_br == depth_a == 0: + # detect assignment operator + op_start = i + op = '=' + if i >= 3 and stmt[i-3:i+1] == '>>>=': + op_start = i-3; op = '>>>=' + elif i >= 2 and stmt[i-2:i+1] in ('>>=', '<<='): + op_start = i-2; op = stmt[i-2:i+1] + elif i >= 1 and stmt[i-1] in '+-*/&|^%': + op_start = i-1; op = stmt[i-1] + '=' + else: + prevc = stmt[i-1] if i-1 >= 0 else '' + if prevc in ('=', '!', '<', '>'): + i += 1; continue # not an assignment (==, !=, <=, >=) + lhs = stmt[:op_start].strip() + rhs = stmt[i+1:].strip() + return lhs, op, rhs + i += 1 + return None + + def collect_locals_single_pass(body: str) -> list: + if not body: + return [] + if (';' not in body) and ('for' not in body) and ('catch' not in body) and ('try(' not in body) and ('try (' not in body): + return [] + + scrubbed = strip_anon_and_local_class_bodies(body) + + locals_out = [] + i = 0 + n = len(scrubbed) + depth_p = depth_b = depth_br = 0 + in_str = in_chr = False + stmt_start = 0 + + while i < n: + ch = scrubbed[i] + + if in_str: + if ch == '"': in_str = False + i += 1; continue + if in_chr: + if ch == "'": in_chr = False + i += 1; continue + if ch == '"': in_str = True; i += 1; continue + if ch == "'": in_chr = True; i += 1; continue + + if ch == '(': + depth_p += 1; i += 1; continue + if ch == ')': + depth_p -= 1; i += 1; continue + if ch == '[': + depth_b += 1; i += 1; continue + if ch == ']': + depth_b -= 1; i += 1; continue + + if ch == '{' and depth_p == 0 and depth_b == 0: + depth_br += 1 + stmt_start = i + 1 + i += 1 + continue + if ch == '}' and depth_p == 0 and depth_b == 0: + depth_br = max(0, depth_br - 1) + stmt_start = i + 1 + i += 1 + continue + + if ch == ';' and depth_p == 0 and depth_b == 0: + stmt = scrubbed[stmt_start:i+1].strip() + stmt_start = i + 1 + if stmt: + st0 = stmt.lstrip() + t0, _ = scan_identifier(st0, 0) + if not (t0 and t0 in NON_TYPE_STARTS): + decl = parse_decl_with_inits(stmt) + if decl: + t, dlist = decl + for d in dlist: + locals_out.append({ + "type": t, + "name": d["name"], + "array_dims": d["dims"], + "init": canon_init(d["init"]) + }) + i += 1 + continue + + if depth_p == 0 and depth_b == 0 and (ch.isalpha() or ch == '_'): + j = i + 1 + while j < n and (scrubbed[j].isalnum() or scrubbed[j] == '_'): j += 1 + token = scrubbed[i:j] + k = j + while k < n and scrubbed[k].isspace(): k += 1 + if token == "for" and k < n and scrubbed[k] == '(': + content, endp = balanced_region(scrubbed, k, '(', ')') + if endp is None: + i = k + 1 + continue + c = (content or "").strip() + depth_lp = depth_lb = depth_lbr = depth_la = 0 + colon_idx = None + for idx2, ch2 in enumerate(c): + if ch2 == '(': + depth_lp += 1 + elif ch2 == ')': + depth_lp -= 1 + elif ch2 == '[': + depth_lb += 1 + elif ch2 == ']': + depth_lb -= 1 + elif ch2 == '{': + depth_lbr += 1 + elif ch2 == '}': + depth_lbr -= 1 + elif ch2 == '<': + depth_la += 1 + elif ch2 == '>' and depth_la > 0: + depth_la -= 1 + elif ch2 == ':' and depth_lp == 0 and depth_lb == 0 and depth_lbr == 0 and depth_la == 0: + colon_idx = idx2; break + if colon_idx is not None: + left = c[:colon_idx].strip() + right = c[colon_idx+1:].strip() + left = ANN_SUB(' ', left) + left = remove_modifiers(left) + left = WS_SUB(' ', left).strip() + m = VAR_END_RE.search(left) + if m: + name = m.group(1) + dims = count_array_pairs(m.group(2) or "") + type_part = left[:m.start()].strip() + locals_out.append({ + "type": type_part or None, + "name": name, + "array_dims": dims, + "init": canon_init(right) + }) + else: + segs = split_top_level(c, ';') + if segs: + init_seg = segs[0].strip() + decl = parse_decl_with_inits(init_seg) + if decl: + t, dlist = decl + for d in dlist: + locals_out.append({ + "type": t, + "name": d["name"], + "array_dims": d["dims"], + "init": canon_init(d["init"]) + }) + i = endp + continue + + elif token == "catch" and k < n and scrubbed[k] == '(': + _content, endp = balanced_region(scrubbed, k, '(', ')') + if endp is None: + i = k + 1 + continue + i = endp + continue + + elif token == "try" and k < n and scrubbed[k] == '(': + content, endp = balanced_region(scrubbed, k, '(', ')') + if endp is None: + i = k + 1 + continue + for part in split_top_level((content or "").strip(), ';'): + decl = parse_decl_with_inits(part.strip()) + if decl: + t, dlist = decl + for d in dlist: + locals_out.append({ + "type": t, + "name": d["name"], + "array_dims": d["dims"], + "init": canon_init(d["init"]) + }) + i = endp + continue + + i = j + continue + + i += 1 + + return locals_out + + def collect_all_assignment_values(body: str, declared_locals: List[dict]) -> Dict[str, List[str]]: + """ + For each declared local variable, collect ALL RHS expressions assigned to it within the method body: + - Declaration initializers + - Later simple and augmented assignments (e.g., x = ..., x += ..., x <<= ...) + Returns: { localName: [canon_rhs1, canon_rhs2, ...] } + """ + out: Dict[str, List[str]] = {} + if not body: + return out + + locals_set = {d["name"] for d in declared_locals} + seen: Dict[str, set] = {} + + scrubbed = strip_anon_and_local_class_bodies(body) + + # 1) declaration initializers (always included) + for d in declared_locals: + nm = d["name"] + init = d.get("init", "") + if init: + val = canon_init(init) + out.setdefault(nm, []).append(val) + seen.setdefault(nm, set()).add(val) + + # 2) later reassignments (simple and augmented) + i = 0 + n = len(scrubbed) + depth_p = depth_b = depth_br = 0 + in_str = in_chr = False + stmt_start = 0 + + while i < n: + ch = scrubbed[i] + if in_str: + if ch == '"': in_str = False + i += 1; continue + if in_chr: + if ch == "'": in_chr = False + i += 1; continue + if ch == '"': in_str = True; i += 1; continue + if ch == "'": in_chr = True; i += 1; continue + + if ch == '(': + depth_p += 1; i += 1; continue + if ch == ')': + depth_p -= 1; i += 1; continue + if ch == '[': + depth_b += 1; i += 1; continue + if ch == ']': + depth_b -= 1; i += 1; continue + if ch == '{' and depth_p == 0 and depth_b == 0: + depth_br += 1; stmt_start = i + 1; i += 1; continue + if ch == '}' and depth_p == 0 and depth_b == 0: + depth_br = max(0, depth_br - 1); stmt_start = i + 1; i += 1; continue + + if ch == ';' and depth_p == 0 and depth_b == 0: + stmt = scrubbed[stmt_start:i].strip() + stmt_start = i + 1 + + # skip declarations (we already captured their initializers) + decl = None if _has_comp_ops_before_assign(stmt) else parse_decl_with_inits(stmt + ';') + + if not decl: + parsed = _split_top_level_assignment(stmt) + if parsed: + lhs, _op, rhs = parsed + # Only record locals that are plain identifiers on the LHS + m = RE_SEARCH(r'^[A-Za-z_$][\w$]*$', lhs) + if m: + nm = m.group(0) + if nm in locals_set: + val = canon_init(rhs) + s = seen.setdefault(nm, set()) + if val not in s: + out.setdefault(nm, []).append(val) + s.add(val) + i += 1 + continue + + i += 1 + + return out + + # ---------- helpers for anchoring ---------- + def first_open_brace_outside_strings(s: str) -> int: + in_str = in_chr = False + esc = False + for idx, ch in enumerate(s): + if in_str: + if esc: esc = False + else: + if ch == '\\': esc = True + elif ch == '"': in_str = False + continue + if in_chr: + if esc: esc = False + else: + if ch == '\\': esc = True + elif ch == "'": in_chr = False + continue + if ch == '"': in_str = True; continue + if ch == "'": in_chr = True; continue + if ch == '{': + return idx + return -1 + + def prev_identifier_word(s: str, start: int) -> Optional[str]: + i = start - 1 + while i >= 0 and s[i].isspace(): i -= 1 + if i >= 0 and s[i] == '.': + return '.' + while i >= 0 and s[i] in '.=+-/*%&|^!~?:,([{<': + if s[i] == '.': + return '.' + i -= 1 + if i < 0 or not (s[i].isalnum() or s[i] in '_$'): + return None + j = i + while j >= 0 and (s[j].isalnum() or s[j] in '_$'): + j -= 1 + return s[j+1:i+1] + + # ---------- robust: anchor at the real method body, then walk backward ---------- + def parse_single_method(s: str) -> dict: + brace_idx = first_open_brace_outside_strings(s) + if brace_idx == -1: + return {"name": "", "return_type": None, "parameters": [], "locals": [], "body": "", "header_params": []} + + close_paren = s.rfind(')', 0, brace_idx) + if close_paren == -1: + return {"name": "", "return_type": None, "parameters": [], "locals": [], "body": "", "header_params": []} + + param_text, open_paren = balanced_region_reverse(s, close_paren, ')', '(') + if open_paren is None: + return {"name": "", "return_type": None, "parameters": [], "locals": [], "body": "", "header_params": []} + + name_tok_start = prev_non_ws(s, open_paren - 1) + name_tok = None + if name_tok_start is not None: + j = name_tok_start + while j >= 0 and (s[j].isalnum() or s[j] in '_$'): + j -= 1 + name_tok = s[j+1:name_tok_start+1] if j < name_tok_start else None + name_start = j + 1 + else: + name_start = None + + if not name_tok or name_tok in CONTROL: + return {"name": "", "return_type": None, "parameters": [], "locals": [], "body": "", "header_params": []} + + prev_word = prev_identifier_word(s, name_start) + if prev_word == '.' or (prev_word in NON_TYPE_STARTS or prev_word in CONTROL): + return {"name": "", "return_type": None, "parameters": [], "locals": [], "body": "", "header_params": []} + + header_start = scan_back_to_boundary(s, open_paren) + pre_name_raw = s[header_start:name_start].rstrip() + pre_clean = remove_modifiers(ANN_SUB(' ', pre_name_raw)).strip() + ret_type = pre_clean or None + + header_params = parse_param_list(param_text or "") + + body_text, _ = balanced_region(s, brace_idx, '{', '}') + body_text = body_text or "" + + locals_ = collect_locals_single_pass(body_text) if body_text else [] + + catch_params = collect_catch_params(body_text) + all_params = list(header_params) + (catch_params or []) + + return { + "name": name_tok, + "return_type": ret_type, + "parameters": all_params, # used for map (includes catch) + "header_params": header_params, # used for key (excludes catch) + "locals": locals_, + "body": body_text + } + + def parse_single_lambda(s: str) -> dict: + arrow = s.find("->") + j = arrow - 1 + while j >= 0 and s[j].isspace(): j -= 1 + if j >= 0 and s[j] == ')': + ptext, _start = balanced_region_reverse(s, j, ')', '(') + params_text = ptext or "" + else: + m = RE_SEARCH(r'[A-Za-z_$][\w$]*$', s[:arrow]) + params_text = m.group(0) if m else "" + + i = arrow + 2 + n = len(s) + while i < n and s[i].isspace(): i += 1 + is_block = False + if i < n and s[i] == '{': + body_text, _ = balanced_region(s, i, '{', '}') + body_text = body_text or "" + is_block = True + else: + body_text = "" + + p = (params_text or "").strip() + if p.startswith('(') and p.endswith(')'): + p = p[1:-1] + header_params = [] + if p: + for raw in split_top_level(p, ','): + item = ANN_SUB(' ', raw.strip()) + item = remove_modifiers(item) + item = WS_SUB(' ', item).strip() + if not item: continue + m = VAR_END_RE.search(item) + if not m: + header_params.append({"name": item, "type": None, "varargs": False, "array_dims": 0}) + continue + name = m.group(1) + arr_after = count_array_pairs(m.group(2) or "") + left = item[:m.start()].rstrip() + varargs = left.endswith('...') + if varargs: left = left[:-3].rstrip() + type_str = left if left else None + header_params.append({"name": name, "type": type_str, "varargs": varargs, "array_dims": arr_after}) + + locals_ = collect_locals_single_pass(body_text) if is_block else [] + + catch_params = collect_catch_params(body_text) if is_block else [] + all_params = list(header_params) + (catch_params or []) + + return { + "kind": "lambda", + "return_type": None, + "parameters": all_params, # used for map (includes catch) + "header_params": header_params, # not used in key for lambda anyway + "locals": locals_, + "body": body_text + } + + # ------------------------- + # Main loop over documents + # ------------------------- + mappings: Dict[str, Dict] = {} + docs = functions_methods_documents + + for doc in tqdm(docs, total=len(docs), desc="Parsing methods/lambda documents for signature and local variables..."): + raw = doc.page_content + source = str(doc.metadata.get("source", "")) + s = strip_comments_keep_strings(raw) + + md5 = hashlib.md5(s.strip().encode("utf-8")).hexdigest() + + # Prefer method parsing even if the body contains lambdas. + m_entry = parse_single_method(s) + is_valid_method = ( + m_entry.get("name") not in (None, "") + and (m_entry.get("body") or m_entry.get("header_params")) + ) + + if is_valid_method: + entry = m_entry + # Use ONLY header parameters for the signature part (exclude catch params) + header_params = entry.get("header_params") or [] + params_sig = ", ".join(render_param_decl(p) for p in header_params) + func_key = f"{entry.get('name','')}({params_sig})@{md5}@{source}" + else: + # Fallback to lambda-only documents + entry = parse_single_lambda(s) + func_key = f"lambda@{md5}@{source}" + + all_vars: Dict[str, Dict[str, object]] = {} + + # parameters (method/lambda + catch) + for p in entry.get("parameters", []): + ptype = render_param_type(p) + all_vars[p["name"]] = {"value": PARAMETER, "type": ptype} + + # return type + ret = canon_ws(entry.get("return_type")) + all_vars[RETURN_TYPES] = ([] if (not ret or ret == "void") else [ret]) + + # collect ALL assignments for locals + locals_list = entry.get("locals", []) or [] + assigns = collect_all_assignment_values(entry.get("body", "") or "", locals_list) + + # locals (do not override parameters) + for loc in locals_list: + name = loc["name"] + if name in all_vars and all_vars[name].get("value") == PARAMETER: + continue + dtype = canon_ws(loc.get("type") or "") + if loc.get("array_dims", 0) > 0: + dtype = (dtype or "var") + "[]" * loc["array_dims"] + dtype_out = LOCAL_IMPLICIT if dtype == "var" else dtype + values_list = assigns.get(name, []) + all_vars[name] = {"value": values_list, "type": dtype_out} + + mappings[func_key] = all_vars + + return mappings + + def __prepare_package_lookup(self, parts, variables_mappings, the_part: int): + struct_initializer_expression_list: Dict[str, str] = {} # TODO not sure its needed at all + var_properties = variables_mappings.get(parts[the_part], None) + if var_properties is not None: + resolved_type = var_properties.get("type") + + value_list = var_properties.get("value") + for val in value_list: + struct_initializer_expression = re.search(r"""\bnew\s+(?:<[^>]*>\s*)?(?:[A-Za-z_$][\w$]*\.)*[A-Za-z_$][\w$]* + (?:\s*<[^<>]*(?:<[^<>]*>[^<>]*)*>)?\s* + (?: + (?:\[\s*\])+\s*\{ # array initializer + | + \((?:[^()"']+|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\( + (?:[^()"']+|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*')* + \))*\) # constructor args (approx) + (?:\s*\{)? # optional anon-class brace + )""", val) + + if struct_initializer_expression is not None: + struct_initializer_expression_list[val] = struct_initializer_expression.group(0) + resolved_type = str(resolved_type) + return resolved_type, struct_initializer_expression_list, value_list, var_properties + else: + return None, None, None, None + + @staticmethod + def _normalize_type_token(t: str) -> str: + """ + Normalize a Java type token for matching: + - strip leading/trailing whitespace + - strip generics + - strip trailing array suffixes + Keeps dots/$ for FQCNs. + """ + if not t: + return "" + t = t.strip() + t = strip_java_generics(t).strip() + t = _ARRAY_SUFFIX_RE.sub("", t).strip() + return t + + @staticmethod + def _simple_name_from_type_token(t: str) -> str: + """ + Extract the simple name from either a simple type, FQCN, or inner-class FQCN. + Uses the last separator among '.' and '$'. + """ + if not t: + return "" + t = t.strip() + i_dot = t.rfind(".") + i_dol = t.rfind("$") + i = i_dot if i_dot > i_dol else i_dol + return t[i + 1:] if i != -1 else t + + @staticmethod + @lru_cache(maxsize=256) + def _simple_to_fqcns_index(target_class_names: frozenset[str]) -> dict[str, tuple[str, ...]]: + """ + Build an index: simpleName -> tuple(FQCNs). Cached per frozenset instance/value. + This avoids O(|target_class_names|) scans per resolution. + """ + idx: dict[str, list[str]] = {} + for fq in target_class_names: + simple = JavaLanguageFunctionsParser._simple_name_from_type_token(fq) + if not simple: + continue + idx.setdefault(simple, []).append(fq) + return {k: tuple(v) for k, v in idx.items()} + + def _fqcn_candidates_from_token(self, type_token: str, target_class_names: frozenset[str], + caller_explicit_imports: dict[str, str] | None = None, + caller_package: str = "", + type_documents: list | None = None) -> tuple[str, ...]: + """ + Map a possibly-simple type token to FQCN candidates strictly within `target_class_names`. + - If token already equals an allowed FQCN => (token,) + - Else => all allowed FQCNs that share the same simple name + - If caller_explicit_imports maps the simple name to a FQCN NOT in target_class_names, + the caller's type is definitively different => () + - If the caller's own package contains a class with this simple name and that FQCN + is NOT in target_class_names, the caller's type is the same-package one => () + """ + t = self._normalize_type_token(type_token) + if not t: + return () + if t in target_class_names: + return (t,) + simple = self._simple_name_from_type_token(t) + if not simple: + return () + if caller_explicit_imports: + imported_fqcn = caller_explicit_imports.get(simple) + if imported_fqcn and imported_fqcn not in target_class_names: + return () + if (not caller_explicit_imports or simple not in caller_explicit_imports) and caller_package and type_documents: + same_pkg_fqcn = f"{caller_package}.{simple}" + if same_pkg_fqcn not in target_class_names: + pkg_path = caller_package.replace('.', '/') + suffix = f"/{simple}.java" + for td in type_documents: + td_src = td.metadata.get('source', '') + if td_src.endswith(suffix) and pkg_path in td_src: + return () + return self._simple_to_fqcns_index(target_class_names).get(simple, ()) + + def _has_matching_type_in_package( + self, + callee_package: str, + type_token: str, + type_documents: list[Document], + target_class_names: frozenset[str], + caller_explicit_imports: dict[str, str] | None = None, + caller_package: str = "", + ) -> bool: + """ + Calls __get_type_docs_matched_with_callee_type with FQCN candidates only. + """ + for fq in self._fqcn_candidates_from_token(type_token, target_class_names, + caller_explicit_imports=caller_explicit_imports, + caller_package=caller_package, + type_documents=type_documents): + if self.__get_type_docs_matched_with_callee_type(callee_package, fq, type_documents, target_class_names): + return True + return False + + @staticmethod + def _split_top_level_dots(expr: str) -> list[str]: + parts, buf = [], [] + dp = db = dbr = da = 0 + in_str = in_chr = False + i, n = 0, len(expr) + while i < n: + ch = expr[i] + if in_str: + buf.append(ch) + if ch == '\\' and i + 1 < n: + buf.append(expr[i + 1]); i += 1 + elif ch == '"': + in_str = False + i += 1 + continue + if in_chr: + buf.append(ch) + if ch == '\\' and i + 1 < n: + buf.append(expr[i + 1]); i += 1 + elif ch == "'": + in_chr = False + i += 1 + continue + + if ch == '"': + in_str = True; buf.append(ch); i += 1; continue + if ch == "'": + in_chr = True; buf.append(ch); i += 1; continue + if ch == '(': + dp += 1; buf.append(ch); i += 1; continue + if ch == ')': + dp = max(0, dp - 1); buf.append(ch); i += 1; continue + if ch == '[': + db += 1; buf.append(ch); i += 1; continue + if ch == ']': + db = max(0, db - 1); buf.append(ch); i += 1; continue + if ch == '{': + dbr += 1; buf.append(ch); i += 1; continue + if ch == '}': + dbr = max(0, dbr - 1); buf.append(ch); i += 1; continue + if ch == '<': + da += 1; buf.append(ch); i += 1; continue + if ch == '>': + if da > 0: da -= 1 + buf.append(ch); i += 1; continue + + if ch == '.' and dp == db == dbr == da == 0: + parts.append(''.join(buf).strip()); buf = []; i += 1; continue + + buf.append(ch); i += 1 + + parts.append(''.join(buf).strip()) + return [p for p in parts if p] + + @staticmethod + def _strip_call_parens(s: str) -> str: + s2 = s.rstrip() + if not s2.endswith(')'): + return s + depth = 0 + in_str = in_chr = False + i = len(s2) - 1 + while i >= 0: + ch = s2[i] + if in_str: + if ch == '\\': i -= 2; continue + if ch == '"': in_str = False; i -= 1; continue + i -= 1; continue + if in_chr: + if ch == '\\': i -= 2; continue + if ch == "'": in_chr = False; i -= 1; continue + i -= 1; continue + if ch == '"': in_str = True; i -= 1; continue + if ch == "'": in_chr = True; i -= 1; continue + if ch == ')': depth += 1; i -= 1; continue + if ch == '(': + depth -= 1 + if depth == 0: + return s2[:i].rstrip() + i -= 1; continue + i -= 1 + return s + + @staticmethod + @lru_cache(maxsize=2048) + def _method_sig_re(method_name: str) -> re.Pattern: + return re.compile( + rf""" + (?[A-Za-z_.$?][\w$.<>\[\]?]*?) # return type + \s+{re.escape(method_name)}\s*\( # method name + '(' + """, + re.VERBOSE, + ) + + @staticmethod + @lru_cache(maxsize=2048) + def _type_header_re(simple_type_name: str) -> re.Pattern: + return re.compile(rf"\b(class|interface|enum|record)\b\s+{re.escape(simple_type_name)}\b") + + # ------------------------- + # Updated functions (DEDUPED) + # ------------------------- + + def __lookup_package( + self, + callee_package, + resolved_type, + struct_initializer_expression, + type_documents, + value_list, + target_class_names: frozenset[str], + caller_explicit_imports: dict[str, str] | None = None, + caller_package: str = "", + ) -> bool: + if not struct_initializer_expression and resolved_type not in JAVA_METHOD_PRIM_TYPES: + if resolved_type: + if self._has_matching_type_in_package( + callee_package, resolved_type, type_documents, target_class_names, + caller_explicit_imports=caller_explicit_imports, + caller_package=caller_package, + ): + return True + + if PARAMETER in value_list: + return False + + elif struct_initializer_expression: + struct_type = struct_initializer_expression.group(0) # TODO list of expressions + if self._has_matching_type_in_package( + callee_package, struct_type, type_documents, target_class_names, + caller_explicit_imports=caller_explicit_imports, + caller_package=caller_package, + ): + return True + + return False + + def __get_type_docs_matched_with_callee_type( + self, + callee_package, + checked_type, + type_documents, + target_class_names: frozenset[str] + ) -> list[Document]: + checked = strip_java_generics(checked_type or "").strip() + if not checked or checked not in target_class_names: + return [] + + _, _, callee_pkg_tail = callee_package.partition(":") + callee_pkg_tail_lower = callee_pkg_tail.lower() + + result: list[Document] = [] + for a_type in type_documents: + src = a_type.metadata["source"] + if extract_jar_name(src).lower() != callee_pkg_tail_lower and self.dir_name_for_3rd_party_packages() not in src : + continue + + fqcn = self.get_class_name_from_class_function(a_type) + if fqcn == checked: # FIX: match the requested target, not any target + result.append(a_type) + + return result + + def _find_method_return_type_in_type_docs( + self, + receiver_type: str, # simple or qualified type name + method_name: str, + type_documents: list[Document], + ) -> str | None: + """ + Same logic, but reuses cached regex helpers (no duplicated regex construction). + """ + simple = re.sub(r"\s+", "", receiver_type) + simple = simple.rsplit(".", 1)[-1] + simple = re.sub(r"<[^<>]*>", "", simple) + simple = _ARRAY_SUFFIX_RE.sub("", simple) + + sig = self._method_sig_re(method_name) + header = self._type_header_re(simple) + + for doc in type_documents: + src = doc.page_content + if not src: + continue + if not header.search(src): + continue + + m = sig.search(src) + if not m: + continue + + ret = m.group("ret").strip() + ret = re.sub(r"<[^<>]*>", "", ret) + ret = _ARRAY_SUFFIX_RE.sub("", ret).strip() + return ret or None + + return None + + def _infer_type_from_var_initializer( + self, + initializer: str, + variables_mappings: dict, + type_documents: list[Document], + caller_src: str, + code_documents: dict[str, Document], + ) -> str | None: + """ + Same logic, but uses shared helpers for dot-splitting and call-paren stripping. + """ + @lru_cache(maxsize=1024) + def _file_sig_re(method_name: str) -> re.Pattern: + return re.compile( + rf"""(?[A-Za-z_.$?][\w$.<>\[\]?]*?)\s+{re.escape(method_name)}\s*\( + """, + re.VERBOSE, + ) + + def _find_method_return_type_in_file(source_text: str, method_name: str) -> str | None: + m = _file_sig_re(method_name).search(source_text or "") + if not m: + return None + ret = m.group("ret").strip() + ret = re.sub(r"<[^<>]*>", "", ret) + ret = _ARRAY_SUFFIX_RE.sub("", ret).strip() + return ret or None + + expr = (initializer or "").strip().rstrip(';') + chain = self._split_top_level_dots(expr) + if not chain: + return None + + start = chain[0].strip() + ident_only = re.match(r'^[A-Za-z_$][\w$]*$', start) + if ident_only and (len(chain) == 1 or not start.endswith(')')): + start_entry = variables_mappings.get(start, {}) + start_type = (start_entry.get('type') if isinstance(start_entry, dict) else None) + if not start_type: + return None + cur_type = re.sub(r'<[^<>]*>', '', start_type) + cur_type = _ARRAY_SUFFIX_RE.sub("", cur_type).strip() + else: + head = self._strip_call_parens(start).strip() + + # CASE 2: unqualified method call in same file: foo(a,b) + if start.endswith(')') and '.' not in head: + mname = re.match(r'^([A-Za-z_$][\w$]*)$', head) + if not mname: + return None + method_name = mname.group(1) + caller_doc = code_documents.get(caller_src) + caller_text = caller_doc.page_content if caller_doc else "" + ret = _find_method_return_type_in_file(caller_text, method_name) + if not ret: + return None + cur_type = re.sub(r'<[^<>]*>', '', ret) + cur_type = _ARRAY_SUFFIX_RE.sub("", cur_type).strip() + + # CASE 3: static-style or qualified head: ClassName.staticMethod(...) or pkg.Class.m(...) + elif start.endswith(')') and '.' in head: + left, _, right = head.rpartition('.') + base_type = re.sub(r'<[^<>]*>', '', left) + base_type = _ARRAY_SUFFIX_RE.sub("", base_type).strip() + method_name = right + if not re.match(r'^[A-Za-z_$][\w$]*$', method_name): + return None + ret = self._find_method_return_type_in_type_docs(base_type, method_name, type_documents) + if not ret: + return None + cur_type = re.sub(r'<[^<>]*>', '', ret) + cur_type = _ARRAY_SUFFIX_RE.sub("", cur_type).strip() + + else: + return None + + # Walk remaining segments; method calls change the type + for seg in chain[1:]: + seg_core = self._strip_call_parens(seg).strip() + if not seg.endswith(')'): + continue + mname = re.match(r'^([A-Za-z_$][\w$]*)\s*$', seg_core) + if not mname: + continue + method_name = mname.group(1) + ret = self._find_method_return_type_in_type_docs(cur_type, method_name, type_documents) + if not ret: + return None + cur_type = re.sub(r'<[^<>]*>', '', ret) + cur_type = _ARRAY_SUFFIX_RE.sub("", cur_type).strip() + + return cur_type or None + + def get_package_name(self, function: Document, package_name: str) -> str: + jar_name = extract_jar_name(function.metadata['source']) + if not jar_name: + return '' + # Extract artifactId:version from GAV for exact match instead of substring. + # Substring matching (jar_name in package_name) caused false positives when + # short jar_names like "st" matched unrelated GAVs containing "xstream". + parts = package_name.split(':') + if len(parts) == 3: + artifact_version = f"{parts[1]}:{parts[2]}" + else: + artifact_version = package_name + return package_name if jar_name == artifact_version else '' + + def get_import_search_patterns(self, package_name: str) -> list[re.Pattern]: + escaped = re.escape(package_name) + return [ + re.compile(rf"import\s+(?:static\s+)?({escaped}[\w.]*)\s*;", re.IGNORECASE | re.MULTILINE), + ] \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py b/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py new file mode 100644 index 000000000..e2b376128 --- /dev/null +++ b/src/exploit_iq_commons/utils/functions_parsers/javascript_functions_parser.py @@ -0,0 +1,1095 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import List, Tuple + +from langchain_core.documents import Document + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser + +logger = LoggingFactory.get_agent_logger(__name__) + +# Keywords whose control-flow syntax resembles a function call — if(cond), +# for(...), while(...), new Foo(), typeof x — causing false positives in the +# fallback pattern of get_function_name. Keywords like case, default, export, +# import, in, super are deliberately excluded: they never appear with call-like +# name(...) syntax, and they ARE valid method names (obj.default(), query.in()). +_JS_KEYWORDS = frozenset({ + 'if', 'for', 'while', 'switch', 'catch', 'with', 'do', 'else', + 'return', 'throw', 'new', 'delete', 'typeof', 'void', + 'try', 'finally', 'function', 'class', 'extends', + 'break', 'continue', 'var', 'let', 'const', 'yield', 'await', + 'debugger', 'instanceof', +}) + + +class JavaScriptFunctionsParser(LanguageFunctionsParser): + + def __init__(self): + self._class_hierarchy_cache: dict[str, tuple[str | None, str | None]] = {} + self._class_hierarchy_cache_key: int = -1 + + def get_function_name(self, function: Document) -> str: + """Extract function name from JavaScript/TypeScript code.""" + content = function.page_content + + if not self.is_function(function): + raise ValueError('Only function document is supported') + + if re.match(r'export\s+default\s+(?:async\s+)?function\s*\*?\s*\(', content): + return '' + + if re.match(r'export\s+default\s+(?:async\s+)?\(', content): + return '' + + if re.match(r'export\s+default\s+(?:async\s+)?[\w$]+\s*=>', content): + return '' + + body_start = content.find('{') + header = content[:body_start] if body_start != -1 else content + # For arrow functions with destructured params like ({a, b}) => {body}, + # the first '{' is inside the params, not the body start. + # Extend header to include '=>' only when followed by '{' or end (top-level arrow). + arrow_header = header + if body_start != -1 and not header.rstrip().endswith(')') and not re.match(r'(?:(?:async|static|get|set)\s+)*[\w$]+\s*\(', header): + search_from = body_start + while search_from < len(content): + arrow_pos = content.find('=>', search_from) + if arrow_pos == -1 or arrow_pos > body_start + 500: + break + after_arrow = content[arrow_pos + 2:].lstrip() + if after_arrow.startswith('{') or not after_arrow: + arrow_header = content[:arrow_pos + 2] + break + search_from = arrow_pos + 2 + + match = re.search(r'(?:async\s+)?function\b\s*\*?\s*([\w$]+)\s*\(', header) + if match: + return match.group(1) + + if '=>' in arrow_header: + match = re.search(r'(?:const|let|var)\s+([\w$]+)\s*=', arrow_header) + if match: + return match.group(1) + + match = re.search(r'(?:const|let|var)\s+([\w$]+)\s*=\s*(?:async\s+)?[\w$]+\s*=>', header) + if match: + return match.group(1) + + match = re.search(r'(?:const|let|var)\s+([\w$]+)\s*=\s*(?:async\s+)?function\s*\(', header) + if match: + return match.group(1) + + match = re.search(r'(?:const|let|var)\s+([\w$]+)\s*=\s*[\w$.]+\s*\(', header) + if match: + return match.group(1) + + match = re.search(r'([\w$]+)\s*=\s*(?:async\s+)?function\s*\*?\s*[\w$]*\s*\(', header) + if match: + return match.group(1) + + if '=>' in arrow_header: + match = re.search(r'([\w$]+)\s*=(?!>)', arrow_header) + if match: + return match.group(1) + + match = re.search(r'([\w$]+)\s*=\s*(?:async\s+)?[\w$]+\s*=>', header) + if match: + return match.group(1) + + if '=>' in arrow_header: + match = re.search(r'^([\w$]+)\s*:', arrow_header, re.MULTILINE) + if match: + return match.group(1) + + match = re.search(r'^([\w$]+)\s*:\s*(?:async\s+)?[\w$]+\s*=>', content, re.MULTILINE) + if match: + return match.group(1) + + match = re.search(r'^([\w$]+)\s*:\s*(?:async\s+)?function\s*\(', content, re.MULTILINE) + if match: + return match.group(1) + + match = re.search(r"""^["']([\w$.!-]+)["']\s*[:(\s]""", content, re.MULTILINE) + if match: + return match.group(1) + + match = re.search(r'^\s*\*?\s*(?:static\s+)?(?:async\s+)?(?:get\s+|set\s+)?\[([^\]]+)\]\s*\(', content, re.MULTILINE) + if match: + return match.group(1) + + for match in re.finditer(r'(?:static\s+)?(?:async\s+)?(?:get\s+|set\s+)?([\w$]+)\s*[<(]', content): + if match.group(1) not in _JS_KEYWORDS: + return match.group(1) + + logger.warning(f"Could not extract function name from: {content[:100]}") + return '' + + def get_package_names(self, function: Document) -> list[str]: + source = function.metadata.get('source') + if self.dir_name_for_3rd_party_packages() in source: + parts = source.split(self.dir_name_for_3rd_party_packages()) + if len(parts) > 1: + if len(parts) > 2: + parts = parts[-2:] + pkg_parts = parts[1].split('/') + # Handle scoped packages: @babel/core -> @babel/core + if pkg_parts[1].startswith('@') and len(pkg_parts) > 1: + return [f"{pkg_parts[1]}/{pkg_parts[2]}"] + return [pkg_parts[1]] + return ['root_project'] + + def is_root_package(self, function: Document) -> bool: + source = function.metadata.get('source', '') + return self.dir_name_for_3rd_party_packages() not in source + + + def is_function(self, function: Document) -> bool: + content_type = function.metadata.get('content_type', '') + if content_type != 'functions_classes': + return False + + content = function.page_content.strip() + + if re.match(r'^\s*(?:export\s+(?:default\s+)?)?class\b', content): + return False + + if re.match(r'^\s*(?:export\s+)?(?:declare\s+)?(?:const\s+)?(?:interface|enum)\s+', content): + return False + + return True + + def supported_files_extensions(self) -> list[str]: + return ['.js', '.mjs', '.cjs'] + + def _get_function_calls(self, caller_function: Document, callee_function_name: str, code_documents: dict[str, Document] = None) -> list[str]: + """ + Extract all function calls matching the given callee_function_name from caller_function. + + Detects both: + - Direct function calls: functionName() or obj.functionName() + - Callback references: setTimeout(functionName, ...) or arr.forEach(functionName) + + Also handles aliased imports in JavaScript: + - ES6: import { originalName as alias } from 'module' + - CommonJS: const { originalName: alias } = require('module') + + Args: + caller_function: Document containing the caller function code + callee_function_name: Name of the function to find calls to + code_documents: Optional dict mapping source paths to full file Documents (for alias resolution) + + Returns: + List of function call patterns found (e.g., ['func', 'obj.func']) + """ + if not callee_function_name: + return [] + + content = caller_function.page_content + content = '\n'.join([line if not self.is_comment_line(line) else ''for line in content.splitlines()]) + + # 1. Direct calls: foo(), obj.foo(), obj?.foo(), ns.sub.foo() + direct_call_pattern = rf'((?:[\w.?()]+\.)?(? bool: + + calls = self._get_function_calls(caller_function, callee_function_name, code_documents) + + if not calls: + return False + + caller_source = caller_function.metadata.get('source') + caller_document = code_documents.get(caller_source) + caller_content = caller_document.page_content if caller_document else caller_function.page_content + + for call in calls: + parts = call.split('.') + + # Direct function call (no qualifier) + if len(parts) == 1: + identifier = parts[0].rstrip('(') + + caller_pkg = self.get_package_names(caller_function)[0] + callee_pkg = self.get_package_names(callee_function)[0] + + if caller_pkg == callee_pkg and identifier == callee_function_name: + return True + + # Check if package is imported and identifier matches + if self.is_package_imported(caller_content, identifier, callee_function_package): + return True + + # Qualified call (obj.func or module.func) + else: + if parts[-1].rstrip('(') == callee_function_name: + identifier = parts[-2] + + if identifier == 'this': + callee_class_name = self.get_class_name_from_class_function(callee_function) + caller_class_name = self.get_class_name_from_class_function(caller_function) + if callee_class_name and callee_class_name == caller_class_name: + return True + + if callee_class_name := self.get_class_name_from_class_function(callee_function): + if self.is_function(caller_function): + caller_func_name = self.get_function_name(caller_function) + func_key = f"{caller_func_name}@{caller_source}" + local_vars = functions_local_variables_index.get(func_key, {}) + var_info = local_vars.get(identifier, {}) + var_type = var_info.get('type') + if var_type == callee_class_name or self._is_subclass_of(var_type, callee_class_name, code_documents): + return True + + if self.is_package_imported(caller_content, identifier, callee_function_package): + return True + + return False + + @staticmethod + def _build_class_hierarchy(code_documents: dict[str, Document]) -> dict[str, tuple[str | None, str | None]]: + """ + Build a class hierarchy index from code_documents in a single pass. + + Returns: + dict mapping child_class -> (extends_clause, immediate_parent) + extends_clause is the raw text after 'extends' (for mixin matching), + immediate_parent is the resolved parent class name. + """ + hierarchy: dict[str, tuple[str | None, str | None]] = {} + + es6_pattern = re.compile(r'class\s+([\w$]+)\s+extends\s+([^{]+)') + proto_patterns = [ + re.compile(r'(?:util\.)?inherits\s*\(\s*([\w$]+)\s*,\s*([\w$]+)\s*\)'), + re.compile(r'([\w$]+)\.prototype\s*=\s*Object\.create\s*\(\s*([\w$]+)\.prototype\s*\)'), + re.compile(r'Object\.setPrototypeOf\s*\(\s*([\w$]+)\.prototype\s*,\s*([\w$]+)\.prototype\s*\)'), + ] + + for doc in code_documents.values(): + content = doc.page_content + + for match in es6_pattern.finditer(content): + child = match.group(1) + extends_clause = match.group(2).strip() + if innermost := re.search(r'([\w$]+)\s*\)+\s*$', extends_clause): + parent = innermost.group(1) + elif simple := re.match(r'^([\w$]+)\s*$', extends_clause): + parent = simple.group(1) + else: + parent = None + hierarchy[child] = (extends_clause, parent) + + for pattern in proto_patterns: + for match in pattern.finditer(content): + child = match.group(1) + parent = match.group(2) + if child not in hierarchy: + hierarchy[child] = (None, parent) + + return hierarchy + + def _get_class_hierarchy(self, code_documents: dict[str, Document]) -> dict[str, tuple[str | None, str | None]]: + cache_key = id(code_documents) + if self._class_hierarchy_cache_key != cache_key: + self._class_hierarchy_cache = self._build_class_hierarchy(code_documents) + self._class_hierarchy_cache_key = cache_key + return self._class_hierarchy_cache + + def _is_subclass_of(self, child_class: str, parent_class: str, code_documents: dict[str, Document], + visited: set = None) -> bool: + """ + Check if child_class extends parent_class using pre-built hierarchy index. + Supports: + - Simple: class X extends Y + - Mixin: class X extends Mixin(Y) + - Chained: class X extends Mixin1(Mixin2(Y)) + - Transitive: class A extends B, class B extends C -> A is subclass of C + - Prototype-based: util.inherits(Child, Parent), Child.prototype = Object.create(Parent.prototype) + """ + if not child_class or not parent_class: + return False + + if visited is None: + visited = set() + + if child_class in visited: + return False + visited.add(child_class) + + hierarchy = self._get_class_hierarchy(code_documents) + entry = hierarchy.get(child_class) + if not entry: + return False + + extends_clause, immediate_parent = entry + + if extends_clause and re.search(rf'\b{re.escape(parent_class)}\b', extends_clause): + return True + + if immediate_parent: + if immediate_parent == parent_class: + return True + return self._is_subclass_of(immediate_parent, parent_class, code_documents, visited) + + return False + + def _get_direct_parent(self, child_class: str, code_documents: dict[str, Document]) -> str | None: + """Extract the immediate parent from ES6 class extends syntax using the hierarchy index.""" + hierarchy = self._get_class_hierarchy(code_documents) + entry = hierarchy.get(child_class) + if entry and entry[0] is not None: + return entry[1] + return None + + def _get_prototype_parent(self, child_class: str, code_documents: dict[str, Document]) -> str | None: + """Extract parent from prototype-based inheritance using the hierarchy index.""" + hierarchy = self._get_class_hierarchy(code_documents) + entry = hierarchy.get(child_class) + if entry and entry[0] is None and entry[1] is not None: + return entry[1] + return None + + def _get_parent(self, child_class: str, code_documents: dict[str, Document]) -> str | None: + hierarchy = self._get_class_hierarchy(code_documents) + entry = hierarchy.get(child_class) + if entry: + return entry[1] + return None + + def create_map_of_local_vars(self, functions_methods_documents: list[Document]) -> dict[str, dict]: + mappings = {} + + for func_method in functions_methods_documents: + try: + func_name = self.get_function_name(func_method) + except ValueError: + continue + if not func_name: + continue + func_key = f"{func_name}@{func_method.metadata.get('source', '?')}" + content = func_method.page_content + + all_vars = {} + + # Extract parameters from function signature + param_match = re.search(r'\(([^)]*)\)', content) + if param_match: + all_vars.update(self._parse_declarations(param_match.group(1), is_param=True)) + + all_vars['return_types'] = [] + + # Extract local variables from function body + if param_match: + first_brace = content.find('{', param_match.end()) + else: + first_brace = content.find('{') + + last_brace = content.rfind('}') + if first_brace != -1 and last_brace != -1: + body = content[first_brace + 1:last_brace] + for statement in self._split_into_statements(body): + if not self.is_comment_line(statement): + if match := re.match(r'(const|let|var)\s+(.+)', statement): + all_vars.update(self._parse_declarations(match.group(2), is_param=False)) + + # Add 'this' reference for class/object methods + if class_name := self.get_class_name_from_class_function(func_method): + all_vars['this'] = {"value": f'{class_name}()', "type": class_name} + + mappings[func_key] = all_vars + + return mappings + + @staticmethod + def _split_into_statements(text: str) -> list[str]: + """ + Split text by semicolons and newlines, respecting string boundaries. + + Handles: + - Statements ending with semicolons + - Comments (// and /* */) + - Function/class declarations (including multiline parameter lists) + - Braces { and } + """ + def normalize(line: str) -> str: + """Normalize whitespace: collapse multiple spaces/newlines into single space and strip.""" + return re.sub(r'\s+', ' ', line).strip() + + parts = [] + current = '' + in_string = False + string_char = None + in_block_comment = False + in_line_comment = False + paren_depth = 0 + just_closed_paren = False + just_closed_block_comment = False + + for i, char in enumerate(text): + if not in_string and not in_line_comment: + if char == '/' and i + 1 < len(text) and text[i + 1] == '*': + in_block_comment = True + elif char == '*' and i + 1 < len(text) and text[i + 1] == '/' and in_block_comment: + in_block_comment = False + just_closed_block_comment = True + elif char == '/' and i + 1 < len(text) and text[i + 1] == '/' and not in_block_comment: + in_line_comment = True + + if not in_string and not in_block_comment and not in_line_comment: + if char == '(': + paren_depth += 1 + just_closed_paren = False + elif char == ')': + paren_depth -= 1 + if paren_depth == 0: + just_closed_paren = True + elif char not in (' ', '\t', '\n', '\r', '=', '>'): + # Reset flag if we see a non-whitespace character that's not part of => or { + if char != '{': + just_closed_paren = False + + if (char in ('"', "'", '`') and (i == 0 or text[i-1] != '\\') + and not in_block_comment and not in_line_comment): + if not in_string: + in_string = True + string_char = char + elif char == string_char: + in_string = False + string_char = None + + current += char + + if just_closed_block_comment and char == '/': + normalized = normalize(current) + if normalized.startswith('/*') and normalized.endswith('*/'): + parts.append(normalized) + current = '' + just_closed_block_comment = False + + if char == '\n' and in_line_comment: + in_line_comment = False + parts.append(normalize(current)) + current = '' + + elif char == ';' and not in_string and not in_block_comment and not in_line_comment: + parts.append(normalize(current)) + current = '' + + # Split on opening brace { for function/method body + elif char == '{' and not in_string and not in_block_comment and not in_line_comment: + normalized = normalize(current) + if just_closed_paren or any(normalized.startswith(kw) for kw in ['class ', 'export class ']): + if normalized: + parts.append(normalized) + current = '' + just_closed_paren = False + + elif char == '\n' and not in_string and not in_block_comment: + normalized = normalize(current) + if normalized: + if normalized == '}': + parts.append(normalized) + current = '' + + # Add remaining content + if current: + normalized = normalize(current) + if normalized: + parts.append(normalized) + + return parts + + def _parse_declarations(self, declaration_str: str, is_param: bool = False, is_multiline: bool = False) -> dict[str, dict]: + """ + Parse JavaScript variable/parameter declarations into a dict. + + Handles: rest params (...x), destructuring ({a,b}, [x,y]), defaults (a=1), multi-declarations (x=1,y=2) + """ + result = {} + + # Split by comma respecting nesting + for item in self._split_by_comma(declaration_str): + item = item.strip() + if not item: + continue + + # Rest parameter: ...rest + if item.startswith('...'): + result[item[3:].strip()] = {"value": "parameter", "type": "Array"} + + # Object/Array destructuring: {a, b} or [x, y] + elif item.startswith(('{', '[')): + closing_char = '}' if item[0] == '{' else ']' + if (end := item.find(closing_char)) != -1: + value = self._extract_value_after_pattern(item, closing_char) + for elem in item[1:end].split(','): + # Handle {oldName: newName} and {prop = default} + name = elem.split(':')[-1].split('=')[0].strip() + if name: + result[name] = {"value": value if not is_param else "parameter", "type": ""} + + # Regular declaration with or without value + else: + name, _, value = item.partition('=') + name = name.strip().rstrip(';') + value = value.strip().rstrip(';') if value else ("parameter" if is_param else "") + + # Extract type from 'new ClassName(...)' pattern + var_type = "" + if new_match := re.match(r'new\s+(\w+)\s*\(', value): + var_type = new_match.group(1) + + result[name] = {"value": value, "type": var_type} + + return result + + def _split_by_comma(self, text: str) -> list[str]: + """Split by comma, respecting nested (), [], {}.""" + items, current, depth = [], [], 0 + for char in text: + if char in '([{': + depth += 1 + elif char in ')]}': + depth -= 1 + if char == ',' and depth == 0: + items.append(''.join(current)) + current = [] + continue + current.append(char) + if current: + items.append(''.join(current)) + return items + + @staticmethod + def _extract_value_after_pattern(text: str, closing_char: str) -> str: + """Extract value after destructuring pattern, e.g., '} = obj' -> 'obj'.""" + if match := re.search(rf'{re.escape(closing_char)}\s*=\s*(.+?)(?:;|$)', text): + return match.group(1).strip() + return "" + + def parse_all_type_struct_class_to_fields(self, types: list[Document], + type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]] = {}) -> dict[tuple, list[tuple]]: + """ + Parse JavaScript class documents and extract field declarations. + + Extracts: + - Class field declarations: class MyClass { myField = value; } + - Constructor property assignments: constructor() { this.field = value; } + + Returns: + Dict mapping (class_name, source_path) -> list of (field_name, field_type) tuples + """ + types_mapping = {} + for type_doc in types: + class_name = self._extract_class_name(type_doc.page_content) + if not class_name: + continue + fields_list = self._extract_class_fields(type_doc.page_content) + source_path = type_doc.metadata.get('source') + types_mapping[(class_name, source_path)] = fields_list + return types_mapping + + @staticmethod + def _extract_class_name(class_code: str) -> str: + """Extract class name from class definition.""" + match = re.search(r'class\s+(\w+)', class_code) + if match: + return match.group(1) + return '' + + def _extract_class_fields(self, class_code: str) -> list[tuple]: + """ + Extract field declarations from a JavaScript class. + + Returns: + List of (field_name, field_type) tuples. field_type is 'any' for JavaScript. + """ + fields = set() + field_pattern = r'^\s*#?(\w+)\s*[;=]' + this_pattern = r'this\.(\w+)\s*=' + in_constructor = False + in_method = False + brace_depth = 0 + class_body_started = False + + for line in class_code.split('\n'): + line = line.strip() + if self.is_comment_line(line): + continue + if not class_body_started and '{' in line: + class_body_started = True + continue + + if class_body_started: + if re.match(r'^\s*constructor\s*\(', line): + in_constructor = True + in_method = True + brace_depth = 0 + elif re.match(r'^\s*(?:async\s+)?(?:static\s+)?(?:get\s+)?(?:set\s+)?(\w+)\s*\(', line) and not in_method: + in_method = True + in_constructor = False + brace_depth = 0 + + if in_method: + brace_depth += line.count('{') - line.count('}') + if in_constructor: + match = re.search(this_pattern, line) + if match: + field_name = match.group(1) + fields.add((field_name, 'any')) + + if brace_depth == 0 and ('{' in line or '}' in line): + in_method = False + in_constructor = False + + elif not in_method: + match = re.search(field_pattern, line) + if match: + field_name = match.group(1) + if field_name not in ['constructor', 'function', 'class', 'return', 'const', 'let', 'var', 'if', 'for', 'while']: + fields.add((field_name, 'any')) + + return list(fields) + + @staticmethod + def get_comment_line_notation() -> str: + """Return JavaScript comment notation.""" + return '//' + + def get_function_reserved_word(self) -> str: + """Return JavaScript function keyword.""" + return 'function' + + def get_dummy_function(self, function_name): + """Create a dummy JavaScript function for testing.""" + return f"{self.get_function_reserved_word()} {function_name}() {{}}" + + def get_type_reserved_word(self) -> str: + """Return JavaScript class keyword.""" + return "class" + + def is_searchable_file_name(self, function: Document) -> bool: + """Check if file should be searched (exclude test files).""" + file_path = function.metadata['source'] + # Exclude test files (.test.js, .spec.js, __tests__ directory, etc.) + return re.search(r"(\.test\.|\.spec\.|__tests__|/test/|/tests/)", file_path) is None + + def dir_name_for_3rd_party_packages(self) -> str: + return 'node_modules' + + @classmethod + def is_comment_line(cls, line: str) -> bool: + stripped = line.strip() + return stripped.startswith('//') or stripped.startswith('/*') + + def is_doc_type(self, doc: Document) -> bool: + if doc.metadata.get('content_type') != 'functions_classes': + return False + content = doc.page_content.strip() + return re.match(r'^\s*(?:export\s+(?:default\s+)?)?class\s+\w+', content) is not None + + @staticmethod + def is_script_language(): + return True + + @staticmethod + def get_constructor_method_name() -> str: + return 'constructor' + + def get_class_name_from_class_function(self, func: Document) -> str: + pattern = r'//\(class:\s*([\w_]+)\)$' + match = re.search(pattern, func.page_content) + class_name = match.group(1) if match else None + return class_name + + @staticmethod + def _is_position_inside_string_literal(line: str, position: int) -> bool: + """ + Check if a character position in a line is inside a string literal. + + Uses a simple heuristic: if there's an odd number of quotes before the position, + it's likely inside a string literal. + + Args: + line: The line of code to check + position: The character position to check + + Returns: + True if the position appears to be inside a string literal, False otherwise + """ + before_position = line[:position] + + single_quotes = before_position.count("'") - before_position.count("\\'") + double_quotes = before_position.count('"') - before_position.count('\\"') + backticks = before_position.count('`') + + return single_quotes % 2 != 0 or double_quotes % 2 != 0 or backticks % 2 != 0 + + def _trace_variable_to_value(self, variable_name: str, lines: list[str], depth: int = 0, visited: set = None) -> str: + + # Prevent infinite recursion and circular references + MAX_DEPTH = 10 + if depth > MAX_DEPTH: + return '' + + if visited is None: + visited = set() + + if variable_name in visited: + return '' + + visited.add(variable_name) + string_pattern = rf'(?:const|let|var)\s+{re.escape(variable_name)}\s*=\s*([\'"`])([^\1]*?)\1' + var_pattern = rf'(?:const|let|var)\s+{re.escape(variable_name)}\s*=\s*(\w+)' + + for line in lines: + if not self.is_comment_line(line): + if match := re.search(string_pattern, line): + return match.group(2) + + if match := re.search(var_pattern, line): + source_var = match.group(1) + return self._trace_variable_to_value(source_var, lines, depth + 1, visited) + return '' + + def is_package_imported(self, code_content: str, identifier: str, callee_package: str = "") -> bool: + if not identifier: + return False + + if callee_package and callee_package not in code_content: + return False + + lines = self._split_into_statements(code_content) + + for line in lines: + if self.is_comment_line(line): + continue + + # Check for ES6 imports + if 'import' in line and 'from' in line: + # import ... from 'package' + if callee_package and (f"'{callee_package}'" in line or f'"{callee_package}"' in line): + if identifier: + if identifier == callee_package: + if f'import {identifier}' in line: + return True + else: + # import { template } from 'lodash' + # import * as lodash from 'lodash' + before_from = line.split(' from ')[0] + if re.search(rf'\b{re.escape(identifier)}\b', before_from): + return True + else: + return True + + # Check for CommonJS require + if 'require(' in line: + if callee_package and (f"'{callee_package}'" in line or f'"{callee_package}"' in line): + if identifier: + # const template = require('lodash').template + # const { template } = require('lodash') + # const lodash = require('lodash') + before_require = line.split('require')[0] + if re.search(rf'\b{re.escape(identifier)}\b', before_require): + return True + else: + return True + + # Check for re-exports: export { func } from 'package' + if 'export' in line and 'from' in line: + if callee_package and (f"'{callee_package}'" in line or f'"{callee_package}"' in line): + if identifier: + before_from = line.split(' from ')[0] + if re.search(rf'\b{re.escape(identifier)}\b', before_from): + return True + else: + return True + + if dynamic_import_match := re.search(r'\bimport\s*\(', line): + import_pos = dynamic_import_match.start() + + if self._is_position_inside_string_literal(line, import_pos): + continue + + if string_literal_match := re.search(r'import\s*\(\s*([\'"`])([^\1]*?)\1\s*\)', line): + imported_module = string_literal_match.group(2) + + if callee_package and callee_package in imported_module: + if identifier: + assignment_part = line[:import_pos] + if identifier in assignment_part: + return True + # Pattern: .then(({ identifier }) or .then(({ identifier, ... + after_import = line[string_literal_match.end():] + then_match = re.search(r'\.then\s*\(\s*\(\s*\{[^}]*\b' + re.escape(identifier) + r'\b[^}]*\}', after_import) + if then_match: + return True + else: + return True + + elif var_match := re.search(r'import\s*\(\s*(\w+)\s*\)', line): + var_name = var_match.group(1) + + # TODO: Future enhancement - support object property access for traceable constants + # Currently skips ALL dots, but could support cases like: + # const MODULES = { UTILS: './utils.js' }; + # import(MODULES.UTILS); <- This is statically determinable + # Would need to: + # 1. Update regex to allow dots: r'import\s*\(\s*([\w.]+)\s*\)' + # 2. Create _trace_property_to_value() to parse object literals + # 3. Distinguish local objects (traceable) from runtime objects like process.env (not traceable) + + if traced_value := self._trace_variable_to_value(var_name, lines): + if callee_package and callee_package in traced_value: + if identifier: + assignment_part = line[:import_pos] + if identifier in assignment_part: + return True + else: + return True + return False + + def is_same_package(self, package_name_from_input: str, package_name_from_tree: str) -> bool: + return package_name_from_input.lower() == package_name_from_tree.lower() + + def is_exported_function(self, function: Document, documents_of_full_sources: dict[str, Document]) -> bool: + + """ + Check if a function is explicitly exportable + + This method checks the actual export status of a function by looking for: + - ES6 export syntax: export function name() or export default + - ES6 named exports: export { functionName } + - CommonJS module.exports or exports assignments + - For class methods: check if the class itself is exported + - Re-exports: Functions exported from package index.js files + """ + func_content = function.page_content.strip() + source_file = function.metadata.get('source') + + if source_file not in documents_of_full_sources: + return False + + full_file_content = documents_of_full_sources[source_file].page_content + + # Check for ES6 export syntax in the function content itself + # Pattern: export function name() or export default function + if re.match(r'^\s*export\s+(?:default\s+)?(?:async\s+)?function\s+', func_content): + return True + + # Pattern: export default name (where name is an arrow or regular function) + if re.match(r'^\s*export\s+default\s+', func_content): + return True + + # Check if this is a class/object method + class_name = self.get_class_name_from_class_function(function) + + if class_name: + # Check both class export patterns and object/variable export patterns + if self._is_exportable_class(class_name, full_file_content): + return True + if self._is_exportable_function(class_name, full_file_content): + return True + return False + + # This is a standalone function - check if the function itself is exported + function_name = self.get_function_name(function) + if function_name and source_file: + # Check if exported from the same file + if self._is_exportable_function(function_name, full_file_content): + return True + + # Check if function is re-exported from package index.js + # This handles the pattern where functions are defined in subdirectories + # but exported through the package's main entry point + if self.dir_name_for_3rd_party_packages() in source_file: + return self._check_package_reexport(function_name, source_file, documents_of_full_sources) + + return False + + def _check_package_reexport(self, function_name: str, source_file: str, documents_of_full_sources: dict[str, Document]) -> bool: + """ + Check if a function is re-exported from the package's index.js or main entry point. + + For example: + - Function defined in: node_modules/my-pkg/lib/utils.js + - Exported from: node_modules/my-pkg/index.js + """ + parts = source_file.split(self.dir_name_for_3rd_party_packages()) + if len(parts) < 2: + return False + + after_node_modules = parts[1] + pkg_parts = after_node_modules.split('/') + + # Handle scoped packages: @babel/core + if pkg_parts[1].startswith('@') and len(pkg_parts) > 1: + pkg_path = f"{self.dir_name_for_3rd_party_packages()}/{pkg_parts[1]}/{pkg_parts[2]}" + else: + pkg_path = f"{self.dir_name_for_3rd_party_packages()}/{pkg_parts[1]}" + + entry_points = [ + f"{pkg_path}/index.js", + f"{pkg_path}/index.mjs", + f"{pkg_path}/index.cjs" + ] + + for entry_point in entry_points: + entry_doc = documents_of_full_sources.get(entry_point) + if entry_doc: + if self._is_exportable_function(function_name, entry_doc.page_content): + return True + + return False + + @staticmethod + def _is_exported_by_name(name: str, full_file_content: str) -> bool: + """Check if a name is exported via ES6 named export, export default, or CommonJS.""" + escaped = re.escape(name) + + # ES6: export default name + if re.search(rf'export\s+default\s+{escaped}\b', full_file_content): + return True + + # ES6: export { name } or export { x as name } + if re.search(rf'export\s+\{{[^}}]*\b{escaped}\b[^}}]*\}}', full_file_content): + return True + + # CommonJS: module.exports = name + if re.search(rf'module\.exports\s*=\s*{escaped}\b', full_file_content): + return True + + # CommonJS: module.exports = { name } or module.exports = { name: name } + if re.search(rf'module\.exports\s*=\s*\{{[^}}]*\b{escaped}\b[^}}]*\}}', full_file_content): + return True + + # CommonJS: module.exports.name = ... + if re.search(rf'module\.exports\.{escaped}\s*=', full_file_content): + return True + + # CommonJS: exports.name = ... + if re.search(rf'exports\.{escaped}\s*=', full_file_content): + return True + + return False + + @staticmethod + def _is_exportable_class(class_name: str, full_file_content: str) -> bool: + """ + Check if a class is exported using any export syntax. + + Handles: + - ES6: export class ClassName + - ES6: export default ClassName or export default class ClassName + - ES6: export { ClassName } + - CommonJS: module.exports = ClassName + - CommonJS: module.exports = { ClassName } + - CommonJS: exports.ClassName = ClassName + """ + # ES6: export class ClassName or export default class ClassName + if re.search(rf'export\s+(?:default\s+)?class\s+{re.escape(class_name)}\b', full_file_content): + return True + + return JavaScriptFunctionsParser._is_exported_by_name(class_name, full_file_content) + + @staticmethod + def _is_exportable_function(function_name: str, full_file_content: str) -> bool: + """ + Check if a standalone function is exported using any export syntax. + + Handles: + - ES6: export function functionName + - ES6: export default functionName + - ES6: export { functionName } + - CommonJS: module.exports = functionName + - CommonJS: module.exports = { functionName } + - CommonJS: module.exports.functionName = functionName + - CommonJS: exports.functionName = functionName + """ + # ES6: export function functionName / export async function functionName + if re.search(rf'export\s+(?:async\s+)?function\b\s*\*?\s*{re.escape(function_name)}\b', full_file_content): + return True + + # ES6: export const/let/var functionName = + if re.search(rf'export\s+(?:const|let|var)\s+{re.escape(function_name)}\b', full_file_content): + return True + + return JavaScriptFunctionsParser._is_exported_by_name(function_name, full_file_content) + + + def document_imports_package(self, documents: dict[str, Document], package_name: str) -> list[Document]: + """Find documents that import a specific package.""" + importing_docs = [] + for doc in documents.values(): + content = doc.page_content + imports_syntaxes = [f'import {package_name}', + f"from '{package_name}'", + f'from "{package_name}"', + f"require('{package_name}')", + f'require("{package_name}")'] + + if any([import_syntax in content for import_syntax in imports_syntaxes]): + importing_docs.append(doc) + return importing_docs + + def get_import_search_patterns(self, package_name: str) -> list[re.Pattern]: + escaped = re.escape(package_name) + return [ + re.compile(rf"""require\s*\(\s*['"]({escaped}[^'"]*)['"]\s*\)""", re.IGNORECASE | re.MULTILINE), + re.compile(rf"""import\s+.*?\s+from\s+['"]({escaped}[^'"]*)['"]\s*""", re.IGNORECASE | re.MULTILINE), + re.compile(rf"""import\s+['"]({escaped}[^'"]*)['"]\s*""", re.IGNORECASE | re.MULTILINE), + ] + + def is_a_package(self, package_name: str, doc: Document) -> bool: + return package_name in self.get_package_names(doc) + diff --git a/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py new file mode 100644 index 000000000..65cc2c6d1 --- /dev/null +++ b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers.py @@ -0,0 +1,212 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from typing import Tuple, List +from warnings import warn +import re +from langchain_core.documents import Document + + +class LanguageFunctionsParser(ABC): + + # This method returns for the given ecosystem , a map of all functions/methods in all packages to a dict containing + # mappings of all local variables defined in these functions/methods, to their type/value/expression. + # Implementation should be consistent with search_for_called_function, as functions_methods_documents is being used + # in this method as functions_local_variables_index + @abstractmethod + def create_map_of_local_vars(self, functions_methods_documents: list[Document]) -> dict[str, dict]: + pass + + # This method returns (for the given ecosystem) , a mapping of all types/classes in all packages to a dict + # containing mapping of all fields names in the type/class to their defining types. + # a list of + @abstractmethod + def parse_all_type_struct_class_to_fields(self, types: list[Document], type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]]) -> dict[tuple, list[tuple]]: + pass + + @abstractmethod + def get_function_name(self, function: Document) -> str: + pass + + # This method get a caller function document, a callee function, and a callee package, and returns True + # if caller function + @abstractmethod + def search_for_called_function(self, caller_function: Document, callee_function_name: str, callee_function: Document, + callee_function_package: str, code_documents: list[Document], type_documents: list[Document], + callee_function_file_name: str, fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict], + documents_of_functions: list[Document], + type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]]) -> bool: + pass + + # This method gets a function document, and return a list with the containing package name. if + # more than one potential package name is possible, then it should return it also + @abstractmethod + def get_package_names(self, function: Document) -> list[str]: + pass + + # This method gets a function document as an argument, a package name as an argument , and return the containing + # package name, when you have multiple options. + def get_package_name(self, function: Document, package_name: str) -> str: + package_names = self.get_package_names(function) + for package in package_names: + if package_name.lower() == package.lower(): + return package.lower() + return '' + # This method get a function document, and returns true, only if it's a function in the root package ( application) + @abstractmethod + def is_root_package(self, function: Document) -> bool: + pass + + # This method gets a line from a source code, and returns True if it's a comment line for the ecosystem, + # otherwise it returns False. + @classmethod + def is_comment_line(cls, line: str) -> bool: + return line.strip().startswith(cls.get_comment_line_notation()) + + # This method returns the ecosystem' comment character(s) + @staticmethod + @abstractmethod + def get_comment_line_notation() -> str: + pass + + # This method gets a function document as an argument, and returns True if this contained function is exported to be + # Used in other files/packages ( returns True only if it can be called from outside the source file) + @abstractmethod + def is_exported_function(self, function: Document, documents_of_full_sources: dict[str, Document]) -> bool: + pass + + # This method get a source document, and returns True only if the document type + # is a function ( not type/class/full_source) for the ecosystem. + def is_function(self, function: Document) -> bool: + pass + + # This method returns the name of the directory containing the 3rd party packages/libs source code files for the + # ecosystem + @abstractmethod + def dir_name_for_3rd_party_packages(self) -> str: + pass + + # This method returns the list of the supported source file extensions for the ecosystem. + @abstractmethod + def supported_files_extensions(self) -> list[str]: + pass + + # This method gets an extension as an argument, and returns True only if it's supported in the ecosystem. + def is_supported_file_extensions(self, extension: str) -> bool: + return extension in self.supported_files_extensions() + + # This method gets a document function as an argument, and returns True only if this function should be taken into + # Account as candidate to be included in chains of calls paths ( for example , tests and dev resources files should + # be excluded from the search process as they're not indication for exploitability as these are not executed on + # runtime at target environment. + @abstractmethod + def is_searchable_file_name(self, function: Document) -> bool: + pass + + # This method returns the function/method reserved word in code for the ecosystem/programming language. + @abstractmethod + def get_function_reserved_word(self) -> str: + warn("This function is deprecated. Use is_function() instead.", DeprecationWarning, stacklevel=2) + return "" + + # This function returns the type reserved word in code for the ecosystem/programming language. + @abstractmethod + def get_type_reserved_word(self) -> str: + warn("This function is deprecated. Use is_type() instead.", DeprecationWarning, stacklevel=2) + return "" + + @abstractmethod + def get_dummy_function(self, function_name) -> str: + pass + + @staticmethod + def is_script_language(): + return False + + def is_same_package(self, package_name_from_input, package_name_from_tree): + return package_name_from_input.lower() == package_name_from_tree.lower() + + def is_tree_key_match(self, package_from_doc: str, tree_key: str) -> bool: + """Match a document's package name against a dependency tree key. + Used by _resolve_tree_key for tree traversal. Default: delegates to + is_same_package. Go overrides with boundary-aware '/' matching.""" + return self.is_same_package(package_from_doc, tree_key) + + @staticmethod + def get_constructor_method_name(): + return None + + # This method gets a function document as an argument, and returns the name of the class + # that contains this function. Returns None if the function is not a class method or if + # the class name cannot be determined for the given ecosystem. + def get_class_name_from_class_function(self, func: Document): + return None + + def is_package_imported(self, code_content: str, identifier: str, callee_package: str = "") -> bool: + pass + + # This method gets a list of documents and a package name as arguments, and returns a list of + # documents that contain import statements for the specified package. It searches for both + # single import statements and grouped import statements that include the package name. + def document_imports_package(self, documents: dict[str, Document], package_name: str) -> list[Document]: + importing_docs = [value for (file, value) in documents.items() + if re.search( + rf"(import {package_name}|import\s*\(\s*[\w\s\/.\"-]*{package_name}[\w\s\/.\"-]*\s*\))" + , value.page_content, flags=re.MULTILINE)] + return importing_docs + + def is_doc_type(self, doc: Document) -> bool: + return doc.page_content.startswith(self.get_type_reserved_word()) + + def filter_docs_by_func_pkg_name(self, function_name: str, package_name: str, documents: list[Document]) -> list[Document]: + if not function_name: + return [] + relevant_docs = [ + doc for doc in documents + if doc.metadata.get('source').__contains__(package_name) and + doc.page_content.__contains__(function_name) + ] + return relevant_docs + + def is_a_package(self, package_name: str, doc: Document) -> bool: + b_3party = doc.metadata.get('source').startswith(self.dir_name_for_3rd_party_packages()) + b_package = (not self.is_root_package(doc) and (self.get_package_name(function=doc, package_name=package_name) != "")) + return b_3party and b_package + + # This method determines if the search algorithm uses Depth-First Search (DFS) for traversing + # function call graphs and dependency trees. Returns True if DFS is used, False for other algorithms. + def is_search_algo_dfs(self) -> bool: + return True + + # This method checks if the programming language supports macro definitions/preprocessor + # capabilities that can affect function parsing and analysis. Returns True for languages + # with macros (like C/C++), False otherwise. + def is_language_supports_macro(self) -> bool: + return False + + # This method determines if a dummy function should be created for standard library packages + # that may not have complete source code available for analysis. Returns True if a dummy + # function should be created for the given package, False if complete source is available. + def create_dummy_for_standard_lib(self, package_name: str) -> bool: + return False + + # is this call allowed by the language? + def is_call_allowed(self,pkg_docs: list[Document], caller_function: Document, callee_function: Document) -> bool: + return True + + def get_import_search_patterns(self, package_name: str) -> list[re.Pattern]: + return [re.compile(re.escape(package_name), re.IGNORECASE)] \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers_factory.py b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers_factory.py new file mode 100644 index 000000000..e4f9e693c --- /dev/null +++ b/src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers_factory.py @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from exploit_iq_commons.utils.dep_tree import Ecosystem, DependencyTree +from exploit_iq_commons.utils.functions_parsers.golang_functions_parsers import GoLanguageFunctionsParser +from exploit_iq_commons.utils.functions_parsers.python_functions_parser import PythonLanguageFunctionsParser +from exploit_iq_commons.utils.functions_parsers.java_functions_parsers import JavaLanguageFunctionsParser +from exploit_iq_commons.utils.functions_parsers.javascript_functions_parser import JavaScriptFunctionsParser +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser +from exploit_iq_commons.utils.functions_parsers.c_lang_function_parsers import CLanguageFunctionsParser + +def get_language_function_parser(ecosystem: Ecosystem, tree: DependencyTree | None) -> LanguageFunctionsParser: + """ + Get an ecosystem(e.g programming language) parameter, and returns the right language functions parser associated + with the ecosystem. + :param ecosystem: the desired programming language parser. + :param tree: the dependency tree for the ecosystem, can be None. + :return: + The right language functions parser associated to the ecosystem, if not exists, return ABC parent that + doesn't do anything + """ + if ecosystem == Ecosystem.GO: + return GoLanguageFunctionsParser() + elif ecosystem == Ecosystem.PYTHON: + parser = PythonLanguageFunctionsParser() + if tree is not None and tree.builder is not None: + parser.set_dependency_builder(tree.builder) + return parser + elif ecosystem == Ecosystem.JAVASCRIPT: + return JavaScriptFunctionsParser() + elif ecosystem == Ecosystem.C_CPP: + parser_obj = CLanguageFunctionsParser() + if tree is not None: + parser_obj.init_CParser(tree) + return parser_obj + elif ecosystem == Ecosystem.JAVA: + return JavaLanguageFunctionsParser() + else: + return LanguageFunctionsParser() diff --git a/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py b/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py new file mode 100644 index 000000000..e2599191c --- /dev/null +++ b/src/exploit_iq_commons/utils/functions_parsers/python_functions_parser.py @@ -0,0 +1,474 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ast +import os.path +import re +from typing import Tuple, List + +from langchain_core.documents import Document + +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser + +PARAMETER = "parameter" +RETURN_TYPES = "return_types" + +PRIMITIVE_TYPES = [int, + float, + bool, + str, + complex, + type(None), + bytes, + frozenset, + tuple, + list, + dict, + set, + range, + memoryview] + + +class PythonLanguageFunctionsParser(LanguageFunctionsParser): + + def get_dummy_function(self, function_name): + return f"{self.get_function_reserved_word()} {function_name}(): pass" + + def set_dependency_builder(self, builder): + """Store a reference to the PythonDependencyTreeBuilder so is_same_package + can consult the PyPI-to-import mapping built during build_tree.""" + self._builder = builder + + @property + def _import_to_pypi(self) -> dict[str, str]: + if hasattr(self, '_builder') and self._builder and hasattr(self._builder, '_import_to_pypi'): + return self._builder._import_to_pypi + return {} + + @staticmethod + def _pep503(name: str) -> str: + return re.sub(r'[-_.]', '-', name.lower()) + + def is_same_package(self, package_name_from_input, package_name_from_tree): + if self._pep503(package_name_from_input) == self._pep503(package_name_from_tree): + return True + + mapping = self._import_to_pypi + if mapping: + pypi_name = mapping.get(package_name_from_tree.lower()) + if pypi_name and self._pep503(package_name_from_input) == self._pep503(pypi_name): + return True + + return False + + def create_map_of_local_vars(self, functions_methods_documents: list[Document]) -> dict[str, dict]: + mappings = dict() + for func_method in functions_methods_documents: + func_key = f"{self.get_function_name(func_method)}@{func_method.metadata['source']}" + all_vars = dict() + for row in func_method.page_content.splitlines(): + if not self.is_comment_line(row): + # Extract arguments and receiver argument of type as parameters + if row.startswith("def"): + match = re.finditer(r"(def|\w+)\s*\([a-zA-Z0-9\s*,.\[\]]+\)" + , func_method.page_content[:func_method.page_content.find(":")] + , flags=re.MULTILINE) + for current_match in match: + current_args = (current_match.group(0).replace("\n\t", "") + .replace("\t", "").replace("\n", "")) + current_params = re.search(r"\(.*\)", current_args) + params = (current_params.group(0).replace("(", "") + .replace(")", "").split(",")) + params_tuple = tuple(params) + param_type: str = "" + for param in reversed(params_tuple): + data = param.strip().split(":") + param_name = data[0] + if ":" in param: + param_type = data[1].split('=')[0] + the_value = PARAMETER + all_vars[param_name] = {"value": the_value, "type": param_type} + # Gets return types from function starting_function_colon - 1].strip() + try: + return_parameters = func_method.page_content.split('->')[1].split(':')[0].strip() + except IndexError: + return_parameters = '' + return_parameters = return_parameters.replace(")", "").replace("(", "") + + if return_parameters.strip() != "": + all_vars[RETURN_TYPES] = return_parameters.split(",") + else: + all_vars[RETURN_TYPES] = [] + + elif re.search(r"^[A-Za-z0-9]*", row.strip()) and re.search(r'(? str: + return "class" + + def get_function_reserved_word(self) -> str: + return "def" + + def is_searchable_file_name(self, function: Document) -> bool: + file_path = function.metadata['source'] + file_name = os.path.basename(file_path) + return re.search(r"^test_|.*_test.py", file_name) is None + + def supported_files_extensions(self) -> list[str]: + return [".py"] + + def dir_name_for_3rd_party_packages(self) -> str: + return "site-packages" + + def is_function(self, function: Document) -> bool: + return function.page_content.startswith("def") + + def is_exported_function(self, function: Document, documents_of_full_sources: dict[str, Document]) -> bool: + return True + + @staticmethod + def get_comment_line_notation() -> str: + return "#" + + def is_root_package(self, function: Document) -> bool: + return not function.metadata['source'].__contains__(self.dir_name_for_3rd_party_packages()) + + def get_package_names(self, function: Document) -> list[str]: + function_source = function.metadata["source"] + function_names = [] + if self.dir_name_for_3rd_party_packages() in function_source: + function_names.append(function_source.split(self.dir_name_for_3rd_party_packages())[1].split('/')[1]) + else: + function_names.append(function_source.split('/')[0]) + return function_names + + def search_for_called_function(self, caller_function: Document, callee_function_name: str, callee_function: Document, + callee_function_package: str, code_documents: dict[str, Document], + type_documents: list[Document], callee_function_file_name: str, + fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict], + documents_of_functions: list[Document]=None, + type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]]=None) -> bool: + + calls = self._get_function_calls(caller_function, callee_function_name, code_documents) + if not calls: + return False + for call in calls: + + parts = call.split(".") + # If there is no qualifier identifier , then check whether callee function and caller + # function are in the same package + caller_document = code_documents[caller_function.metadata.get('source')] + if len(parts) == 1: + identifier = parts[0] + identifier = identifier.rstrip('(') + try: + callee_function_doc = code_documents[callee_function_file_name] + callee_function_package = self.get_package_names(callee_function_doc)[0] + except KeyError: + callee_function_package = callee_function_file_name + + caller_function_package = self.get_package_names(caller_function)[0] + if callee_function_package == caller_function_package: + return True + return (self.is_package_imported(caller_document.page_content, identifier, callee_function_package) or + self.is_package_imported(caller_function.page_content, identifier, caller_function_package)) + # There is a qualifier identifier. + else: + identifier = parts[-2] + if identifier.startswith("return"): + identifier = identifier.replace("return", "").strip() + if "(" in identifier: + identifier = identifier[identifier.index("(") + 1:] + + # verify that identifier resolves to the package name. if identifier is imported in same file, and if so , + # if it's the same as callee package name + if self.is_package_imported(caller_document.page_content, identifier, callee_function_package): + return True + ## otherwise, the identifier is in the caller function body or in signature/receiver function argument. + + else: + callee_class_name = self.get_class_name_from_class_function(callee_function) + if identifier in ['self', 'cls']: + caller_class_name = self.get_class_name_from_class_function(caller_function) + if callee_class_name == caller_class_name: + return True + # todo: test this: + function_header = self.get_function_header_from_document(caller_function) + caller_function_body = self.get_function_body_from_document(caller_function) + escaped_identifier = re.escape(identifier) + match_regex = re.finditer(rf"^(\s*|\n){escaped_identifier}\s*(=)\s*[^=]+\n*$", + caller_function_body, flags=re.MULTILINE) + matches = [match.group(0) for match in match_regex] + + regex_arguments = r"\([a-zA-Z0-9\s*,.]+\)" + if len(matches) > 0 or re.search(regex_arguments, function_header): + caller_function_file = caller_function.metadata.get('source') + caller_function_name = self.get_function_name(caller_function) + caller_function_index = f"{caller_function_name}@{caller_function_file}" + return self._trace_down_package(expression=identifier, code_documents=code_documents, + type_documents=type_documents, + callee_package=callee_function_package, + callee_class=callee_class_name, + fields_of_types=fields_of_types, + functions_local_variables_index=functions_local_variables_index, + caller_function_index=caller_function_index) + return False + + @staticmethod + def __get_type_docs_matched_with_callee_package(callee_package, checked_type, type_documents) -> list[ + Document]: + return [a_type for a_type in type_documents if callee_package in a_type.metadata['source'] and + f'class {checked_type}' in a_type.page_content] + + def __lookup_package(self, callee_package, resolved_type, struct_initializer_expression, type_documents, + value) -> bool: + result = False + if not struct_initializer_expression and resolved_type not in PRIMITIVE_TYPES: + docs = self.__get_type_docs_matched_with_callee_package(callee_package, resolved_type, type_documents) + + if len(docs) > 0: + result = True + elif value == PARAMETER: + result = False + + elif struct_initializer_expression: + struct_type = (struct_initializer_expression.group(0).replace("*", "")) + docs = self.__get_type_docs_matched_with_callee_package(callee_package, struct_type, type_documents) + if len(docs) > 0: + result = True + return result + + def _trace_down_package(self, expression: str, code_documents: dict[str, Document], type_documents: list[Document], + callee_package: str, callee_class: str, fields_of_types: dict[tuple, list[tuple]], + functions_local_variables_index: dict[str, dict], + caller_function_index: str) -> bool: + variables_mappings = functions_local_variables_index[caller_function_index] + parts = expression.split(".") + result = False + + resolved_type, value, var_properties = self.__prepare_package_lookup(parts[-1], variables_mappings) + + if var_properties and (resolved_type and resolved_type not in PRIMITIVE_TYPES): + if resolved_type.strip() == callee_class.strip(): + return True + + elif var_properties and value: + return self._trace_down_package(value, code_documents, type_documents, callee_package, + callee_class, + fields_of_types, functions_local_variables_index, + caller_function_index) + + # Property/member is not in function, check if it's member/property of a type + elif not var_properties and len(parts) > 1: + identifier_index = -1 + identifier_list = parts[:identifier_index] + property_list = parts[identifier_index:] + + resolved_type, value, var_properties = self.__prepare_package_lookup( + identifier_list[-1], variables_mappings) + if (var_properties and resolved_type and resolved_type not in PRIMITIVE_TYPES): + while property_list: + field_name = property_list[0] + possible_types = {key: value for key, value in fields_of_types.items() + if resolved_type == key + and any([field for field in value if field_name in field])} + for the_type, mappings in possible_types.items(): + result = False + for mapping in mappings: + if field_name in mapping: + matched_types = self.__get_type_docs_matched_with_callee_package(callee_package, + mapping[1], + type_documents) + if matched_types: + result = True + else: + result = False + break + resolved_type = property_list[0] + property_list = property_list[1:] + if result: + return True + identifier_index = identifier_index - 1 + try: + expression[identifier_index] + except IndexError: + return False + identifier_list = expression[:identifier_index] + property_list = expression[identifier_index:] + + return result + + def __prepare_package_lookup(self, expression, variables_mappings): + var_properties = variables_mappings.get(expression, None) + if var_properties: + resolved_type = var_properties.get("type") + value = var_properties.get("value") + return resolved_type, value, var_properties + else: + return None, None, None + + def _get_function_calls(self, caller_function: Document, callee_function: str, code_documents: list[Document] = None): + caller_function_body = self.get_function_body_from_document(caller_function) + regex = fr'\b([a-zA-Z0-9_\[\]\(\).]+\.)?{re.escape(callee_function)}\(' + calls = [matching.group(0) for matching in re.finditer(regex, caller_function_body, re.MULTILINE)] + if code_documents: + code_document = code_documents[caller_function.metadata['source']] + for line in code_document.page_content.split(os.linesep): + if not self.is_comment_line(line): + if all(word in line for word in ['import', callee_function, 'as']): + splitted_identifier = line.split(f'{callee_function} as') + identifier = splitted_identifier[-1].split()[0].rstrip(',') + calls.extend(self._get_function_calls(caller_function, identifier)) + return list(set(calls)) + + def get_function_body_from_document(self, function: Document): + if function.metadata.get('content_type') == 'simplified_code': + return function.page_content + try: + function_tree = ast.parse(function.page_content) + except Exception as e: + # logger.error(e) + raise e + lines = function.page_content.splitlines(keepends=True) + body_start_line = function_tree.body[0].lineno + body_end_line = function_tree.body[-1].end_lineno + body_lines = lines[body_start_line:body_end_line] + body_lines = [line for line in body_lines if not self.is_comment_line(line)] + body_indentation = len(body_lines[0]) - len(body_lines[0].lstrip()) + indent_body_lines = [line[body_indentation:] for line in body_lines] + function_body = ''.join(indent_body_lines) + return function_body + + def get_function_header_from_document(self, function: Document) -> str: + function_tree = ast.parse(function.page_content) + lines = function.page_content.splitlines(keepends=True) + for node in function_tree.body: + if isinstance(node, ast.FunctionDef): + header_start_line = node.lineno - 1 + body_start_line = node.lineno + if node.body: + body_start_line = node.body[0].lineno - 1 + header_lines = lines[header_start_line: body_start_line] + full_header_content = "".join(header_lines) + header_indentation = len(full_header_content) - len(full_header_content.lstrip()) + indent_header_lines = [line[header_indentation:] for line in full_header_content] + return "".join(indent_header_lines).strip() + + def get_function_name(self, function: Document) -> str: + if function.page_content[:3] == self.get_function_reserved_word(): + function_name = function.page_content[4:].split('(')[0].strip() + return function_name + raise ValueError('Only function document is supported') + + @staticmethod + def __get_variable_data(variable_declaration:str) -> tuple[str, str, str]: + """ + Parse a variable declaration string and extract its components. + Args: + variable_declaration (str): A variable declaration string in one of these formats: + Returns: + tuple[str, str, str]: A tuple containing: + - variable_name (str): The name of the variable + - variable_type (str): The type annotation if present, empty string otherwise + - variable_value (str): The assigned value + """ + variable_name = variable_declaration.split()[0].strip() + variable_value = variable_declaration.split('=')[-1] + variable_type = '' + if ':' in variable_declaration: + variable_type = variable_declaration.split(":")[-1].split("=")[0].strip() + + return variable_name, variable_type, variable_value + + def parse_all_type_struct_class_to_fields(self, types: list[Document], type_inheritance: dict[Tuple[str, str], List[Tuple[str, str]]]=None) -> dict[tuple, list[tuple]]: + types_mapping = dict() + for the_type in types: + type_key = the_type.page_content.split('class')[1].split('(')[0].strip() + fields_list = [] + for row in the_type.page_content.split(os.linesep): + if re.search(r"^[A-Za-z0-9]*", row.strip()) and '=' in row: + variable_name, variable_type, variable_value = self.__get_variable_data(row) + fields_list.append((variable_name, variable_type)) + types_mapping[(type_key, the_type.metadata['source'])] = fields_list + return types_mapping + + @staticmethod + def is_script_language(): + return True + + @staticmethod + def get_constructor_method_name(): + return '__init__' + + def is_package_imported(self, code_content: str, identifier: str, callee_package: str = "") -> bool: + for line in code_content.split(os.linesep): + if not self.is_comment_line(line): + if 'import' in line and callee_package in line: + if identifier: + if identifier == callee_package: + if all(word in line for word in ['import', callee_package]): + return True + else: + if all(word in line for word in ['import', callee_package, 'as', identifier]) \ + or all(word in line for word in ['from', callee_package, 'import', identifier]): + return True + else: + if all(word in line for word in ['from', callee_package, 'import']): + return True + return False + + def get_import_search_patterns(self, package_name: str) -> list[re.Pattern]: + escaped = re.escape(package_name) + return [ + re.compile(rf"import\s+({escaped}[\w.]*)", re.IGNORECASE | re.MULTILINE), + re.compile(rf"from\s+({escaped}[\w.]*)\s+import\s+", re.IGNORECASE | re.MULTILINE), + ] + + def get_package_name(self, function: Document, package_name: str) -> str: + package_names = self.get_package_names(function) + for package in package_names: + if self.is_same_package(package_name, package): + return package.lower() + return '' + + def filter_docs_by_func_pkg_name(self, function_name: str, package_name: str, documents: list[Document]) -> list[Document]: + source_form = re.sub(r'[-.]', '_', package_name) + return [ + doc for doc in documents + if source_form in doc.metadata.get('source', '') and + function_name in doc.page_content + ] + + def is_a_package(self, package_name: str, doc: Document) -> bool: + return (not self.is_root_package(doc) and + self.get_package_name(function=doc, package_name=package_name)) \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/functions_parsers/tests/__init__.py b/src/exploit_iq_commons/utils/functions_parsers/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/exploit_iq_commons/utils/functions_parsers/tests/test_go_is_tree_key_match.py b/src/exploit_iq_commons/utils/functions_parsers/tests/test_go_is_tree_key_match.py new file mode 100644 index 000000000..fb32672e6 --- /dev/null +++ b/src/exploit_iq_commons/utils/functions_parsers/tests/test_go_is_tree_key_match.py @@ -0,0 +1,245 @@ +import pytest +from unittest.mock import MagicMock, patch +from langchain_core.documents import Document + +from exploit_iq_commons.utils.functions_parsers.golang_functions_parsers import GoLanguageFunctionsParser +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser +from exploit_iq_commons.utils.functions_parsers.python_functions_parser import PythonLanguageFunctionsParser + + +class TestGoIsTreeKeyMatch: + """Tests for GoLanguageFunctionsParser.is_tree_key_match — boundary-aware + '/' matching that prevents fan-out explosion in _resolve_tree_key.""" + + def setup_method(self): + self.parser = GoLanguageFunctionsParser() + + # --- Bug scenario --- + + def test_bug_scenario_no_false_cross_org_match(self): + """The original bug: 'github.com/hashicorp' substring-matched + 'github.com/hashicorp-terraform/foo' because 'hashicorp' is a + substring of 'hashicorp-terraform'. Boundary matching rejects this.""" + assert self.parser.is_tree_key_match( + "github.com/hashicorp", "github.com/hashicorp-terraform/foo" + ) is False + + def test_bug_scenario_hyphenated_suffix_no_match(self): + """'github.com/foo' must NOT match 'github.com/foo-bar'.""" + assert self.parser.is_tree_key_match( + "github.com/foo", "github.com/foo-bar" + ) is False + + def test_bug_scenario_partial_segment_no_match(self): + """'github.com/go' must NOT match 'github.com/golang/protobuf'.""" + assert self.parser.is_tree_key_match( + "github.com/go", "github.com/golang/protobuf" + ) is False + + # --- Exact match --- + + def test_exact_match(self): + assert self.parser.is_tree_key_match( + "github.com/golang-jwt/jwt/v5", "github.com/golang-jwt/jwt/v5" + ) is True + + def test_exact_match_case_insensitive(self): + assert self.parser.is_tree_key_match( + "GitHub.Com/GoLang-JWT/JWT/v5", "github.com/golang-jwt/jwt/v5" + ) is True + + # --- Input is prefix of tree key (doc package is parent of tree module) --- + + def test_input_prefix_of_tree_key(self): + """2-level path from get_package_names should match child module.""" + assert self.parser.is_tree_key_match( + "github.com/hashicorp", "github.com/hashicorp/vault" + ) is True + + def test_input_prefix_deeper_nesting(self): + assert self.parser.is_tree_key_match( + "github.com/hashicorp", "github.com/hashicorp/vault/api/sub" + ) is True + + def test_input_prefix_three_level(self): + assert self.parser.is_tree_key_match( + "github.com/hashicorp/vault", "github.com/hashicorp/vault/api" + ) is True + + # --- Tree key is prefix of input (rejected: would conflate sub-packages) --- + + def test_tree_key_prefix_of_input_rejected(self): + """Mapping a specific sub-package query to a broader tree entry + would cause false reachability chains (e.g. strvals.Parse matching + ignore.Parse because both live under helm.sh/helm/v3).""" + assert self.parser.is_tree_key_match( + "github.com/hashicorp/vault/api", "github.com/hashicorp/vault" + ) is False + + def test_tree_key_prefix_of_input_deep_rejected(self): + assert self.parser.is_tree_key_match( + "google.golang.org/protobuf/encoding/protojson", + "google.golang.org/protobuf" + ) is False + + # --- No match cases --- + + def test_completely_different_packages(self): + assert self.parser.is_tree_key_match( + "github.com/foo/bar", "github.com/baz/qux" + ) is False + + def test_same_domain_different_org(self): + assert self.parser.is_tree_key_match( + "github.com/foo/bar", "github.com/baz/bar" + ) is False + + def test_domain_only_not_enough(self): + """Sharing only 'github.com' should not match.""" + assert self.parser.is_tree_key_match( + "github.com/orgA", "github.com/orgB" + ) is False + + def test_substring_within_segment_no_match(self): + """'net' is substring of 'netty' but not at a '/' boundary.""" + assert self.parser.is_tree_key_match( + "io.netty/net", "io.netty/netty-codec" + ) is False + + # --- Versioned modules --- + + def test_versioned_module_exact(self): + assert self.parser.is_tree_key_match( + "github.com/golang-jwt/jwt/v5", "github.com/golang-jwt/jwt/v5" + ) is True + + def test_versioned_child(self): + assert self.parser.is_tree_key_match( + "github.com/golang-jwt/jwt/v5", + "github.com/golang-jwt/jwt/v5/parser" + ) is True + + def test_different_versions_no_match(self): + """v4 and v5 are different path segments — no prefix relationship.""" + assert self.parser.is_tree_key_match( + "github.com/golang-jwt/jwt/v5", "github.com/golang-jwt/jwt/v4" + ) is False + + # --- Edge cases --- + + def test_empty_strings(self): + assert self.parser.is_tree_key_match("", "") is True + + def test_empty_input(self): + assert self.parser.is_tree_key_match( + "", "github.com/foo/bar" + ) is False + + def test_empty_tree_key(self): + assert self.parser.is_tree_key_match( + "github.com/foo/bar", "" + ) is False + + def test_single_segment(self): + assert self.parser.is_tree_key_match("fmt", "fmt") is True + + def test_single_segment_prefix_no_match(self): + """'fmt' is not a prefix of 'fmtlib' at a '/' boundary.""" + assert self.parser.is_tree_key_match("fmt", "fmtlib") is False + + def test_trailing_slash_not_treated_as_boundary(self): + """Trailing slash is part of the path, not a boundary marker.""" + assert self.parser.is_tree_key_match( + "github.com/foo/", "github.com/foo/bar" + ) is False + + # --- is_same_package still does substring (unchanged) --- + + def test_is_same_package_still_substring(self): + """Verify is_same_package retains substring behavior for FL/line 545.""" + assert self.parser.is_same_package("jwt", "github.com/golang-jwt/jwt/v5") is True + assert self.parser.is_same_package( + "github.com/hashicorp", "github.com/hashicorp-terraform/foo" + ) is True + + +class TestBaseParserIsTreeKeyMatchDefault: + """Verify base class is_tree_key_match delegates to is_same_package.""" + + def test_base_class_delegates_via_java(self): + """Base class is abstract; use Java parser which inherits default + is_tree_key_match (delegates to exact-match is_same_package).""" + from exploit_iq_commons.utils.functions_parsers.java_functions_parsers import JavaLanguageFunctionsParser + parser = JavaLanguageFunctionsParser() + assert parser.is_tree_key_match("foo", "foo") is True + assert parser.is_tree_key_match("foo", "bar") is False + + def test_python_inherits_base_behavior(self): + """Python parser doesn't override is_tree_key_match — should use + is_same_package (PEP 503 normalization).""" + parser = PythonLanguageFunctionsParser() + assert parser.is_tree_key_match("my-pkg", "my_pkg") is True + assert parser.is_tree_key_match("flask", "Flask") is True + assert parser.is_tree_key_match("urllib3", "requests") is False + + +class TestResolveTreeKeyWithGoParser: + """Integration-style tests: verify _resolve_tree_key uses is_tree_key_match + for Go and prevents the fan-out explosion.""" + + def _make_retriever(self, tree_dict_keys): + """Build a minimal mock ChainOfCallsRetriever with Go parser.""" + from exploit_iq_commons.utils.chain_of_calls_retriever import ChainOfCallsRetriever, _SearchCtx + retriever = object.__new__(ChainOfCallsRetriever) + retriever.language_parser = GoLanguageFunctionsParser() + retriever.tree_dict = {k: ["root"] for k in tree_dict_keys} + ctx = _SearchCtx() + return retriever, ctx + + def test_exact_match_preferred(self): + retriever, ctx = self._make_retriever([ + "github.com/hashicorp/vault", + "github.com/hashicorp/vault/api", + ]) + result = retriever._resolve_tree_key("github.com/hashicorp/vault", ctx) + assert result == "github.com/hashicorp/vault" + + def test_subpackage_does_not_resolve_to_parent(self): + """A sub-package query must NOT resolve to a parent tree entry — + this would conflate all sub-packages under the parent module.""" + retriever, ctx = self._make_retriever([ + "github.com/hashicorp/vault", + ]) + result = retriever._resolve_tree_key("github.com/hashicorp/vault/api", ctx) + assert result is None + + def test_no_cross_org_fan_out(self): + """The bug scenario: short 2-level path must NOT match unrelated modules.""" + retriever, ctx = self._make_retriever([ + "github.com/hashicorp/vault", + "github.com/hashicorp/consul", + "github.com/hashicorp-terraform/aws", + "github.com/hashicorp-labs/experiment", + ]) + result = retriever._resolve_tree_key("github.com/hashicorp", ctx) + assert result in ("github.com/hashicorp/vault", "github.com/hashicorp/consul") + assert result != "github.com/hashicorp-terraform/aws" + assert result != "github.com/hashicorp-labs/experiment" + + def test_no_match_returns_none(self): + retriever, ctx = self._make_retriever([ + "github.com/foo/bar", + ]) + result = retriever._resolve_tree_key("github.com/baz/qux", ctx) + assert result is None + + def test_tree_additions_also_uses_boundary(self): + """Short prefix from get_package_names resolves to child module + in tree_additions, but sub-package does NOT resolve to parent.""" + retriever, ctx = self._make_retriever([]) + ctx.tree_additions["github.com/hashicorp/vault"] = ["root"] + ctx.tree_additions["github.com/hashicorp-terraform/aws"] = ["root"] + result = retriever._resolve_tree_key("github.com/hashicorp", ctx) + assert result == "github.com/hashicorp/vault" + result2 = retriever._resolve_tree_key("github.com/hashicorp/vault/api", ctx) + assert result2 is None \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/functions_parsers/tests/test_python_is_same_package.py b/src/exploit_iq_commons/utils/functions_parsers/tests/test_python_is_same_package.py new file mode 100644 index 000000000..6d9099a41 --- /dev/null +++ b/src/exploit_iq_commons/utils/functions_parsers/tests/test_python_is_same_package.py @@ -0,0 +1,77 @@ +import pytest +from unittest.mock import MagicMock +from exploit_iq_commons.utils.functions_parsers.python_functions_parser import PythonLanguageFunctionsParser +from exploit_iq_commons.utils.dep_tree import Ecosystem, DependencyTree +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers_factory import get_language_function_parser + + +class TestPythonIsSamePackage: + """Test PythonLanguageFunctionsParser.is_same_package with PEP 503 normalization + and PyPI-to-import mapping lookup.""" + + def setup_method(self): + self.parser = PythonLanguageFunctionsParser() + + def test_exact_match(self): + assert self.parser.is_same_package("urllib3", "urllib3") is True + + def test_case_insensitive(self): + assert self.parser.is_same_package("Flask", "flask") is True + + def test_hyphen_underscore_equivalence(self): + assert self.parser.is_same_package("my-package", "my_package") is True + + def test_dot_equivalence(self): + assert self.parser.is_same_package("my.package", "my-package") is True + + def test_no_match(self): + assert self.parser.is_same_package("urllib3", "requests") is False + + def test_mapping_pypi_to_import(self): + """Agent sends PyPI name 'python-dateutil', tree key is import name 'dateutil'.""" + builder = MagicMock() + builder._import_to_pypi = {"dateutil": "python-dateutil"} + self.parser.set_dependency_builder(builder) + assert self.parser.is_same_package("python-dateutil", "dateutil") is True + + def test_mapping_normalized_pypi_to_import(self): + """Agent sends 'python_dateutil' (underscore), tree key is 'dateutil'.""" + builder = MagicMock() + builder._import_to_pypi = {"dateutil": "python-dateutil"} + self.parser.set_dependency_builder(builder) + assert self.parser.is_same_package("python_dateutil", "dateutil") is True + + def test_mapping_pillow_to_pil(self): + builder = MagicMock() + builder._import_to_pypi = {"pil": "pillow"} + self.parser.set_dependency_builder(builder) + assert self.parser.is_same_package("Pillow", "pil") is True + + def test_no_builder_falls_back_to_normalization(self): + assert self.parser.is_same_package("my-pkg", "my_pkg") is True + assert self.parser.is_same_package("python-dateutil", "dateutil") is False + + def test_mapping_does_not_false_positive(self): + """Mapping for one package should not match a different package.""" + builder = MagicMock() + builder._import_to_pypi = {"dateutil": "python-dateutil"} + self.parser.set_dependency_builder(builder) + assert self.parser.is_same_package("requests", "dateutil") is False + + +class TestParserFactoryWiring: + def test_python_parser_receives_builder(self): + """Factory should call set_dependency_builder for Python parsers.""" + tree = DependencyTree(ecosystem=Ecosystem.PYTHON) + parser = get_language_function_parser(Ecosystem.PYTHON, tree) + assert hasattr(parser, '_builder') + assert parser._builder is tree.builder + + def test_python_parser_handles_none_tree(self): + parser = get_language_function_parser(Ecosystem.PYTHON, None) + assert parser._import_to_pypi == {} + + def test_go_parser_unaffected(self): + """Go parser should not have _builder attribute.""" + parser = get_language_function_parser(Ecosystem.GO, None) + assert not hasattr(parser, '_builder') \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/git_utils.py b/src/exploit_iq_commons/utils/git_utils.py new file mode 100644 index 000000000..54f477eba --- /dev/null +++ b/src/exploit_iq_commons/utils/git_utils.py @@ -0,0 +1,108 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from pathlib import Path +from pathlib import PurePath + +from git import Repo + + +def sanitize_git_url_for_path(git_url: str) -> str: + """Sanitizes a git repo URL to create a valid filesystem path component.""" + # Example: 'https://github.com/some/repo' -> 'https.github.com.some.repo' + return git_url.replace('//', '.').replace('/', '.').replace(':', '') + + +def sanitize_ref_for_path(ref: str) -> str: + """Sanitizes a git ref (branch, tag, commit) to create a valid filesystem path component. + + Git refs can contain characters that are problematic for filesystem paths: + - Forward slashes (e.g., 'feature/my-branch', 'refs/tags/v1.0') + - Colons (rare but possible) + + Uses reversible encoding: underscores are doubled first, then slashes/colons + become single underscores. This prevents collisions between refs like + 'feature/test' and 'feature_test'. + + Args: + ref: Git reference (branch name, tag name, or commit SHA) + + Returns: + Sanitized string safe for use in filesystem paths + + Example: + 'feature/my-branch' -> 'feature_my-branch' + 'feature_my-branch' -> 'feature__my-branch' + 'v1.0.0' -> 'v1.0.0' + 'refs/tags/v2.0' -> 'refs_tags_v2.0' + """ + return ref.replace('_', '__').replace('/', '_').replace(':', '_') + + +def get_repo_path_with_ref(base_dir: Path, git_url: str, ref: str) -> Path: + """Compute a ref-scoped directory path for a git repository. + + Creates a unique directory path for each (repo, ref) combination to prevent + conflicts when multiple threads work on different refs of the same repo. + + Args: + base_dir: Base directory for git repositories + git_url: Git repository URL + ref: Git reference (branch, tag, or commit SHA) + + Returns: + Path to the ref-scoped repository directory + + Example: + base_dir='/cache/git', git_url='https://github.com/org/repo', ref='v1.0' + -> '/cache/git/https.github.com.org.repo_v1.0' + """ + sanitized_url = sanitize_git_url_for_path(git_url) + sanitized_ref = sanitize_ref_for_path(ref) + return base_dir / f"{sanitized_url}_{sanitized_ref}" + + +def get_repo_from_path(base_dir: str, git_repo: str = ".git") -> Repo: + """ + Utility function for getting GitPython `Repo` object representing a Git repository. + + Parameters + ---------- + base_dir : str + Path to base directory containing one or more Git repos + git_repo : str + Relative path to Git repo in base_dir, default is ".git" + + Returns + ------- + repo + GitPython `Repo` object representing the Git repository. + + Raises + ------ + ValueError + If the repository path does not exist. + """ + repo_path = base_dir / PurePath(git_repo) + repo_path = Path(repo_path) + if os.path.exists(repo_path): + try: + return Repo(repo_path) + except Exception as e: + raise ValueError(f"Invalid Git repository: {repo_path}") from e + else: + raise ValueError(f"Path {repo_path} does not exist") diff --git a/src/exploit_iq_commons/utils/go_segmenters_with_methods.py b/src/exploit_iq_commons/utils/go_segmenters_with_methods.py new file mode 100644 index 000000000..364bdfbe2 --- /dev/null +++ b/src/exploit_iq_commons/utils/go_segmenters_with_methods.py @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import List + +from langchain_community.document_loaders.parsers.language.go import GoSegmenter + + +def parse_all_methods(code: str) -> list[str]: + """ + This function gets a source code content as multi-line string, and returns a list of all golang methods in the source. + :arg code : multi-line source string of a go source file + :return: List of all golang methods in the source, including signatures + bodies ( implementation) + """ + # regex = r"func\s*\([a-zA-Z0-9\s]*\) [a-zA-X]+\([a-zA-Z0-9\s]*\)([a-zA-Z0-9\s]*){" + regex = r"func\s*\([a-zA-Z0-9\s\*.\[\]]+\) [a-zA-Z_][a-zA-Z0-9_]*\([^)]*\)\s*(\(?[^{]*\)?)?\s*{" + methods = get_all_functions(code, regex) + return methods + + +def parse_all_anonymous_functions(code: str) -> list[str]: + """ + This function gets a source code content as multi-line string, and returns a list of all golang anonymous functions + in that source, if exists. + :param code: multi-line source string of a go source file + :return: List of all anonymous functions in the source, if any. + """ + regex = r"(var)? [a-zA-Z0-9_]+\s*(:)?=\s*func\s*.*{" + functions = get_all_functions(code, regex) + return functions + + +def get_all_functions(code: str, regex: str): + """ + This function gets a source code content as multi-line string, and a regex that identify golang functions/methods + boundaries, and construct a list of all golang functions/methods ( complete implementation, signatures + body) + :param code: multi-line source string of a go source file + :param regex: the regex to identify the needed golang type of functions - anonymous functions/methods + :return: a list containing all golang functions/methods ( each one with a complete implementation, signature + body) + """ + matches = re.finditer(regex, code) + functions = list() + for matchNum, match in enumerate(matches, start=1): + curly_brackets_counter = 1 + internal_offset = 0 + current_method_body = code[match.end():] + while curly_brackets_counter > 0 or internal_offset == 0: + left_bracket_ind = current_method_body[internal_offset:].find("{") + right_bracket_ind = current_method_body[internal_offset:].find("}") + if left_bracket_ind < right_bracket_ind and left_bracket_ind != -1: + curly_brackets_counter += 1 + internal_offset = internal_offset + left_bracket_ind + 1 + else: + curly_brackets_counter -= 1 + internal_offset = internal_offset + right_bracket_ind + 1 + current_method = code[match.start(): match.end() + 1 + internal_offset] + functions.append(current_method) + return functions + + +class GoSegmenterWithMethods(GoSegmenter): + + def __init__(self, code: str): + super().__init__(code) + + def extract_functions_classes(self) -> List[str]: + """ + This method extracts all the golang methods and anonymous functions from a source code, and appends them to + the list of all golang regular functions calculated in parent class' method. + : + :return: a list of all golang functions/methods/anonymous functions ( each one with a complete implementation, + signature + body) + """ + function_classes = super().extract_functions_classes() + methods_classes = parse_all_methods(self.code) + parse_anonymous_functions = parse_all_anonymous_functions(self.code) + function_classes.extend(methods_classes) + function_classes.extend(parse_anonymous_functions) + return function_classes diff --git a/src/exploit_iq_commons/utils/hardening_kb.py b/src/exploit_iq_commons/utils/hardening_kb.py new file mode 100644 index 000000000..22fe6c789 --- /dev/null +++ b/src/exploit_iq_commons/utils/hardening_kb.py @@ -0,0 +1,180 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Hardening Knowledge Base utilities. + +Loads the hardening_kb.json file containing compiler/linker flags that mitigate +specific CWE vulnerability categories. Provides lookup by CWE ID to retrieve +relevant hardening flags and their descriptions for LLM context. +""" + +from __future__ import annotations + +import json +import threading +from pathlib import Path + +from pydantic import BaseModel, Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + +logger = LoggingFactory.get_agent_logger(__name__) + + +class HardeningEntry(BaseModel): + """A single hardening flag entry from the knowledge base.""" + + flag: str = Field(description="Compiler/linker flag(s) for hardening") + flag_type: str = Field(description="Type: runtime, linker, warning, optimization, architecture") + description: str = Field(description="Description of what the flag does") + vulnerability_category: str = Field(description="Category of vulnerability this mitigates") + cwe_ids: list[str] = Field(default_factory=list, description="CWE IDs this flag helps mitigate") + + +# Flag types that provide actual runtime mitigation (not just warnings or optimization changes) +MITIGATING_FLAG_TYPES = frozenset({"runtime", "linker"}) + + +class HardeningKB: + """In-memory cache for hardening flags knowledge base. + + Implements singleton pattern to ensure single instance across the application. + Provides lookup by CWE ID to find relevant hardening flags. + """ + + _instance = None + _lock = threading.Lock() + + def __new__(cls): + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self) -> None: + if not hasattr(self, '_initialized'): + base_path = Path(__file__).resolve().parents[1] + self.json_path = base_path / "data" / "hardening_kb" / "hardening_kb.json" + + self._entries: list[HardeningEntry] = [] + self._cwe_index: dict[str, list[HardeningEntry]] = {} + self._initialized = True + self._load() + + @classmethod + def get_instance(cls) -> "HardeningKB": + """Get the singleton instance of HardeningKB.""" + return cls() + + def _load(self) -> None: + """Load the hardening KB JSON and build the CWE index.""" + try: + data = json.loads(self.json_path.read_text(encoding="utf-8")) + except FileNotFoundError: + logger.warning("Hardening KB JSON not found at %s", self.json_path) + return + except json.JSONDecodeError as exc: + logger.error("Failed to parse hardening KB JSON: %s", exc) + return + + mappings = data.get("mappings", []) + for mapping in mappings: + try: + entry = HardeningEntry( + flag=mapping.get("flag", "").strip(), + flag_type=mapping.get("flag_type", "unknown"), + description=mapping.get("description", ""), + vulnerability_category=mapping.get("vulnerability_category", ""), + cwe_ids=mapping.get("cwe_ids", []), + ) + self._entries.append(entry) + + for cwe_id in entry.cwe_ids: + normalized = self._normalize_cwe_id(cwe_id) + if normalized: + if normalized not in self._cwe_index: + self._cwe_index[normalized] = [] + self._cwe_index[normalized].append(entry) + + except Exception as exc: + logger.warning("Failed to parse hardening entry: %s - %s", mapping, exc) + + logger.info( + "Loaded hardening KB: %d entries, %d unique CWE mappings", + len(self._entries), + len(self._cwe_index), + ) + + @staticmethod + def _normalize_cwe_id(cwe_id: str) -> str | None: + """Normalize CWE ID to uppercase format (e.g., 'CWE-121'). + + Returns None for special values like 'N/A' or 'Multi'. + """ + if not cwe_id: + return None + cwe_id = cwe_id.strip().upper() + if cwe_id in ("N/A", "MULTI"): + return None + if not cwe_id.startswith("CWE-"): + cwe_id = f"CWE-{cwe_id}" + return cwe_id + + def lookup_by_cwe( + self, + cwe_id: str | None, + include_non_mitigating: bool = False, + ) -> list[HardeningEntry]: + """Return hardening entries that match the given CWE ID. + + By default, only returns flags that provide actual runtime mitigation + (flag_type: runtime, linker). Warning-only and optimization flags are + excluded since they don't mitigate vulnerabilities at runtime. + + Args: + cwe_id: The CWE identifier (e.g., 'CWE-121' or '121') + include_non_mitigating: If True, include warning/optimization flags + that don't provide runtime mitigation (for auditing purposes) + + Returns: + List of HardeningEntry objects that help mitigate this CWE + """ + if not cwe_id: + return [] + + normalized = self._normalize_cwe_id(cwe_id) + if not normalized: + return [] + + entries = self._cwe_index.get(normalized, []) + + if not include_non_mitigating: + entries = [e for e in entries if e.flag_type in MITIGATING_FLAG_TYPES] + + logger.debug( + "HardeningKB lookup for %s: found %d entries (include_non_mitigating=%s)", + normalized, + len(entries), + include_non_mitigating, + ) + return list(entries) + + def get_all_entries(self) -> list[HardeningEntry]: + """Return all hardening entries in the knowledge base.""" + return list(self._entries) + + diff --git a/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py b/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py new file mode 100644 index 000000000..1b2a8e6d5 --- /dev/null +++ b/src/exploit_iq_commons/utils/java_chain_of_calls_retriever.py @@ -0,0 +1,1055 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +import re +import threading +from collections import defaultdict +from dataclasses import dataclass, field +from pathlib import Path +from typing import List + +from langchain_core.documents import Document + +from exploit_iq_commons.utils.chain_of_calls_retriever_base import calculate_hashable_string_for_function, \ + ChainOfCallsRetrieverBase +from exploit_iq_commons.utils.data_utils import retrieve_from_cache, save_to_cache, DEFAULT_PICKLE_CACHE_DIRECTORY +from exploit_iq_commons.utils.dep_tree import ROOT_LEVEL_SENTINEL, DependencyTree, Ecosystem +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers_factory import ( + get_language_function_parser, +) + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.java_utils import convert_from_maven_artifact, extract_jar_name, is_maven_gav, extract_method_name_with_params, \ + create_inheritance_map, get_target_class_names, dummy_package_name +from exploit_iq_commons.data_models.input import SourceDocumentsInfo + +logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") + +# Lowercase package segments; class segments start with uppercase; allow dots or $ for inners +_FQCN_STRICT_RE = re.compile( + r""" + ^(?=.+\.) # must contain at least one dot (have a package) + (?:[a-z_][a-z0-9_]*\.)+ # package segments (conventional lowercase) + [A-Z][A-Za-z0-9_$]* # top-level class (starts uppercase by convention) + (?:[.$][A-Z][A-Za-z0-9_$]*)* # optional inner classes via '.' or '$' + $""", + re.X, +) + +# Matches "--sources/" in the path +_GAV_DIR_RE = re.compile(r'([^/\\]+)-([0-9][^/\\]*)-sources(?:/|\\)') + +# Module-level cache: one _JavaRepoData per (git_repo, ref). +# Entries are explicitly removed via _release_repo_data when ref count drops to 0. +_repo_data_cache: dict[tuple, '_JavaRepoData'] = {} +_repo_data_cache_lock = threading.Lock() + + +class _JavaRepoData: + """Repo-level data shared across all JavaChainOfCallsRetriever instances + for the same (git_repo, ref). + + Holds the 5 CCA structures + source-keyed function index. Multiple + package-specific retrievers reference the same instance, eliminating + duplicate in-memory copies of multi-GB data structures. + + Immutable after construction — safe for concurrent reads. + """ + + def __init__(self, documents_of_functions_raw: list[Document], + documents_of_types: list, + documents_of_full_sources: dict, + type_inheritance: dict, + types_classes_fields_mapping: dict, + functions_local_variables_index: dict): + logger.debug("_JavaRepoData - building source-keyed function index") + # Build source-keyed function index (always needed by FL) + self.source_to_fn_docs: dict[str, list[Document]] = {} + for doc in documents_of_functions_raw: + src = doc.metadata.get('source', '') + if src in self.source_to_fn_docs: + self.source_to_fn_docs[src].append(doc) + else: + self.source_to_fn_docs[src] = [doc] + logger.debug("_JavaRepoData - source_to_fn_docs built (%d sources, %d docs)", + len(self.source_to_fn_docs), len(documents_of_functions_raw)) + + # CCA structures — read-only, shared by all package retrievers + self.documents_of_types = documents_of_types + self.documents_of_full_sources = documents_of_full_sources + self.type_inheritance = type_inheritance + self.types_classes_fields_mapping = types_classes_fields_mapping + self.functions_local_variables_index = functions_local_variables_index + logger.debug("_JavaRepoData - assigned CCA structures (types=%d, full_sources=%d, " + "type_inheritance=%d, fields_mapping=%d, local_vars=%d)", + len(documents_of_types), len(documents_of_full_sources), + len(type_inheritance), len(types_classes_fields_mapping), + len(functions_local_variables_index)) + + # Reference count: how many JavaChainOfCallsRetriever instances reference + # this repo data. Managed by _get_or_create_repo_data / _release_repo_data. + self._ref_count = 0 + self._cache_key: tuple | None = None + + +def _release_repo_data(repo_data: _JavaRepoData) -> None: + """Decrement ref count and remove from cache when no searcher references it. + + Called from transitive_code_search.py when a searcher is evicted from _searcher_cache. + """ + with _repo_data_cache_lock: + repo_data._ref_count -= 1 + logger.debug("_release_repo_data - decremented ref_count to %d for %s", repo_data._ref_count, repo_data._cache_key) + if repo_data._ref_count <= 0 and repo_data._cache_key is not None: + _repo_data_cache.pop(repo_data._cache_key, None) + logger.info("Released shared _JavaRepoData for %s", repo_data._cache_key) + + +def _get_or_create_repo_data(documents: List[Document], + code_source_info: SourceDocumentsInfo, + language_parser) -> _JavaRepoData: + """Get or create shared repo-level data for the given (git_repo, ref). + + Thread-safe: uses a lock to prevent duplicate creation. In practice, + concurrent calls for the same key are already serialized by the upstream + repo_lock in _build_or_get_cached, but the lock here provides defense-in-depth. + """ + key = (code_source_info.git_repo, code_source_info.ref) + logger.debug("_get_or_create_repo_data - looking up key %s", key) + + # Fast path: already cached + with _repo_data_cache_lock: + repo_data = _repo_data_cache.get(key) + if repo_data is not None: + repo_data._ref_count += 1 + logger.info("Reusing shared _JavaRepoData for %s (ref_count=%d)", key, repo_data._ref_count) + return repo_data + + logger.debug("_get_or_create_repo_data - cache miss, building repo data for %s", key) + + # Build all repo-level structures (same logic as old _build_repo_data) + allowed_files_extensions = language_parser.supported_files_extensions() + + logger.debug("_get_or_create_repo_data - loading pickle sub-caches") + documents_of_functions_raw, exist_fn = retrieve_from_cache( + DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "documents_of_functions") + documents_of_types, exist_ty = retrieve_from_cache( + DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "documents_of_types") + documents_of_full_sources, exist_fs = retrieve_from_cache( + DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "documents_of_full_sources") + + need_fn = not exist_fn + need_ty = not exist_ty + need_fs = not exist_fs + logger.debug("_get_or_create_repo_data - pickle cache status: functions=%s, types=%s, full_sources=%s", + "hit" if exist_fn else "miss", "hit" if exist_ty else "miss", "hit" if exist_fs else "miss") + + # Single pass over raw documents — only needed if any of fn/ty/fs are missing from pickle + if need_fn or need_ty or need_fs: + logger.debug("_get_or_create_repo_data - classifying raw documents (need_functions=%s, need_types=%s, need_full_sources=%s)", + need_fn, need_ty, need_fs) + if need_fn: + fn_list = [] + if need_ty: + ty_list = [] + if need_fs: + full_dict = {} + + for doc in documents: + source = str(doc.metadata['source']) + if not any(source.endswith(ext) for ext in allowed_files_extensions): + continue + content_type = doc.metadata.get('content_type') + if need_fs and content_type == 'simplified_code': + full_dict[doc.metadata.get('source')] = doc + if need_fn and language_parser.is_function(doc): + fn_list.append(doc) + if need_ty and language_parser.is_doc_type(doc): + ty_list.append(doc) + + if need_fn: + documents_of_functions_raw = fn_list + logger.debug("_get_or_create_repo_data - saving documents_of_functions to pickle (%d docs)", len(fn_list)) + save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, + documents_of_functions_raw, "documents_of_functions") + if need_ty: + documents_of_types = ty_list + logger.debug("_get_or_create_repo_data - saving documents_of_types to pickle (%d docs)", len(ty_list)) + save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, + documents_of_types, "documents_of_types") + if need_fs: + documents_of_full_sources = full_dict + logger.debug("_get_or_create_repo_data - saving documents_of_full_sources to pickle (%d docs)", len(full_dict)) + save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, + documents_of_full_sources, "documents_of_full_sources") + + type_inheritance, exist = retrieve_from_cache( + DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "type_inheritance") + if not exist: + logger.debug("_get_or_create_repo_data - building type_inheritance from documents_of_full_sources") + type_inheritance = create_inheritance_map(documents_of_full_sources.values()) + save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, + type_inheritance, "type_inheritance") + else: + logger.debug("_get_or_create_repo_data - type_inheritance loaded from pickle") + + types_classes_fields_mapping, exist = retrieve_from_cache( + DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "types_classes_fields_mapping") + if not exist: + logger.debug("_get_or_create_repo_data - building types_classes_fields_mapping") + types_classes_fields_mapping = language_parser.parse_all_type_struct_class_to_fields( + documents_of_types, type_inheritance) + save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, + types_classes_fields_mapping, "types_classes_fields_mapping") + else: + logger.debug("_get_or_create_repo_data - types_classes_fields_mapping loaded from pickle") + + functions_local_variables_index, exist = retrieve_from_cache( + DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, "functions_local_variables_index") + if not exist: + logger.debug("_get_or_create_repo_data - building functions_local_variables_index") + functions_local_variables_index = language_parser.create_map_of_local_vars(documents_of_functions_raw) + save_to_cache(DEFAULT_PICKLE_CACHE_DIRECTORY, code_source_info.git_repo, code_source_info.ref, + functions_local_variables_index, "functions_local_variables_index") + else: + logger.debug("_get_or_create_repo_data - functions_local_variables_index loaded from pickle") + + logger.debug("_get_or_create_repo_data - creating _JavaRepoData instance") + repo_data = _JavaRepoData( + documents_of_functions_raw=documents_of_functions_raw, + documents_of_types=documents_of_types, + documents_of_full_sources=documents_of_full_sources, + type_inheritance=type_inheritance, + types_classes_fields_mapping=types_classes_fields_mapping, + functions_local_variables_index=functions_local_variables_index, + ) + + with _repo_data_cache_lock: + # Double-check: another thread may have created it while we were building + existing = _repo_data_cache.get(key) + if existing is not None: + existing._ref_count += 1 + logger.debug("_get_or_create_repo_data - lost race, reusing existing _JavaRepoData for %s (ref_count=%d)", + key, existing._ref_count) + return existing + repo_data._cache_key = key + repo_data._ref_count = 1 + _repo_data_cache[key] = repo_data + logger.info("Created new shared _JavaRepoData for %s", key) + + return repo_data + + +@dataclass +class _JavaSearchCtx: + """Per-search mutable state, created fresh for each get_relevant_documents call. + + Keeps the retriever instance fully immutable after __init__, so concurrent + CCA calls can safely share it without deep-copy. + """ + found_path: bool = False + exclusions: defaultdict = field(default_factory=lambda: defaultdict(list)) + method_exclusions: defaultdict = field(default_factory=lambda: defaultdict(dict)) + last_visited_parent_package_indexes: dict = field(default_factory=dict) + tree_additions: dict = field(default_factory=dict) + root_docs: list = field(default_factory=list) + jar_to_docs: dict = field(default_factory=dict) + + +class JavaChainOfCallsRetriever(ChainOfCallsRetrieverBase): + """A ChainOfCall retriever that Knows how to perform a deep search, looking for a function usage, whether it's being + called from the application code base or not. + """ + def __init__(self, documents: List[Document], + ecosystem: Ecosystem, + manifest_path: Path, + query: str, + code_source_info: SourceDocumentsInfo, + uber_jar_file_threshold: int): + """ + :param documents: + List of documents containing the functions/methods and classes/types of the application code + + application dependencies' code + :param ecosystem: Ecosystem + The programming language of the list of documents + :param manifest_path: str + The manifest of the application, defining all the direct and transitive dependencies of the application, being + used here to build the dependency tree for a more efficient lookup and search. + """ + + logger.debug("Chain of Calls Retriever - creating instance for ecosystem=%s", ecosystem) + self.ecosystem = ecosystem + logger.debug("Chain of Calls Retriever - creating dependency tree") + # Build dependency tree based on the parameter programming language/package manager. + self.dependency_tree = DependencyTree(ecosystem=ecosystem, query=query) + logger.debug("Chain of Calls Retriever - getting language parser") + self.language_parser = get_language_function_parser(ecosystem, self.dependency_tree) + self.manifest_path = manifest_path + # A dependency tree object is a must, because otherwise, the search is much more expensive and not efficient. + if self.dependency_tree.builder is None: + raise RuntimeError("Couldn't continue as dependency tree weren't generated") + + self.tree_dict = dict() + + # Build a dependency tree using the dependency tree builder logic. + logger.debug("Chain of Calls Retriever - building dependency tree from manifest") + tree = self.dependency_tree.builder.build_tree(manifest_path=manifest_path) + for package, parents in tree.items(): + parents.extend([package]) + self.tree_dict[package] = parents + self.supported_packages = list(self.tree_dict.keys()) + logger.debug("Chain of Calls Retriever - dependency tree built (%d packages)", len(self.supported_packages)) + + # Get or create shared repo-level data. Multiple retrievers for the + # same (git_repo, ref) share one _JavaRepoData instance, eliminating + # duplicate multi-GB data structures in memory. + logger.debug("Chain of Calls Retriever - getting or creating shared repo data") + self._repo_data = _get_or_create_repo_data(documents, code_source_info, self.language_parser) + + # Set direct attribute references to shared repo data for backward + # compatibility — all existing code reads self.xxx and works unchanged. + self._source_to_fn_docs = self._repo_data.source_to_fn_docs + self.documents_of_types = self._repo_data.documents_of_types + self.documents_of_full_sources = self._repo_data.documents_of_full_sources + self.type_inheritance = self._repo_data.type_inheritance + self.types_classes_fields_mapping = self._repo_data.types_classes_fields_mapping + self.functions_local_variables_index = self._repo_data.functions_local_variables_index + logger.debug("Chain of Calls Retriever - shared repo data linked") + + # Pre-compute which jar_names match any supported package (O(unique_jars × packages) + # instead of O(all_docs × packages)) + logger.debug("Chain of Calls Retriever - computing valid jar names") + prefix_3p = self.language_parser.dir_name_for_3rd_party_packages() + unique_jar_names = {extract_jar_name(doc.metadata['source']) + for doc in self._iter_documents_of_functions() + if doc.metadata.get('content_type') == 'functions_classes' + and doc.metadata['source'].startswith(prefix_3p)} + valid_jar_names = {jar_name for jar_name in unique_jar_names + if any(p for p in self.supported_packages + if p is not None and jar_name in p)} + logger.debug("Chain of Calls Retriever - found %d unique jars, %d valid jars", + len(unique_jar_names), len(valid_jar_names)) + + # Build jar-name index directly from documents_of_functions without + # materializing an intermediate self.documents list. + # Filters the same way: keep docs with content_type 'functions_classes' + # that are either root-level or have a jar_name in valid_jar_names. + logger.debug("Chain of Calls Retriever - building doc index (root_docs + jar_to_docs)") + self._root_docs, self._jar_to_docs = self._build_doc_index_filtered( + self._iter_documents_of_functions(), prefix_3p, valid_jar_names) + logger.debug("Chain of Calls Retriever - doc index built (root_docs=%d, jar_groups=%d)", + len(self._root_docs), len(self._jar_to_docs)) + + # Detect uber-JARs: JARs with more unique source files than the + # threshold contain shaded libraries from unrelated artifacts. + # Disabling _is_same_artifact for these prevents catastrophic + # performance (O(all_files) type resolution instead of O(same_jar)). + uber_jars = set() + for jar_name, docs in self._jar_to_docs.items(): + unique_sources = len({doc.metadata['source'] for doc in docs}) + if unique_sources > uber_jar_file_threshold: + uber_jars.add(jar_name) + logger.info("Detected uber-JAR '%s' with %d unique source files (threshold=%d), " + "disabling same-artifact bypass", jar_name, unique_sources, uber_jar_file_threshold) + if uber_jars: + self.language_parser.set_uber_jar_dirs(frozenset(uber_jars)) + + # Satisfy base class documents attribute via property (avoids duplicate list) + self._documents_materialized = None + logger.debug("Chain of Calls Retriever - init complete") + + @property + def documents(self): + """Lazily reconstruct the filtered documents list from the pre-built index. + Only materialized if explicitly accessed (e.g. via get_documents()). + """ + if self._documents_materialized is None: + self._documents_materialized = list(self._root_docs) + for docs in self._jar_to_docs.values(): + self._documents_materialized.extend(docs) + return self._documents_materialized + + @documents.setter + def documents(self, value): + self._documents_materialized = value + + def _iter_documents_of_functions(self): + """Iterate over all function documents from the source-keyed index + without materializing a flat list.""" + for docs in self._source_to_fn_docs.values(): + yield from docs + + @property + def documents_of_functions(self): + """Provide the documents_of_functions interface expected by callers + (base class attribute, FL, search_for_called_function) by iterating + the source-keyed index lazily. + + Returns a list for callers that need indexing/len; built on demand + from the memory-efficient _source_to_fn_docs index. + """ + return list(self._iter_documents_of_functions()) + + @documents_of_functions.setter + def documents_of_functions(self, value): + """Allow direct assignment (used by tests that bypass __init__).""" + self._source_to_fn_docs = {} + for doc in (value or []): + src = doc.metadata.get('source', '') + if src in self._source_to_fn_docs: + self._source_to_fn_docs[src].append(doc) + else: + self._source_to_fn_docs[src] = [doc] + + def get_fn_docs_by_source(self, source: str) -> list[Document]: + """Fast O(1) lookup of function documents by source path. + Used by find_function() instead of iterating all documents.""" + return self._source_to_fn_docs.get(source, []) + + def _get_parents(self, package: str, ctx: _JavaSearchCtx) -> list | None: + """Look up parents for a package, checking both the immutable tree_dict + and any packages added during the current search (ctx.tree_additions).""" + parents = self.tree_dict.get(package) + if parents is not None: + return parents + return ctx.tree_additions.get(package) + + @staticmethod + def _build_doc_index_from(documents, language_parser): + """Build root_docs list and jar_to_docs dict from a documents list.""" + root_docs = [] + jar_to_docs: dict[str, list[Document]] = {} + for doc in documents: + if language_parser.is_root_package(doc): + root_docs.append(doc) + else: + jar_name = extract_jar_name(doc.metadata['source']) + if jar_name not in jar_to_docs: + jar_to_docs[jar_name] = [] + jar_to_docs[jar_name].append(doc) + return root_docs, jar_to_docs + + def _build_doc_index_filtered(self, documents_of_functions, prefix_3p, valid_jar_names): + """Build root_docs and jar_to_docs directly from documents_of_functions, + applying the content_type and valid_jar_names filter inline. + + This avoids materializing an intermediate self.documents list. + """ + root_docs = [] + jar_to_docs: dict[str, list[Document]] = {} + for doc in documents_of_functions: + if doc.metadata.get('content_type') != 'functions_classes': + continue + source = doc.metadata['source'] + if source.startswith(prefix_3p): + jar_name = extract_jar_name(source) + if jar_name not in valid_jar_names: + continue + if jar_name not in jar_to_docs: + jar_to_docs[jar_name] = [] + jar_to_docs[jar_name].append(doc) + else: + if self.language_parser.is_root_package(doc): + root_docs.append(doc) + else: + jar_name = extract_jar_name(source) + if jar_name not in jar_to_docs: + jar_to_docs[jar_name] = [] + jar_to_docs[jar_name].append(doc) + return root_docs, jar_to_docs + + def __find_caller_function(self, document_function: Document, function_package: str, ctx: _JavaSearchCtx) -> Document: + """ + This method gets function and package as arguments, search and return a caller function of a package, if exists + :param document_function: the document containing the function code and signature + :param function_package: the package name containing the function + :param ctx: per-search mutable state + :return: a document of a function that is calling document_function + """ + direct_parents = list() + # gets list of all direct parents of function + + parents = self._get_parents(function_package, ctx) + if parents: + direct_parents.extend(parents) + # gets list of documents to search in only from parents of function' package. + function_name_to_search = self.language_parser.get_function_name(document_function) + + function_file_name = document_function.metadata.get('source') + last_visited_package_index = (ctx.last_visited_parent_package_indexes + .get(calculate_hashable_string_for_function(function_file_name, + function_name_to_search), 0)) + package_exclusions = ctx.exclusions[function_package] + method_exclusions = ctx.method_exclusions[function_package] + + target_class_names: frozenset[str] + + fqcn = self.language_parser.get_class_name_from_class_function(document_function) + if "dummy" not in function_file_name: + key = (fqcn, function_file_name) + target_class_names = get_target_class_names(self.type_inheritance[key]) + # Non-dummy path: use self.documents_of_types directly (read-only, no copy needed) + documents_of_types = self.documents_of_types + else: + fqcn_no_dummy = fqcn.replace(dummy_package_name, "") + target_class_names = frozenset([fqcn_no_dummy]) + target_type_doc = Document(page_content="public class " + fqcn.rpartition('.')[2] + "{}", + metadata={"source": self.language_parser.dir_name_for_3rd_party_packages() + + "/" + fqcn_no_dummy.replace(".", "/") + + ".java", + "ecosystem": self.ecosystem}) + # Dummy path: copy only here since we need to append a synthetic doc + documents_of_types = self.documents_of_types + [target_type_doc] + document_function = target_type_doc + + # Search for caller functions only at parents according to dependency tree. + # Iterate candidate docs lazily via generators instead of collecting into a list first. + logger.debug("__find_caller_function: searching for callers of '%s' in '%s', fqcn='%s'", + function_name_to_search, function_package, fqcn) + def _candidate_docs(): + for package in direct_parents[last_visited_package_index:]: + sources_location_packages = True + if self._get_parents(package, ctx)[0] == ROOT_LEVEL_SENTINEL: + sources_location_packages = False + + artifact_id = package.split(":", 2)[1] + possible_docs = self._get_possible_docs(function_name_to_search, + artifact_id, + sources_location_packages, + target_class_names, + method_exclusions, + ctx.root_docs, ctx.jar_to_docs) + + jar_name = convert_from_maven_artifact(package) + yield from self.get_functions_for_package(package_name=jar_name, + documents=possible_docs, + sources_location_packages=sources_location_packages, + function_to_search=function_name_to_search, + callee_function_file_name=function_file_name) + + # Perform the search on candidate caller functions (lazily generated) + for doc in _candidate_docs(): + function_is_being_called = self.language_parser.search_for_called_function(caller_function=doc, + callee_function_name= + function_name_to_search, + callee_function=document_function, + callee_function_package= + function_package, + code_documents= + self.documents_of_full_sources, + type_documents= + documents_of_types, + callee_function_file_name= + function_file_name, + fields_of_types= + self.types_classes_fields_mapping, + functions_local_variables_index= + self.functions_local_variables_index, + documents_of_functions= + self._source_to_fn_docs, + type_inheritance=self.type_inheritance) + + method_name = extract_method_name_with_params(doc.page_content) + key = (doc.metadata['source'], function_name_to_search, target_class_names, method_name) + method_exclusions[key] = True + + # If current document is found to be calling document_function in function_package, then return it as a + # match, and add it to exclusions so it will not consider it when backtracking in order to prevent cycles. + if function_is_being_called: + package_exclusions.append(doc) + return doc + + # If didn't find a matching caller function document, returns None. + logger.debug("__find_caller_function: no caller found for '%s' in '%s'", function_name_to_search, function_package) + return None + + def _is_doc_excluded(self, doc: Document, exclusions: list[Document]) -> bool: + """ + Checks if a document is in the exclusions list based on its + function body and source metadata. + """ + if not exclusions: + return False + + # Normalize once for the doc we are checking + doc_function_content = doc.page_content.strip() + doc_source = doc.metadata.get('source').strip() + + for exclusion_doc in exclusions: + # First compare sources – cheap and often different + exclusion_source = exclusion_doc.metadata.get('source').strip() + if exclusion_source != doc_source: + continue + + # Only if source matches, compare (more expensive) function content + exclusion_function_content = exclusion_doc.page_content.strip() + if exclusion_function_content == doc_function_content: + return True + + return False + + def _is_method_excluded(self, function_name_to_search: str, target_class_names: frozenset[str], doc: Document, method_exclusions: dict) -> bool: + method_source = doc.metadata['source'] + method_name = extract_method_name_with_params(doc.page_content) + + key = (method_source, function_name_to_search, target_class_names, method_name) + + return key in method_exclusions + + def _get_possible_docs(self, function_name_to_search: str, package: str, + sources_location_packages: bool, + target_class_names: frozenset[str], + method_exclusions: dict, + root_docs: list, jar_to_docs: dict) -> list[Document]: + """Core filtering logic used by both get_possible_docs and __find_caller_function.""" + if sources_location_packages: + candidates = [] + for jar_name, docs in jar_to_docs.items(): + if package in jar_name: + candidates.extend(docs) + result = [doc for doc in candidates + if not self._is_method_excluded(function_name_to_search, target_class_names, doc, method_exclusions) and + (f"{function_name_to_search}(" in doc.page_content or f"::{function_name_to_search}" in doc.page_content)] + else: + result = [doc for doc in root_docs + if not self._is_method_excluded(function_name_to_search, target_class_names, doc, method_exclusions) and + (f"{function_name_to_search}(" in doc.page_content or f"::{function_name_to_search}" in doc.page_content)] + + return result + + def get_possible_docs(self, function_name_to_search: str, package: str, exclusions: list[Document], + sources_location_packages: bool, + target_class_names: frozenset[str], + method_exclusions: dict) -> (list[Document], bool): + """ + Filter out irrelevant functions that cannot be caller functions. + Delegates to _get_possible_docs using the instance's pre-built index. + """ + return self._get_possible_docs(function_name_to_search, package, + sources_location_packages, target_class_names, + method_exclusions, self._root_docs, self._jar_to_docs) + + # This method is the entry point for the chain_of_calls_retriever transitive search, it gets a query, + # in the form of "package_name, function", and returns a 2-tuple of (list_of_documents_in_path, bool_result). + # if bool_result is True, then list_of_documents_in_path containing a list of documents that is being a chain of + # calls from application to input function in the input package. + def get_relevant_documents(self, query: str) -> tuple[List[Document], bool]: + """Sync implementations for retriever.""" + ctx = _JavaSearchCtx() + + class_name, method_name, package_name = self.extract_from_query(query) + + end_loop = False + found_package = False + matching_documents = [] + # Check if input package is in dependency tree + for package in self.tree_dict: + if self.language_parser.is_same_package(package_name, package): + package_name = package + found_package = True + break + # If it's found, then create a document for it. + if found_package: + target_function_doc = self.__find_initial_function(class_name=class_name, + method_name=method_name, + package_name=package_name, + ctx=ctx) + # Use the pre-built index from __init__ (immutable, shared across calls) + ctx.root_docs = self._root_docs + ctx.jar_to_docs = self._jar_to_docs + + if target_function_doc: + matching_documents.append(target_function_doc) + logger.info(f"\nmatching_documents size is {len(matching_documents)}") + # Otherwise, don't even start the process as the target function is not in dependencies tree or not a standard + # library + else: + end_loop = True + logger.error(f"Cannot find initial function=${method_name}, in package=${package_name}") + # If not, there is a chance that the package is some standard library in the ecosystem. + else: + # Try to create dummy package for ecosystem standard library function, in such case, build a document for + # vulnerable function in standard lib package of language. + package_name = dummy_package_name + + page_content = self.language_parser.get_dummy_function(method_name) + target_function_doc = Document(page_content=page_content + , metadata={"source": package_name + "/" + class_name.replace(".", "/") + ".java", + "ecosystem": self.ecosystem}) + matching_documents.append(target_function_doc) + logger.info(f"\nmatching_documents size is {len(matching_documents)}") + + importing_docs = self.language_parser.document_imports_package(self.documents_of_full_sources, class_name) + + root_package = next((key for key, value in self.tree_dict.items() if ROOT_LEVEL_SENTINEL in value), None) + prefix_of_3rd_parties_libs = self.language_parser.dir_name_for_3rd_party_packages() + # find all parents ( all importing packages) of the input package so we'll have candidate pkgs to search in. + parents = list({ + key + for doc in importing_docs + if doc.metadata['source'].startswith(prefix_of_3rd_parties_libs) + for key in self.tree_dict.keys() + if extract_jar_name(doc.metadata['source']) in key + }) + + for doc in importing_docs: + if not doc.metadata.get('source').startswith(prefix_of_3rd_parties_libs): + parents.append(root_package) + break + ctx.tree_additions[package_name] = parents + + # Build local doc index for the dummy package search by selecting matching + # entries from the pre-built index (avoids O(N) rescan of all documents) + dummy_parents = ctx.tree_additions[package_name] + ctx.root_docs = list(self._root_docs) + ctx.jar_to_docs = {} + for jar_name, docs in self._jar_to_docs.items(): + if any(parent for parent in dummy_parents + if parent is not None and jar_name in parent): + ctx.jar_to_docs[jar_name] = docs + + current_package_name = package_name + # main loop. + while not end_loop: + # Find a caller function and containing package in the dependency tree according to hierarchy + found_document = self.__find_caller_function(document_function=target_function_doc, + function_package=current_package_name, + ctx=ctx) + # If found, then add it to path + if found_document: + matching_documents.append(found_document) + logger.info(f"\nmatching_documents size is {len(matching_documents)}") + # If the function is in the application ( root package), then we finished and found such a path. + if self.language_parser.is_root_package(found_document): + end_loop = True + ctx.found_path = True + # Otherwise, we continue to search for callers for the current found function, in order to extend + # the chain of calls and potentially find a path from application to the vulnerable + # function in input package + else: + target_function_doc = found_document + # extract package name from function document + current_package_name = self.__determine_doc_package_name(target_function_doc, ctx) + else: + # end loop because didn't find a caller for initial function + if len(matching_documents) == 1: + end_loop = True + # Backtrack - means that we're going back because current function has no callers anywhere, so we + # need to remove it, and continue with other possible potential called functions from its caller + else: + dead_end_node = matching_documents.pop() + logger.info(f"\nmatching_documents size is {len(matching_documents)}") + # Excludes dead end function node from future searches, as it led to nowhere. + ctx.exclusions[current_package_name].append(dead_end_node) + target_function_doc = matching_documents[-1] + current_package_name = self.__determine_doc_package_name(target_function_doc, ctx) + + # When the loop is finished, return list of documents ( path) and boolean indicating whether a path was + # found or not. + return matching_documents, ctx.found_path + + def extract_from_query(self, query: str) -> tuple[str, str, str]: + (package_name, function) = tuple( query.splitlines()[0].strip('"\'\u2018\u2019\u201c\u201d').replace("#", ".").split(",")) + + class_name = function.rpartition('.')[0] + method_name = re.sub(r'\(.*\)$', '', function.rpartition('.')[2]) + + if not class_name and self.is_java_fqcn(package_name): + class_name = package_name + + package_name, class_name = self.infer_class_name_and_package_name(method_name, class_name, package_name) + + return class_name, method_name, package_name + + def __determine_doc_package_name(self, target_function_doc, ctx: _JavaSearchCtx): + """ + Determine the logical package/dependency identifier for a given document. + + Checks both self.tree_dict (immutable) and ctx.tree_additions (search-local). + """ + src = target_function_doc.metadata['source'] + if self.language_parser.dir_name_for_3rd_party_packages() in src: + jar_name = extract_jar_name(src) + for dep_vals in itertools.chain(self.tree_dict.values(), ctx.tree_additions.values()): + for dep_val in dep_vals: + if jar_name in dep_val: + return dep_val + else: + jar_name = extract_jar_name('/' + src) + parents = self._get_parents(jar_name, ctx) + if parents: + return jar_name + + return "" + + def __find_initial_function(self, class_name: str, method_name: str, package_name: str, + ctx: _JavaSearchCtx) -> Document | None: + jar_name = convert_from_maven_artifact(package_name) + prefix_3p = self.language_parser.dir_name_for_3rd_party_packages() + + # Iterate over _root_docs + _jar_to_docs values (same docs as old self.documents) + # using a generator to avoid materializing the filtered list. + def _candidates(): + for doc in self._root_docs: + yield doc + for docs in self._jar_to_docs.values(): + for doc in docs: + yield doc + + package_exclusions = ctx.exclusions[package_name] + # TODO handle method overloading + for doc in _candidates(): + source = doc.metadata.get('source') + if prefix_3p in source and jar_name not in source: + continue + if (self.language_parser.get_class_name_from_class_function(doc) == class_name and + self.language_parser.get_function_name(doc) == method_name): + package_exclusions.append(doc) + return doc + + return None + + # This method prints as a multi-line string the path of chains of calls , from the start ( first caller) to the end + # (target function). + def print_call_hierarchy(self, call_hierarchy_list: list[Document]) -> list[str]: + results = [] + for i, package_function in enumerate(reversed(call_hierarchy_list)): + class_name = self.language_parser.get_class_name_from_class_function(package_function) + function_name = self.language_parser.get_function_name(package_function) + jar_name = extract_jar_name(package_function.metadata['source']) + current_level = f"(jar_name={jar_name}, class_name={class_name}, function={function_name},depth={i})" + + results.append(current_level) + + return results + + def function_called_from_caller_body(self, document: Document, function_to_search: str) -> bool: + """ + Return True iff the Java method body (document.page_content) contains a call + to `function_to_search`. Assumptions: the source is a single method (no + comments/annotations/javadocs before it). + + Supported matches: + - Bare method calls: getProperty(...) + - Dotted method calls: obj.getProperty(...), BeanUtils.getProperty(...) + - Chained method calls: PropertyUtilsBean.getInstance().getProperty(...) + - Generic method calls: obj.getProperty(...) + - Constructor calls: new Test(...), new pkg.Test<>(...), new Test(){...} + - Method references: obj::method, ClassName::method, pkg.Class::method + - Constructor references: ClassName::new, pkg.Class::new + """ + name_raw = (function_to_search or "").strip() + if not name_raw: + return True # preserve original behavior + + # Strip any "(...)" suffix if given and extract optional qualifier + base = re.sub(r"\s*\(.*\)\s*$", "", name_raw) + m = re.match(r"^(?:(?P[\w$.]+)\.)?(?P[A-Za-z_$][\w$]*)$", base) + if m: + target_qual = m.group("qual") # e.g., "org.foo.Bar" + target_simple = m.group("mname") # e.g., "Bar" or a method name like "getProperty" + else: + target_qual, target_simple = None, re.sub(r"[^\w$]", "", base) + + src = document.page_content or "" + + # Heuristic: treat as "type-like" if it looks like a class/simple type token (enables ::new) + def _looks_like_type_name(s: str) -> bool: + return bool(s) and (s[0].isupper() or s.endswith("[]")) + + # ---------- fast early exit ---------- + # If none of the key tokens appear (simple name, qualifier, new , ::, or ::new with class tokens), bail. + if ( + target_simple not in src + and (not target_qual or target_qual not in src) + and not ( + "::new" in src and ( + (_looks_like_type_name(target_simple)) or + (target_qual and target_qual.split(".")[-1] in src) + ) + ) + ): + return False + + # ---------- restrict to method body (avoid matching header) ---------- + body_start = src.find("{") + body = src[body_start + 1:] if body_start != -1 else src + + # ---------- mask string/char literals (cheap) ---------- + def _strip_strings(s: str) -> str: + out = [] + i, n = 0, len(s) + while i < n: + c = s[i] + if c == '"': + i += 1 + while i < n: + if s[i] == '\\': + i += 2 + elif s[i] == '"': + i += 1 + break + else: + i += 1 + out.append(' ') + elif c == "'": + i += 1 + while i < n: + if s[i] == '\\': + i += 2 + elif s[i] == "'": + i += 1 + break + else: + i += 1 + out.append(' ') + else: + out.append(c) + i += 1 + return ''.join(out) + + body = _strip_strings(body) + + # ---------- assemble patterns ---------- + simple_esc = re.escape(target_simple) + parts = [] + + # Method-call patterns (unchanged, plus optional before method name for generics) + parts.extend([ + rf"\.\s*(?:<[^>]*>\s*)?{simple_esc}\s*\(", # chained/dotted: obj.name( + rf"(?]*>\s*)?{simple_esc}\s*\(") + parts.append(rf"(?]*>\s*)?{simple_esc}\s*\(") + + # Constructor-call patterns: new (...), allowing optional generics and anonymous class + parts.append(rf"(?]*>)?\s*\(") + if target_qual: + parts.append(rf"(?]*>)?\s*\(") + + # ------- method reference patterns (::) ------- + # Unqualified/any-LHS method reference to the target method: + # this::method, super::method, obj::method, Type::method, pkg.Type::method + parts.append(rf"::\s*(?:<[^>]*>\s*)?{simple_esc}\b") + if target_qual: + parts.append(rf"(?]*>\s*)?{simple_esc}\b") + parts.append(rf"(?]*>\s*)?{simple_esc}\b") + + # Constructor reference patterns: + # If the searched name looks like a type, accept ClassName::new and pkg.Class::new + if _looks_like_type_name(target_simple): + parts.append(rf"(? bool: + return (not self.language_parser.is_root_package(document) and + document.metadata.get('source').__contains__(package_name)) # package_name = jar_name + + def is_java_fqcn(self, s: str) -> bool: + """ + Return True iff `s` looks like a fully qualified Java class name. + + Strict mode (default): + - package: one or more lowercase segments like 'com.example.' + - class & inners: segments start uppercase (e.g., 'Map.Entry' or 'Map$Entry') + Permissive mode: + - allows uppercase/underscore in package segments and lowercase class names + + Rejects arrays/generics/whitespace/etc. Examples: + True: 'java.lang.String', 'java.util.Map.Entry', 'java.util.Map$Entry' + False: 'String', 'java.util', 'java.lang.String[]', 'com..example.Class', + 'com.example.Class' + """ + s = s.strip() + if not s: + return False + + return _FQCN_STRICT_RE.match(s) is not None + + def infer_class_name_and_package_name(self, method_name: str, class_name: str, package_name: str) -> (str, str): + maven_gav = is_maven_gav(package_name) + + # Two-pass approach: prefer exact FQCN match, fall back to simple-name match. + # This prevents substring false positives (e.g., XStream matching XStreamer). + for exact_first in (True, False): + for doc in self._iter_documents_of_functions(): + if method_name in doc.page_content and method_name == self.language_parser.get_function_name(doc): + doc_fqcn = self.language_parser.get_class_name_from_class_function(doc) + + if exact_first: + # Pass 1: exact FQCN match only + if doc_fqcn != class_name: + continue + else: + # Pass 2: simple class name match (for short/partial class names) + doc_simple = doc_fqcn.rsplit('.', 1)[-1] + class_simple = class_name.rsplit('.', 1)[-1] + if doc_simple != class_simple: + continue + + source = doc.metadata['source'] + artifact_name = self.extract_maven_artifact(source) + + # If the library is provided make sure the doc belongs to library + if maven_gav and artifact_name in package_name: + return package_name, doc_fqcn + # If no library is provided look for the function + elif not maven_gav: + for val in self.tree_dict: + if artifact_name in val: + return val, doc_fqcn + + return package_name, class_name + + def extract_maven_artifact(self, path: str) -> str: + """ + Extract "artifactId:version" from a path like: + '.../hibernate-core-6.6.13.Final-sources/org/hibernate/type/.../ArrayJavaType.java' + -> 'org.hibernate:hibernate-core:6.6.13.Final' + + If the input is only 'org/hibernate/.../ArrayJavaType.java' (no artifact/version in the path), + returns "". + + """ + p = path.replace("\\", "/") + m = _GAV_DIR_RE.search(p) + if not m: + return "" # no artifact/version info in the path + + artifact, version = m.group(1), m.group(2) + + # Tail after the "-sources/" marker; use the first 2 dirs for groupId (e.g., "org.hibernate") + tail = p[m.end():] + pkg_dirs = tail.split("/") + # drop the filename + if pkg_dirs and pkg_dirs[-1].endswith(".java"): + pkg_dirs = pkg_dirs[:-1] + + if len(pkg_dirs) < 2: + return "" + + return f"{artifact}:{version}" \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/java_segmenters_with_methods.py b/src/exploit_iq_commons/utils/java_segmenters_with_methods.py new file mode 100644 index 000000000..c0dbf2d95 --- /dev/null +++ b/src/exploit_iq_commons/utils/java_segmenters_with_methods.py @@ -0,0 +1,3235 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import bisect +import contextvars +import os +import re +from dataclasses import dataclass, field + +from typing import List, Tuple, Optional, Dict, FrozenSet + +from langchain_community.document_loaders.parsers.language.java import JavaSegmenter + +_TEST_METHOD_ANNOTATIONS: FrozenSet[str] = frozenset({ + # JUnit 4/5 + "Test", + "ParameterizedTest", + "RepeatedTest", + "TestFactory", + "TestTemplate", + # TestNG + "org.testng.annotations.Test", # handled by simple-name "Test" too +}) + +_TEST_LIFECYCLE_ANNOTATIONS: FrozenSet[str] = frozenset({ + # JUnit 4 + "Before", + "After", + "BeforeClass", + "AfterClass", + # JUnit 5 + "BeforeEach", + "AfterEach", + "BeforeAll", + "AfterAll", + # TestNG + "BeforeMethod", + "AfterMethod", + "BeforeClass", + "AfterClass", + "BeforeSuite", + "AfterSuite", + "BeforeTest", + "AfterTest", + "BeforeGroups", + "AfterGroups", +}) + +# Class-level "this is a test class" markers (Quarkus + common Spring test slices) +_TEST_CLASS_ANNOTATIONS: FrozenSet[str] = frozenset({ + "QuarkusTest", + + # Spring / Spring Boot test slices (common ones) + "SpringBootTest", + "WebMvcTest", + "WebFluxTest", + "DataJpaTest", + "DataJdbcTest", + "DataMongoTest", + "JdbcTest", + "JooqTest", + "JsonTest", + "RestClientTest", + "GraphQlTest", + "ContextConfiguration", + "SpringJUnitConfig", + "SpringJUnitWebConfig", + "AutoConfigureMockMvc", + "AutoConfigureWebTestClient", + + # Older Spring JUnit 4 runners/config + "RunWith", # handled specially (needs SpringRunner) + "ExtendWith", # handled specially (needs SpringExtension) +}) + +# Scan only these chars; everything else can be skipped in big jumps. +# (We deliberately include '-' because we only care about "->".) +_LAMBDA_SCAN_SPECIAL_RE = re.compile(r'[/"\'-]') + +_pkg_imp_re = re.compile(r"(?m)^[ \t]*(?:package|import)\s+[^;\n]+;\s*\n") + +class JavaSegmenterWithMethods(JavaSegmenter): + + def __init__(self, code: str): + super().__init__(code) + + # ------------------------------ test filtering -------------------------------- + + def _looks_like_template_source(self, src: str) -> bool: + """ + Return True if the source contains template-language markers in *code*. + + This scanner ignores: + - Java // line comments + - Java /* ... */ block/Javadoc comments + - Java string literals "..." + - Java char literals '...' + - Java text blocks \"\"\" ... \"\"\" (Java 15+) + + Therefore, template markers that occur *only* inside strings/comments/text blocks + will NOT cause the file to be skipped. + """ + # Cheap prefilter + if "<" not in src and "$" not in src: + return False + + n = len(src) + i = 0 + + while i < n: + ch = src[i] + + # ---- comments ---- + if ch == '/' and i + 1 < n: + nxt = src[i + 1] + if nxt == '/': + # line comment + i = src.find('\n', i + 2) + if i == -1: + return False + i += 1 + continue + if nxt == '*': + # block/Javadoc comment + end = src.find('*/', i + 2) + if end == -1: + return False + i = end + 2 + continue + + # ---- text blocks / strings / chars ---- + if src.startswith('"""', i): + # text block + i += 3 + while i < n: + j = src.find('"""', i) + if j == -1: + return False + # if escaped \"\"\", keep scanning + k = j - 1 + backslashes = 0 + while k >= 0 and src[k] == '\\': + backslashes += 1 + k -= 1 + if (backslashes % 2) == 1: + i = j + 1 + continue + i = j + 3 + break + continue + + if ch == '"': + # regular string + i += 1 + while i < n: + if src[i] == '\\': + i += 2 + continue + if src[i] == '"': + i += 1 + break + i += 1 + continue + + if ch == "'": + # char literal + i += 1 + while i < n: + if src[i] == '\\': + i += 2 + continue + if src[i] == "'": + i += 1 + break + i += 1 + continue + + # ---- marker detection in code only ---- + # Check the handful of marker prefixes cheaply. + if ch == '$' and i + 1 < n and src[i + 1] == '{': + return True + if ch == '<' and i + 1 < n: + c1 = src[i + 1] + if c1 in ('#', '@'): + return True + if c1 == '/' and i + 2 < n and src[i + 2] == '#': + return True + + i += 1 + + return False + + def _read_qualified_ident_after_at(self, src: str, at_idx: int, end: int) -> Tuple[int, str, str]: + """ + Parse an annotation name starting at '@' (src[at_idx] == '@') within [at_idx, end). + + Returns + ------- + (next_idx, full_name, simple_name) + + - full_name can include dots: org.junit.jupiter.api.Test + - simple_name is the last segment: Test + """ + i = at_idx + 1 + n = min(end, len(src)) + + # Special-case annotation type declaration: '@interface' is not a "use" annotation. + if i + 8 <= n and src.startswith("interface", i): + return i + 9, "interface", "interface" + + # Read first identifier segment + if i >= n or not (_is_ident_start(src[i]) or src[i].isalpha()): + return at_idx + 1, "", "" + + start = i + i += 1 + while i < n and _is_ident_part(src[i]): + i += 1 + + # Read optional ".segment" repeats + while i < n and src[i] == '.': + dot = i + j = dot + 1 + if j < n and (_is_ident_start(src[j]) or src[j].isalpha()): + i = j + 1 + while i < n and _is_ident_part(src[i]): + i += 1 + else: + break + + full_name = src[start:i] + simple_name = full_name.rsplit(".", 1)[-1] + return i, full_name, simple_name + + def _source_is_test_source(self, src: str) -> bool: + """Heuristic test-source detector. + + Returns True iff `src` *appears* to be a test class/source. + + Detects: + - JUnit 4/5 + TestNG method/lifecycle annotations (e.g. @Test, @BeforeEach) + - JUnit3-style `extends TestCase` + - Quarkus / Spring / Spring Boot test classes via `_TEST_CLASS_ANNOTATIONS` + + Notes: + - Ignores tokens inside comments and string/text blocks. + - Designed to be fast: a cheap substring prefilter avoids scanning most sources. + """ + # Fast prefilter: + # - If there is no '@', the only supported positive is JUnit3-style `extends ... TestCase`. + # - If '@' exists, avoid scanning typical non-test sources by requiring at least one + # test-ish keyword to appear somewhere in the raw text. + if "@" not in src: + if "TestCase" not in src or "extends" not in src: + return False + else: + # Important: test-class annotations are often present even if no @Test is. + # `Test` is a broad but capitalized token that strongly correlates with test annotations. + if ("Test" not in src and + "RunWith" not in src and + "ExtendWith" not in src and + "ContextConfiguration" not in src and + "SpringJUnit" not in src and + "AutoConfigureMockMvc" not in src and + "AutoConfigureWebTestClient" not in src and + "org.junit" not in src and + "org.testng" not in src): + return False + + n = len(src) + i = 0 + saw_extends = False + + while i < n: + i = _skip_ws_comments(src, i) + if i >= n: + break + + ch = src[i] + + # Skip any string/char/text-block literal quickly. + if ch == '"' or ch == "'" or (ch == 'r' and i + 1 < n and src[i + 1] == '"'): + i = _skip_java_string_like(src, i) + continue + + if ch == '@': + # Parse annotation name (qualified or simple). + ni, _full, simple = self._read_qualified_ident_after_at(src, i, n) + + # Avoid misclassifying annotation type declarations: `public @interface Foo { ... }` + if simple == "interface": + i = ni + continue + + # JUnit/TestNG method-level / lifecycle annotations. + if simple in _TEST_METHOD_ANNOTATIONS or simple in _TEST_LIFECYCLE_ANNOTATIONS: + return True + + # Quarkus / Spring / Spring Boot class-level test annotations. + if simple in _TEST_CLASS_ANNOTATIONS: + # Only treat generic @ExtendWith/@RunWith as a "Spring test" if they bind + # the Spring extension/runner; otherwise they are too generic. + if simple == "ExtendWith": + j = _skip_ws_comments(src, ni) + if j < n and src[j] == '(': + close = _match_balanced_parens(src, j) + if close != -1 and "SpringExtension" in src[j:close]: + return True + i = ni + continue + + if simple == "RunWith": + j = _skip_ws_comments(src, ni) + if j < n and src[j] == '(': + close = _match_balanced_parens(src, j) + if close != -1 and "SpringRunner" in src[j:close]: + return True + i = ni + continue + + return True + + # Skip annotation argument list (if any) to avoid rescanning big nested blocks. + i = _skip_ws_comments(src, ni) + if i < n and src[i] == '(': + close = _match_balanced_parens(src, i) + i = (close + 1) if close != -1 else (i + 1) + continue + + # Detect JUnit3 style `extends ... TestCase` (ignoring comments/strings). + if _is_ident_start(ch): + j = i + 1 + while j < n and _is_ident_part(src[j]): + j += 1 + ident = src[i:j] + + if ident == "extends": + saw_extends = True + elif saw_extends: + if ident == "TestCase": + return True + saw_extends = False + + i = j + continue + + i += 1 + + return False + + def extract_functions_classes(self) -> List[str]: + """ + This method extracts all the golang methods and anonymous functions from a source code, and appends them to + the list of all java regular functions calculated in parent class' method. + :return: a list of all java functions/methods/anonymous functions ( each one with a complete implementation, + signature + body) + """ + if self._looks_like_template_source(self.code): + return [] + + if self._source_is_test_source(self.code): + return [] + + function_classes = super().extract_functions_classes() + methods = extract_methods(self.code) + inner_classes = extract_inner_classes(self.code) # TODO Add anonymous inner classes support + function_classes.extend(methods) + function_classes.extend(inner_classes) + + return function_classes + + def simplify_code(self) -> str: + opt_java_embedding = os.environ.get('OPTIMIZE_JAVA_FOR_EMBEDDING') + if opt_java_embedding: + return self.__original_simplify_code() + + return self.code + + def __original_simplify_code(self) -> str: + """ + Replace each top-level or nested type declaration body with a single + comment line of the form: + // Code for:
+ while preserving package/imports and any non-declaration code. + + Performance notes: + - Uses byte offsets from tree-sitter to avoid per-line per-node loops. + - Finds header via a single regex on the node slice (bytes). + - Records removal intervals (line ranges) and merges them once. + - Produces output with a single linear pass. + """ + import re + from bisect import bisect_right + + language = self.get_language() + query = language.query(self.get_chunk_query()) + + parser = self.get_parser() + # Work with bytes once + code_str = self.code + code_bytes = code_str.encode("utf-8", "surrogatepass") + tree = parser.parse(code_bytes) + + # Precompute newline byte indices for fast byte->line mapping + # lines: 0..n-1; newline_indices holds byte offsets of '\n' + newline_indices = [] + for i, ch in enumerate(code_bytes): + if ch == 10: # b'\n' + newline_indices.append(i) + n_lines = len(self.source_lines) + + def byte_to_line(b: int) -> int: + # Returns 0-based line number for byte offset b + # Count how many '\n' are before b + return bisect_right(newline_indices, b - 1) + + # Header matcher on BYTES (handles @interface and the keywords) + header_re = re.compile(rb"\b(class|interface|enum|record|module)\b|@interface") + + # We'll collect: + # - header_map: line -> replacement string (comment line) + # - intervals_to_remove: list of (start_line, end_line) inclusive, not touching header_line + header_map = {} + intervals_to_remove = [] + + # Iterate captures once; sort by start byte to favor outer declarations first + captures = query.captures(tree.root_node) + captures.sort(key=lambda cap: cap[0].start_byte) + + for node, _ in captures: + sb, eb = node.start_byte, node.end_byte + + # Search header keyword within the node range (first small window, then full if needed) + # Small window optimized to catch typical cases quickly + SMALL = 4096 # 4KB window is enough for very long headers + slice1 = code_bytes[sb:min(eb, sb + SMALL)] + m = header_re.search(slice1) + if not m and eb - sb > SMALL: + # Fallback: search the rest + m = header_re.search(code_bytes[sb + SMALL:eb]) + offset = SMALL if m else 0 + else: + offset = 0 + + if not m: + # Shouldn't happen; fall back to node.start_point line + start_line = node.start_point[0] + end_line = node.end_point[0] + header_line = start_line + else: + header_byte = sb + offset + m.start() + header_line = byte_to_line(header_byte) + start_line = node.start_point[0] + end_line = node.end_point[0] + + # Determine the header line text to echo in the comment + # Prefer the exact header line; fallback to the node's first line if empty + if 0 <= header_line < n_lines: + header_text = self.source_lines[header_line].rstrip() + else: + header_text = "" + + if not header_text and 0 <= start_line < n_lines: + header_text = self.source_lines[start_line].rstrip() + + header_map.setdefault( + header_line, self.make_line_comment(f"Code for: {header_text}") + ) + + # Add removal intervals for everything EXCEPT the header line + if start_line <= end_line: + if start_line <= header_line - 1: + intervals_to_remove.append((start_line, header_line - 1)) + if header_line + 1 <= end_line: + intervals_to_remove.append((header_line + 1, end_line)) + + # Merge intervals to minimize membership checks + if intervals_to_remove: + intervals_to_remove.sort() + merged = [] + cs, ce = intervals_to_remove[0] + for s, e in intervals_to_remove[1:]: + if s <= ce + 1: + ce = max(ce, e) + else: + merged.append((cs, ce)) + cs, ce = s, e + merged.append((cs, ce)) + intervals_to_remove = merged + + # Build the output in a single pass over lines: + # skip any line that falls inside a removal interval; replace header lines from header_map + out_lines = [] + it = iter(intervals_to_remove) + try: + cur = next(it) + except StopIteration: + cur = None + + for ln in range(n_lines): + # Advance intervals until current could include ln + while cur is not None and cur[1] < ln: + try: + cur = next(it) + except StopIteration: + cur = None + break + + # If ln is in a removal interval -> skip + if cur is not None and cur[0] <= ln <= cur[1]: + continue + + # Replace header line if present; else original line + repl = header_map.get(ln) + if repl is not None: + out_lines.append(repl) + else: + out_lines.append(self.source_lines[ln]) + + return "\n".join(out_lines) + + def get_chunk_query(self) -> str: + return CHUNK_QUERY + +CHUNK_QUERY = """ + [ + (class_declaration) @class + (interface_declaration) @interface + (enum_declaration) @enum + (record_declaration) @record + (annotation_type_declaration) @ann_interface + (module_declaration) @module + ] +""".strip() + +_NON_METHOD_TOKENS = { + "if", "for", "while", "switch", "catch", "try", "synchronized", + "assert", "return", "throw", "this", "super", "new", "case", "default" +} + +def _is_ident_start(ch: str) -> bool: + return ch == '_' or ch == '$' or ('A' <= ch <= 'Z') or ('a' <= ch <= 'z') + +def _is_ident_part(ch: str) -> bool: + return _is_ident_start(ch) or ('0' <= ch <= '9') + +# ------------------------- whitespace & comment skipper ----------------------- + +def _strip_body_comments_and_annotations(src: str, lbrace: int, rbrace: int) -> str: + """ + Return the body substring src[lbrace:rbrace+1] with // and /*...*/ comments + and Java annotations (e.g., @SuppressWarnings(...), @Nullable, @A.B) removed. + Strings/chars/text blocks are preserved verbatim. Keeps '@interface' intact. + Inserts a single space when removal would glue tokens together. + """ + out: List[str] = [] + i = lbrace + n = min(rbrace + 1, len(src)) + + def last_char() -> str: + if not out: + return '' + s = out[-1] + return s[-1] if s else '' + + while i < n: + ch = src[i] + + # --- comments --- + if ch == '/' and i + 1 < n and src[i + 1] == '/': + # line comment: drop but preserve newline if present + i += 2 + j = src.find('\n', i, n) + i = n if j == -1 else j + if i < n and src[i] == '\n': + out.append('\n') + continue + if ch == '/' and i + 1 < n and src[i + 1] == '*': + # block / Javadoc comment + j = src.find('*/', i + 2, n) + if j == -1: + break + prev = last_char() + nxt = src[j + 2] if j + 2 < n else '' + if (prev and (prev.isalnum() or prev in '$_)[]>') + and (nxt and (nxt.isalnum() or nxt in '$_(<['))): + out.append(' ') + i = j + 2 + continue + + # --- literals: copy verbatim --- + if src.startswith('"""', i) or ch in ('"', "'"): + j = _skip_java_string_like(src, i) + out.append(src[i:j]) + i = j + continue + + # --- annotations --- + if ch == '@': + # Keep annotation type declarations: '@interface' + if src.startswith('@interface', i): + out.append('@interface') + i += len('@interface') + continue + + prev = last_char() + i += 1 # skip '@' + + # Qualified annotation name: ident ('.' ident)* + if i < n and _is_ident_start(src[i]): + i += 1 + while i < n and _is_ident_part(src[i]): i += 1 + while i < n and src[i] == '.': + j = i + 1 + if j < n and _is_ident_start(src[j]): + i = j + 1 + while i < n and _is_ident_part(src[i]): i += 1 + else: + break + + # optional ws/comments then optional "( ... )" + i = _skip_ws_comments(src, i) + if i < n and src[i] == '(': + i = _match_balanced_parens(src, i) + 1 + + # spacer to avoid token gluing + nxt = src[i] if i < n else '' + if (prev and (prev.isalnum() or prev in '$_)>]') + and (nxt and (nxt.isalnum() or nxt in '$_(<['))): + out.append(' ') + continue + + # default: copy + out.append(ch) + i += 1 + + return ''.join(out) + +def _strip_annotations_in_range(src: str, start: int, end: int) -> str: + """ + Return src[start:end] with Java annotations removed: + - '@Qualified.Name' + - '@Name(...)' (skips balanced parentheses; strings inside respected) + - multiple/dotted names (e.g., @a.b.C) + Works on any header slice (e.g., from the first modifier to the '{'). + Inserts a single space when removal would glue tokens together. + """ + out: List[str] = [] + i = start + n = min(end, len(src)) + + def last_char() -> str: + if not out: + return '' + s = out[-1] + return s[-1] if s else '' + + while i < n: + ch = src[i] + + if ch == '@': + # Remember the previous emitted char to decide if we need a spacer + prev = last_char() + + i += 1 # skip '@' + + # Qualified annotation name: ident ('.' ident)* + if i < n and (_is_ident_start(src[i])): + i += 1 + while i < n and _is_ident_part(src[i]): + i += 1 + while i < n and src[i] == '.': + j = i + 1 + if j < n and _is_ident_start(src[j]): + i = j + 1 + while i < n and _is_ident_part(src[i]): + i += 1 + else: + break + + # Optional whitespace/comments before arguments + i = _skip_ws_comments(src, i) + + # Optional "( ... )" with balanced parens (may contain strings/braces) + if i < n and src[i] == '(': + i = _match_balanced_parens(src, i) + 1 + + # If removing glued two tokens together (e.g., 'T,@A U' -> 'T U'), pad one space + nxt = src[i] if i < n else '' + if (prev and (prev.isalnum() or prev in '$_)>]') and + (nxt and (nxt.isalnum() or nxt in '$_(<['))): + out.append(' ') + continue + + # Copy through normally (comments/strings can remain here; we strip comments later) + out.append(ch) + i += 1 + + return ''.join(out) + +def _strip_comments_keep_strings_range(src: str, start: int, end: int) -> str: + """ + Return src[start:end] with // and /*...*/ comments removed, preserving + string/char literals and text blocks. Inserts a single space if removing + a block comment would glue two tokens together. + """ + out: List[str] = [] + i = start + n = min(end, len(src)) + + def last_char() -> str: + if not out: return '' + s = out[-1] + return s[-1] if s else '' + + while i < n: + ch = src[i] + + # line comment + if ch == '/' and i + 1 < n and src[i + 1] == '/': + i += 2 + j = src.find('\n', i, n) + i = n if j == -1 else j # keep newline (added below if present) + if i < n and src[i] == '\n': + out.append('\n') + continue + + # block/Javadoc comment + if ch == '/' and i + 1 < n and src[i + 1] == '*': + j = src.find('*/', i + 2, n) + if j == -1: + # unterminated: drop rest defensively + break + prev = last_char() + nxt = src[j + 2] if j + 2 < n else '' + if (prev and (prev.isalnum() or prev in '$_)[]>') + and (nxt and (nxt.isalnum() or nxt in '$_(<['))): + out.append(' ') + i = j + 2 + continue + + # literals: copy verbatim + if src.startswith('"""', i) or ch in ('"', "'"): + j = _skip_java_string_like(src, i) + out.append(src[i:j]) + i = j + continue + + out.append(ch) + i += 1 + + return ''.join(out) + +def _skip_ws_comments(src: str, i: int) -> int: + """Skip ASCII whitespace and //... or /*...*/ comments.""" + n = len(src) + while i < n: + ch = src[i] + # whitespace + if ch in (' ', '\t', '\r', '\n', '\f'): + i += 1 + continue + # line comment + if i + 1 < n and ch == '/' and src[i + 1] == '/': + i += 2 + nl = src.find('\n', i) + i = n if nl == -1 else nl + 1 + continue + # block comment (incl. Javadoc) + if i + 1 < n and ch == '/' and src[i + 1] == '*': + j = src.find('*/', i + 2) + i = n if j == -1 else j + 2 + continue + break + return i + +# --------------------------- string / char / text blocks ---------------------- + +def _skip_java_text_block(src: str, i: int) -> int: + # src[i:i+3] == '"""' (Java text block) + n = len(src) + i += 3 + while i < n: + j = src.find('"""', i) + if j == -1: + return n + # if escaped \"\"\", keep scanning + k = j - 1 + backslashes = 0 + while k >= 0 and src[k] == '\\': + backslashes += 1 + k -= 1 + if (backslashes % 2) == 1: + i = j + 1 + continue + return j + 3 + return n + +def _skip_java_string(src: str, i: int) -> int: + # src[i] == '"' + n = len(src) + i += 1 + while i < n: + ch = src[i] + if ch == '\\': + i += 2 + elif ch == '"': + return i + 1 + else: + i += 1 + return n + +def _skip_java_char(src: str, i: int) -> int: + # src[i] == '\'' + n = len(src) + i += 1 + while i < n: + ch = src[i] + if ch == '\\': + i += 2 + elif ch == "'": + return i + 1 + else: + i += 1 + return n + +def _skip_java_string_like(src: str, i: int) -> int: + if src.startswith('"""', i): + return _skip_java_text_block(src, i) + if src[i] == '"': + return _skip_java_string(src, i) + if src[i] == "'": + return _skip_java_char(src, i) + return i + +# --------------- delimiter & ignored-span index (one-pass pre-scan) ---------- + +@dataclass +class _DelimCache: + """Per-task delimiter index cache. Stored in a ContextVar so concurrent + asyncio tasks each get their own isolated copy.""" + par_o2c: Dict[int, int] = field(default_factory=dict) + par_c2o: Dict[int, int] = field(default_factory=dict) + br_o2c: Dict[int, int] = field(default_factory=dict) + br_c2o: Dict[int, int] = field(default_factory=dict) + ign_spans: List[Tuple[int, int]] = field(default_factory=list) + ign_starts: List[int] = field(default_factory=list) + ign_ends: List[int] = field(default_factory=list) + src_id: Optional[int] = None + +_delim_cache_var: contextvars.ContextVar[_DelimCache] = contextvars.ContextVar('_delim_cache_var') + +def _get_delim_cache() -> _DelimCache: + try: + return _delim_cache_var.get() + except LookupError: + c = _DelimCache() + _delim_cache_var.set(c) + return c + +def _ignored_span_at(pos: int) -> Optional[Tuple[int, int]]: + """Return (start,end) ignored span containing pos, or None.""" + c = _get_delim_cache() + if not c.ign_spans: + return None + idx = bisect.bisect_right(c.ign_starts, pos) - 1 + if idx >= 0 and c.ign_starts[idx] <= pos < c.ign_ends[idx]: + return c.ign_spans[idx] + return None + +def _build_delim_index(source: str) -> Tuple[ + Dict[int, int], Dict[int, int], Dict[int, int], Dict[int, int], List[Tuple[int, int]] +]: + """ + Scan `source` once to build delimiter maps and a list of ignored spans. + + Returns + ------- + ( + paren_open_to_close : Dict[int, int], + paren_close_to_open : Dict[int, int], + brace_open_to_close : Dict[int, int], + brace_close_to_open : Dict[int, int], + ignored_spans : List[Tuple[int, int]], + ) + + Where: + - paren_open_to_close maps index of '(' to the index of its matching ')'. + - paren_close_to_open maps index of ')' to the index of its matching '('. + - brace_open_to_close maps index of '{' to the index of its matching '}'. + - brace_close_to_open maps index of '}' to the index of its matching '{'. + - ignored_spans is a list of (start, end) slices for comments and literals + (end is exclusive) covering: + * // line comments + * /* ... */ block/Javadoc comments + * "..." strings, '...' chars, and \"\"\"...\"\"\" text blocks + + Notes + ----- + - Delimiter pairs inside ignored spans are not tracked (they're skipped). + - Runs in O(len(source)) time; memory proportional to number of delimiters/spans. + """ + paren_open_to_close: Dict[int, int] = {} + paren_close_to_open: Dict[int, int] = {} + brace_open_to_close: Dict[int, int] = {} + brace_close_to_open: Dict[int, int] = {} + + ignored_spans: List[Tuple[int, int]] = [] + paren_stack: List[int] = [] + brace_stack: List[int] = [] + + pos, length = 0, len(source) + while pos < length: + ch = source[pos] + + # Whitespace: skip quickly + if ch in (' ', '\t', '\r', '\n', '\f'): + pos += 1 + continue + + # // line comment + if pos + 1 < length and ch == '/' and source[pos + 1] == '/': + span_start = pos + pos += 2 + newline_idx = source.find('\n', pos) + pos = length if newline_idx == -1 else newline_idx + 1 + ignored_spans.append((span_start, pos)) + continue + + # /* block/Javadoc */ comment + if pos + 1 < length and ch == '/' and source[pos + 1] == '*': + span_start = pos + close_idx = source.find('*/', pos + 2) + pos = length if close_idx == -1 else close_idx + 2 + ignored_spans.append((span_start, pos)) + continue + + # Strings / chars / text blocks + if source.startswith('"""', pos) or ch in ('"', "'"): + span_start = pos + pos = _skip_java_string_like(source, pos) + ignored_spans.append((span_start, pos)) + continue + + # Delimiters outside ignored spans + if ch == '(': + paren_stack.append(pos) + elif ch == ')' and paren_stack: + open_idx = paren_stack.pop() + paren_open_to_close[open_idx] = pos + paren_close_to_open[pos] = open_idx + elif ch == '{': + brace_stack.append(pos) + elif ch == '}' and brace_stack: + open_idx = brace_stack.pop() + brace_open_to_close[open_idx] = pos + brace_close_to_open[pos] = open_idx + + pos += 1 + + ignored_spans.sort(key=lambda span: span[0]) + return ( + paren_open_to_close, + paren_close_to_open, + brace_open_to_close, + brace_close_to_open, + ignored_spans, + ) + +def _populate_delim_index_cache(java_source: str): + """ + Build and cache delimiter/ignored-span indexes for a given Java source string. + + Stores the result in a per-task ContextVar so concurrent asyncio tasks + each get their own isolated cache. + """ + par_o2c, par_c2o, br_o2c, br_c2o, spans = _build_delim_index(java_source) + + c = _get_delim_cache() + c.par_o2c = par_o2c + c.par_c2o = par_c2o + c.br_o2c = br_o2c + c.br_c2o = br_c2o + c.ign_spans = spans + c.ign_starts = [a for (a, _) in spans] + c.ign_ends = [b for (_, b) in spans] + c.src_id = id(java_source) + +def _clear_delim_index_cache(): + c = _get_delim_cache() + c.par_o2c.clear(); c.par_c2o.clear(); c.br_o2c.clear(); c.br_c2o.clear() + c.ign_spans.clear(); c.ign_starts.clear(); c.ign_ends.clear() + c.src_id = None + +# -------------------------- forward balanced matchers ------------------------- + +def _match_balanced_angles(source: str, open_angle_index: int) -> int: + """ + Return the index of the '>' that matches the '<' at `open_angle_index`. + + Scans forward from just after the opening '<', tracking nested angle pairs + (e.g., `Map>`). Whitespace and comments are skipped via + `_skip_ws_comments`, and string/char/text-block literals are copied over with + `_skip_java_string_like` so any '<' or '>' inside them are ignored. + + Parameters + ---------- + source : str + The Java source code to scan. + open_angle_index : int + Index in `source` where the opening '<' resides. The caller should ensure + `source[open_angle_index] == '<'`. + + Returns + ------- + int + The index of the matching closing '>' if found; otherwise `len(source) - 1` + (a defensive fallback for unterminated input). + + Notes + ----- + - This routine treats angle brackets as generic-type delimiters. It does not try + to disambiguate comparison operators vs generics; the caller should invoke it + only in contexts where a type parameter list is expected. + - Runs in O(N) from the opening bracket to the end (or until the match is found). + """ + length = len(source) + nesting = 1 # we've consumed the first '<' + i = open_angle_index + 1 + + while i < length: + next_i = _skip_ws_comments(source, i) + if next_i != i: + i = next_i + continue + + # Skip over literals so '<' or '>' inside them don't affect nesting. + if source.startswith('"""', i) or source[i] in ('"', "'"): + i = _skip_java_string_like(source, i) + continue + + char = source[i] + if char == '<': + nesting += 1 + elif char == '>': + nesting -= 1 + if nesting == 0: + return i + i += 1 + + # Fallback: unterminated angle list + return length - 1 + +def _match_balanced_parens(source: str, open_paren_index: int) -> int: + """ + Return the index of the ')' that matches the '(' at `open_paren_index`. + + Fast path: if a delimiter index was precomputed for this exact `source` + (see `_set_delim_index`), use the cached mapping for O(1) lookup. + + Otherwise, scan forward from just after the opening '(' while: + - Skipping whitespace and comments via `_skip_ws_comments`. + - Skipping over string/char/text-block literals via `_skip_java_string_like`. + - Tracking nested parentheses depth. + + Parameters + ---------- + source : str + The Java source to scan. + open_paren_index : int + Index in `source` where the opening '(' resides. Caller must ensure + `source[open_paren_index] == '('`. + + Returns + ------- + int + Index of the matching ')' if found; otherwise `len(source) - 1` + as a defensive fallback for unterminated input. + + Notes + ----- + - Time complexity is O(K) where K is the distance from `open_paren_index` + to the matching ')', or to the end if unterminated. + - The cache is used only when the delim cache was built for this exact source object. + """ + # Cached O(1) lookup when indices were prebuilt for this exact source object. + _dc = _get_delim_cache() + if _dc.src_id == id(source): + cached_close = _dc.par_o2c.get(open_paren_index) + if cached_close is not None: + return cached_close + + # Fallback: linear scan with nesting. + length = len(source) + nesting = 1 # we've consumed the first '(' + i = open_paren_index + 1 + + while i < length: + next_i = _skip_ws_comments(source, i) + if next_i != i: + i = next_i + continue + + # Skip over literals so any '(' or ')' inside them don't affect nesting. + if source.startswith('"""', i) or source[i] in ('"', "'"): + i = _skip_java_string_like(source, i) + continue + + ch = source[i] + if ch == '(': + nesting += 1 + elif ch == ')': + nesting -= 1 + if nesting == 0: + return i + i += 1 + + # Unterminated parentheses list – return last valid index defensively. + return length - 1 + +def _match_balanced_braces(source: str, open_brace_index: int) -> int: + """ + Return the index of the '}' that matches the '{' at `open_brace_index`. + + Fast path: + If delimiter indexes were precomputed for this exact `source` + via `_set_delim_index`, use the cached brace map for O(1) lookup. + + Fallback scan: + Starting just after the opening '{', scan forward while: + - Skipping whitespace and comments with `_skip_ws_comments`. + - Skipping over string/char/text-block literals with `_skip_java_string_like`. + - Tracking nested braces `{ ... }` with a depth counter. + - When encountering '(', skip the entire parenthesized section with + `_match_balanced_parens` to avoid confusing braces that may appear inside. + + Parameters + ---------- + source : str + The Java source text to scan. + open_brace_index : int + Index in `source` where the opening '{' resides. + Caller must ensure `source[open_brace_index] == '{'`. + + Returns + ------- + int + Index of the matching '}' if found; otherwise `len(source) - 1` + as a defensive fallback for unterminated input. + + Notes + ----- + - This function does not treat '<' specially inside bodies (to avoid + misinterpreting comparisons as generics). + - Time complexity is O(K), where K is the distance from the opening brace + to its match (or to the end for unterminated input). + """ + # Cached O(1) close-brace lookup if indexes were built for this exact source object. + _dc = _get_delim_cache() + if _dc.src_id == id(source): + cached_close = _dc.br_o2c.get(open_brace_index) + if cached_close is not None: + return cached_close + + # Fallback: linear scan with nesting depth. + length = len(source) + nesting = 1 # we've consumed the first '{' + i = open_brace_index + 1 + + while i < length: + next_i = _skip_ws_comments(source, i) + if next_i != i: + i = next_i + continue + + # Skip over literals so braces inside them don't affect nesting. + if source.startswith('"""', i) or source[i] in ('"', "'"): + i = _skip_java_string_like(source, i) + continue + + ch = source[i] + if ch == '{': + nesting += 1 + elif ch == '}': + nesting -= 1 + if nesting == 0: + return i + elif ch == '(': + # Skip an entire (...) block to avoid miscounting braces seen within. + i = _match_balanced_parens(source, i) + + # NOTE: do not treat '<' specially inside bodies (comparisons). + i += 1 + + # Unterminated block – return last valid index defensively. + return length - 1 + +# ------------------------------ backward helpers ------------------------------ + +def _strip_java_noise(snippet: str) -> str: + """ + Remove comments, javadocs, annotations, and (defensively) package/import lines. + Designed to be fast: cheap substring checks first, heavy work only if needed. + """ + if not snippet: + return snippet + + # Fast path: almost all snippets do NOT need stripping. + if ( + "/*" not in snippet + and "//" not in snippet + and "@" not in snippet + and "package" not in snippet + and "import" not in snippet + ): + return snippet + + # Reuse existing (previously unused) stripper: removes comments + annotations safely. + snippet = _strip_body_comments_and_annotations(snippet, 0, len(snippet)) + + # Remove package/import lines if present (defensive; inner-class extraction used to prepend imports). + if "package" in snippet or "import" in snippet: + snippet = _pkg_imp_re.sub("", snippet) + + return snippet + +def _match_paren_backwards(source: str, close_paren_index: int) -> int: + """ + Return the index of the '(' that matches the ')' at `close_paren_index`. + + Fast path + --------- + If delimiter indexes were precomputed for this exact `source` (see + `_set_delim_index`), use the cached map `_PAR_C2O` for O(1) lookup. + + Fallback scan + ------------- + Walk left from just before `close_paren_index` while: + - Tracking nested parentheses depth. + - Skipping over string and char literals (so any parentheses inside them + don't affect nesting). + + Parameters + ---------- + source : str + The Java source text to scan. + close_paren_index : int + Index in `source` where the closing ')' resides. Caller must ensure + `source[close_paren_index] == ')'`. + + Returns + ------- + int + Index of the matching '(' if found; otherwise 0 as a defensive fallback. + + Notes + ----- + - Time complexity is O(K), where K is the distance from the closing paren + to its match (or to the start if unterminated). + - The cache is used only when the delim cache was built for this exact source object. + """ + # Cached O(1) lookup when indices were built for this exact source object. + _dc = _get_delim_cache() + if _dc.src_id == id(source): + cached_open = _dc.par_c2o.get(close_paren_index) + if cached_open is not None: + return cached_open + + # Fallback: linear backward scan with nesting. + nesting = 1 # we've seen one ')' + i = close_paren_index - 1 + + while i >= 0: + ch = source[i] + + if ch == ')': + nesting += 1 + elif ch == '(': + nesting -= 1 + if nesting == 0: + return i + elif ch in ('"', "'"): + # Skip over a string/char literal going left, handling escapes. + quote = ch + i -= 1 + while i >= 0: + if source[i] == '\\': + i -= 2 # skip the escape and the escaped char + continue + if source[i] == quote: + i -= 1 + break + i -= 1 + continue + + i -= 1 + + # Unterminated or not found – safest fallback. + return 0 + +# ------------------------------- throws skipper ------------------------------- + +def _skip_throws_clause(src: str, i: int) -> int: + n = len(src) + i = _skip_ws_comments(src, i) + if not src.startswith('throws', i): + return i + i += len('throws') + while i < n: + i = _skip_ws_comments(src, i) + if i >= n: + break + if src.startswith('"""', i) or src[i] in ('"', "'"): + i = _skip_java_string_like(src, i); continue + ch = src[i] + if ch == '<': + i = _match_balanced_angles(src, i) + 1; continue + if ch == '(': + i = _match_balanced_parens(src, i) + 1; continue + if ch == '[': + # zero or more [] after a type + while i < n and src[i] == '[': + rb = src.find(']', i + 1) + if rb == -1: return n + i = rb + 1 + continue + if ch == '@': + # annotation on thrown type + i += 1 + while i < n and (src[i].isalnum() or src[i] in ('_', '$', '.')): + i += 1 + i = _skip_ws_comments(src, i) + if i < n and src[i] == '(': + i = _match_balanced_parens(src, i) + 1 + continue + if ch == ',': + i += 1; continue + if ch in ('{', ';'): + break + i += 1 + return _skip_ws_comments(src, i) + +# --------------------------------- lambdas ------------------------------------ + +def _capture_lambda_at_arrow(src: str, arrow: int) -> Optional[Tuple[int, int]]: + """ + Given '->' at src[arrow:arrow+2], return (start, end) slice indices of the + lambda expression WITHOUT trailing delimiters. + """ + n = len(src) + # body start + j = _skip_ws_comments(src, arrow + 2) + # find parameter start + p = arrow - 1 + while p >= 0 and src[p] in ' \t\r\n\f': + p -= 1 + if p < 0: + return None + if src[p] == ')': + param_start = _match_paren_backwards(src, p) + else: + # single identifier parameter + end = p + 1 + start = p + while start >= 0 and _is_ident_part(src[start]): + start -= 1 + start += 1 + if start >= end or not _is_ident_start(src[start]): + return None + param_start = start + + # BODY + if j < n and src[j] == '{': + end = _match_balanced_braces(src, j) + 1 + return (param_start, end) + + # expression-bodied: consume until delimiter at depth 0 + depth_p = depth_b = depth_c = 0 + k = j + while k < n: + k2 = _skip_ws_comments(src, k) + if k2 != k: + k = k2; continue + if k < n and (src.startswith('"""', k) or src[k] in ('"', "'")): + k = _skip_java_string_like(src, k); continue + ch = src[k] + if ch == '(': + depth_p += 1 + elif ch == ')': + if depth_p > 0: + depth_p -= 1 + else: + break + elif ch == '[': + depth_b += 1 + elif ch == ']': + if depth_b > 0: + depth_b -= 1 + elif ch == '{': + depth_c += 1 + elif ch == '}': + if depth_c == 0 and depth_p == 0 and depth_b == 0: + break + if depth_c > 0: + depth_c -= 1 + elif ch in (';', ','): + if depth_p == 0 and depth_b == 0 and depth_c == 0: + break + k += 1 + end = k + return (param_start, end) + +def _extract_lambdas_in_range(src: str, start: int, end: int) -> List[str]: + """ + Extract Java lambda expressions within [start, end). + + Optimized for huge files: + - Fast early-out if there's no '->' at all in the range. + - Uses a regex "find next interesting char" jump-table so we don't + call helpers per-character. + + Semantics are intended to match the previous implementation: + - Skip // and /*...*/ comments + - Skip strings/chars/text blocks via _skip_java_string_like() + - Capture lambdas via _capture_lambda_at_arrow() + - Strip comments in the captured lambda via _strip_comments_keep_strings_range() + """ + out: List[str] = [] + n = len(src) + if start < 0: + start = 0 + if end > n: + end = n + if start >= end: + return out + + # Cheap win for big files with no lambdas at all (like many autogenerated sources): + if src.find("->", start, end) == -1: + return out + + i = start + while True: + m = _LAMBDA_SCAN_SPECIAL_RE.search(src, i, end) + if not m: + break + + i = m.start() + ch = src[i] + + # Comments + if ch == '/': + if i + 1 < end: + nxt = src[i + 1] + # line comment + if nxt == '/': + nl = src.find('\n', i + 2, end) + i = end if nl == -1 else nl + 1 + continue + # block / javadoc comment + if nxt == '*': + j = src.find('*/', i + 2, end) + i = end if j == -1 else j + 2 + continue + i += 1 + continue + + # Strings / chars / text blocks + if ch in ('"', "'"): + i = _skip_java_string_like(src, i) + continue + + # Potential lambda arrow + if ch == '-': + if i + 1 < end and src[i + 1] == '>': + span = _capture_lambda_at_arrow(src, i) + if span: + s, e = span + if s >= start and e <= end: + clean = src[s:e] + clean = _strip_java_noise(clean) + out.append(clean) + i = e + continue + i += 2 + continue + + i += 1 + continue + + # Fallback (should be unreachable due to regex character class) + i += 1 + + return out + +# ------------------------------- core extractor ------------------------------- + +def _extract_methods_anywhere( + src: str, + start: int = 0, + end: int | None = None, + types: list | None = None, +) -> list[str]: + """ + Extract Java method definitions (with bodies) anywhere within [start, end), + INCLUDING constructors. Anonymous/inner/local class bodies are scanned. + + Key behavior: + - For named inner types, emitted comment includes ALL nesting levels + EXCLUDING the top-level declaring type: + e.g. "Outer.Inner1.Inner2" -> "Inner1.Inner2", + "CreateMultipartUploadRequest.BuilderImpl" -> "BuilderImpl". + + Annotation behavior: + - Lines that contain ONLY annotations (and whitespace/comments) immediately + preceding a member are excluded from the returned slice. + + Enum behavior (critical fix): + - Do NOT treat enum constant initializers as class members. + Method parsing inside a named enum body is disabled until the FIRST ';' + at that enum body's brace depth (the end of the enum-constant section). + + Notes on performance: + - Single forward scan. + - Balanced matchers are used only for candidates at class-member level. + - No regex usage in hot paths. + """ + + # ------------------------- + # Small internal structs + # ------------------------- + class _TypeRegion: + __slots__ = ("kind", "name", "start", "end", "body_start", "body_end", "parent", "qual_name") + + def __init__( + self, + kind: str, + name: str, + start: int, + end: int, + body_start: int, + body_end: int, + parent: int, + qual_name: str, + ) -> None: + self.kind = kind + self.name = name + self.start = start + self.end = end + self.body_start = body_start + self.body_end = body_end + self.parent = parent + self.qual_name = qual_name + + class _ClassBodyFrame: + __slots__ = ("named_type_idx", "body_depth", "member_start", + "is_enum", "enum_constants_done") + + def __init__(self, named_type_idx: int, body_depth: int, member_start: int, is_enum: bool) -> None: + self.named_type_idx = named_type_idx # >=0 for named type, -1 for anonymous class body + self.body_depth = body_depth # brace_depth inside this body + self.member_start = member_start # where next member starts + + self.is_enum = is_enum + # For enums: constants come first. Members start only after the first ';' at body depth. + self.enum_constants_done = (not is_enum) + + # ------------------------- + # Low-level helpers (all local) + # ------------------------- + def _is_id_start(c: str) -> bool: + return c.isalpha() or c == "_" or c == "$" + + def _is_id_part(c: str) -> bool: + return c.isalnum() or c == "_" or c == "$" + + def _skip_ws_and_comments(s: str, i: int, lim: int) -> int: + while i < lim: + c = s[i] + if c.isspace(): + i += 1 + continue + if c == "/" and i + 1 < lim: + n2 = s[i + 1] + if n2 == "/": # line comment + i += 2 + while i < lim and s[i] != "\n": + i += 1 + continue + if n2 == "*": # block comment + j = s.find("*/", i + 2, lim) + i = lim if j == -1 else (j + 2) + continue + break + return i + + def _signature_has_initializer_assignment(sig_start: int, name_start: int) -> bool: + """ + True if there's an '=' before `name_start` that is NOT inside an annotation's (...) args + and not inside comments/strings. + + This is used to reject parsing '(' that belongs to field initializers like: + Foo X = new Foo(...) { ... } + """ + # Fast path: no '=' at all + if s.find("=", sig_start, name_start) == -1: + return False + + NORMAL3, LINE3, BLOCK3, STRING3, CHAR3, TEXT3 = 0, 1, 2, 3, 4, 5 + st = NORMAL3 + i3 = sig_start + while i3 < name_start: + c3 = s[i3] + + if st == NORMAL3: + if c3 == "/" and i3 + 1 < name_start: + n3 = s[i3 + 1] + if n3 == "/": + st = LINE3 + i3 += 2 + continue + if n3 == "*": + st = BLOCK3 + i3 += 2 + continue + + if c3 == '"': + if i3 + 2 < name_start and s[i3:i3 + 3] == '"""': + st = TEXT3 + i3 += 3 + continue + st = STRING3 + i3 += 1 + continue + + if c3 == "'": + st = CHAR3 + i3 += 1 + continue + + # Skip an annotation (and its (...) if present) so '=' inside it doesn't count. + if c3 == "@": + i3 += 1 + # qualified annotation name + while i3 < name_start and (_is_id_part(s[i3]) or s[i3] == "."): + i3 += 1 + i3 = _skip_ws_and_comments(s, i3, name_start) + if i3 < name_start and s[i3] == "(": + r3 = _match_balanced_paren(s, i3, name_start) + if r3 == -1: + # malformed; be conservative (don't reject) + return False + i3 = r3 + 1 + continue + + if c3 == "=": + return True + + i3 += 1 + continue + + if st == LINE3: + if c3 == "\n": + st = NORMAL3 + i3 += 1 + continue + + if st == BLOCK3: + if c3 == "*" and i3 + 1 < name_start and s[i3 + 1] == "/": + st = NORMAL3 + i3 += 2 + continue + i3 += 1 + continue + + if st == STRING3: + if c3 == "\\": + i3 += 2 + continue + if c3 == '"': + st = NORMAL3 + i3 += 1 + continue + + if st == CHAR3: + if c3 == "\\": + i3 += 2 + continue + if c3 == "'": + st = NORMAL3 + i3 += 1 + continue + + # TEXT3 + if i3 + 2 < name_start and s[i3:i3 + 3] == '"""': + st = NORMAL3 + i3 += 3 + continue + i3 += 1 + + return False + + def _match_balanced_paren(s: str, lparen: int, lim: int) -> int: + NORMAL, LINE, BLOCK, STRING, CHAR, TEXT = 0, 1, 2, 3, 4, 5 + state = NORMAL + i = lparen + 1 + depth = 1 + + while i < lim: + c = s[i] + if state == NORMAL: + if c == "/" and i + 1 < lim: + n2 = s[i + 1] + if n2 == "/": + state = LINE + i += 2 + continue + if n2 == "*": + state = BLOCK + i += 2 + continue + elif c == '"': + if i + 2 < lim and s[i: i + 3] == '"""': + state = TEXT + i += 3 + continue + state = STRING + i += 1 + continue + elif c == "'": + state = CHAR + i += 1 + continue + elif c == "(": + depth += 1 + elif c == ")": + depth -= 1 + if depth == 0: + return i + elif state == LINE: + if c == "\n": + state = NORMAL + elif state == BLOCK: + if c == "*" and i + 1 < lim and s[i + 1] == "/": + state = NORMAL + i += 2 + continue + elif state == STRING: + if c == "\\": + i += 2 + continue + if c == '"': + state = NORMAL + elif state == CHAR: + if c == "\\": + i += 2 + continue + if c == "'": + state = NORMAL + else: # TEXT + if i + 2 < lim and s[i: i + 3] == '"""': + state = NORMAL + i += 3 + continue + i += 1 + + return -1 + + def _match_balanced_brace(s: str, lbrace: int, lim: int) -> int: + NORMAL, LINE, BLOCK, STRING, CHAR, TEXT = 0, 1, 2, 3, 4, 5 + state = NORMAL + i = lbrace + 1 + depth = 1 + + while i < lim: + c = s[i] + if state == NORMAL: + if c == "/" and i + 1 < lim: + n2 = s[i + 1] + if n2 == "/": + state = LINE + i += 2 + continue + if n2 == "*": + state = BLOCK + i += 2 + continue + elif c == '"': + if i + 2 < lim and s[i: i + 3] == '"""': + state = TEXT + i += 3 + continue + state = STRING + i += 1 + continue + elif c == "'": + state = CHAR + i += 1 + continue + elif c == "{": + depth += 1 + elif c == "}": + depth -= 1 + if depth == 0: + return i + elif state == LINE: + if c == "\n": + state = NORMAL + elif state == BLOCK: + if c == "*" and i + 1 < lim and s[i + 1] == "/": + state = NORMAL + i += 2 + continue + elif state == STRING: + if c == "\\": + i += 2 + continue + if c == '"': + state = NORMAL + elif state == CHAR: + if c == "\\": + i += 2 + continue + if c == "'": + state = NORMAL + else: # TEXT + if i + 2 < lim and s[i: i + 3] == '"""': + state = NORMAL + i += 3 + continue + i += 1 + + return -1 + + def _line_is_annotation_only(s: str, line_start: int, line_end: int) -> bool: + i = line_start + while i < line_end and s[i].isspace(): + i += 1 + if i >= line_end or s[i] != "@": + return False + + # Parse one-or-more annotations: @Name or @pkg.Name or @Name(...) + while True: + if i >= line_end or s[i] != "@": + break + i += 1 + if i >= line_end or not _is_id_start(s[i]): + return False + i += 1 + while i < line_end and (_is_id_part(s[i]) or s[i] == "."): + i += 1 + + i = _skip_ws_and_comments(s, i, line_end) + if i < line_end and s[i] == "(": + r = _match_balanced_paren(s, i, line_end) + if r == -1: + return False + i = r + 1 + + i = _skip_ws_and_comments(s, i, line_end) + if i < line_end and s[i] == "@": + continue + break + + i = _skip_ws_and_comments(s, i, line_end) + return i >= line_end + + def _skip_annotation_only_lines(s: str, i: int, lim: int) -> int: + n_local = len(s) + while i < lim: + line_end = s.find("\n", i, lim) + if line_end == -1: + line_end = min(lim, n_local) + if not _line_is_annotation_only(s, i, line_end): + return i + i = line_end + 1 + return i + + # ------------------------- + # Type discovery (named inner chain computed as qual_name) + # ------------------------- + def _find_all_named_types(s: str) -> list[_TypeRegion]: + n_local = len(s) + NORMAL, LINE, BLOCK, STRING, CHAR, TEXT = 0, 1, 2, 3, 4, 5 + state = NORMAL + + class_kinds = {"class", "interface", "enum", "record"} + + out_types: list[_TypeRegion] = [] + type_stack: list[int] = [] # indices of open named types + brace_stack: list[tuple[int, int]] = [] # (open_brace_pos, named_type_idx_or_-1) + + pending_kind: str | None = None + pending_start: int = -1 + pending_name: str | None = None + + def _prev_non_ws_local(pos: int) -> int: + while pos >= 0 and s[pos].isspace(): + pos -= 1 + return pos + + i = 0 + while i < n_local: + c = s[i] + + if state == NORMAL: + if c == "/" and i + 1 < n_local: + n2 = s[i + 1] + if n2 == "/": + state = LINE + i += 2 + continue + if n2 == "*": + state = BLOCK + i += 2 + continue + if c == '"': + if i + 2 < n_local and s[i: i + 3] == '"""': + state = TEXT + i += 3 + continue + state = STRING + i += 1 + continue + if c == "'": + state = CHAR + i += 1 + continue + + if _is_id_start(c): + j = i + 1 + while j < n_local and _is_id_part(s[j]): + j += 1 + tok = s[i:j] + + if pending_kind is None: + if tok in class_kinds: + # ------------------------------ + # ✅ CRITICAL GUARD: + # Ignore ".class" and other member-access uses: + # StandardScheme.class.equals(...) + # Foo.class + # This prevents bogus "class equals" types. + # ------------------------------ + p = _prev_non_ws_local(i - 1) + if p >= 0 and s[p] == ".": + i = j + continue + # Defensive: ignore "class." patterns too + if j < n_local and s[j] == ".": + i = j + continue + + pending_kind = tok + pending_start = i + pending_name = None + else: + if pending_name is None: + pending_name = tok + + i = j + continue + + if c == "{": + brace_pos = i + if pending_kind is not None and pending_name is not None: + parent = type_stack[-1] if type_stack else -1 + qual = pending_name if parent == -1 else (out_types[parent].qual_name + "." + pending_name) + out_types.append( + _TypeRegion( + kind=pending_kind, + name=pending_name, + start=pending_start, + end=-1, + body_start=brace_pos + 1, + body_end=-1, + parent=parent, + qual_name=qual, + ) + ) + t_idx = len(out_types) - 1 + brace_stack.append((brace_pos, t_idx)) + type_stack.append(t_idx) + pending_kind = None + pending_start = -1 + pending_name = None + else: + brace_stack.append((brace_pos, -1)) + i += 1 + continue + + if c == "}": + if brace_stack: + _, t_idx = brace_stack.pop() + if t_idx != -1: + out_types[t_idx].body_end = i + out_types[t_idx].end = i + 1 + if type_stack and type_stack[-1] == t_idx: + type_stack.pop() + else: + # defensive fallback + try: + type_stack.remove(t_idx) + except ValueError: + pass + i += 1 + continue + + i += 1 + continue + + if state == LINE: + if c == "\n": + state = NORMAL + i += 1 + continue + + if state == BLOCK: + if c == "*" and i + 1 < n_local and s[i + 1] == "/": + state = NORMAL + i += 2 + continue + i += 1 + continue + + if state == STRING: + if c == "\\": + i += 2 + continue + if c == '"': + state = NORMAL + i += 1 + continue + + if state == CHAR: + if c == "\\": + i += 2 + continue + if c == "'": + state = NORMAL + i += 1 + continue + + # TEXT + if i + 2 < n_local and s[i: i + 3] == '"""': + state = NORMAL + i += 3 + continue + i += 1 + + # Keep only closed types + closed: list[_TypeRegion] = [] + for t in out_types: + if t.end != -1 and t.body_end != -1: + closed.append(t) + return closed + + def _normalize_types(maybe_types: list) -> list[_TypeRegion]: + norm: list[_TypeRegion] = [] + for t in maybe_types: + # attribute-style + if hasattr(t, "kind") and hasattr(t, "name") and hasattr(t, "body_start") and hasattr(t, "body_end"): + kind = getattr(t, "kind") + name = getattr(t, "name") + tstart = getattr(t, "start", -1) + tend = getattr(t, "end", -1) + bstart = getattr(t, "body_start") + bend = getattr(t, "body_end") + parent = getattr(t, "parent", -1) + qual = getattr(t, "qual_name", "") or name + norm.append(_TypeRegion(kind, name, tstart, tend, bstart, bend, parent, qual)) + continue + + # dict-style + if isinstance(t, dict): + kind = t.get("kind", "") + name = t.get("name", "") + tstart = t.get("start", -1) + tend = t.get("end", -1) + bstart = t.get("body_start", -1) + bend = t.get("body_end", -1) + parent = t.get("parent", -1) + qual = t.get("qual_name") or name + norm.append(_TypeRegion(kind, name, tstart, tend, bstart, bend, parent, qual)) + continue + + # tuple/list positional fallback (kind,name,start,end,body_start,body_end,parent,qual_name) + if isinstance(t, (tuple, list)) and len(t) >= 6: + kind = t[0] + name = t[1] + tstart = t[2] if len(t) > 2 else -1 + tend = t[3] if len(t) > 3 else -1 + bstart = t[4] + bend = t[5] + parent = t[6] if len(t) > 6 else -1 + qual = t[7] if len(t) > 7 else name + norm.append(_TypeRegion(kind, name, tstart, tend, bstart, bend, parent, qual)) + continue + + # unknown shape -> ignore + return norm + + def _repair_qual_names(types_list: list[_TypeRegion]) -> None: + """ + If caller-provided `types` lack correct qual_name chains, repair them using `parent`. + Safe no-op if already correct. + """ + any_needs = False + for t in types_list: + if t.parent != -1 and ("." not in (t.qual_name or "")): + any_needs = True + break + if not any_needs: + return + + memo: dict[int, str] = {} + + def qual(i: int) -> str: + if i in memo: + return memo[i] + t = types_list[i] + if t.parent == -1: + memo[i] = t.name + else: + p = qual(t.parent) + memo[i] = p + "." + t.name if p else t.name + return memo[i] + + for idx in range(len(types_list)): + types_list[idx].qual_name = qual(idx) + + # ------------------------- + # Main extraction logic + # ------------------------- + s = src + n = len(s) + if end is None: + end = n + if start < 0: + start = 0 + if end > n: + end = n + if start >= end: + return [] + + if types is None: + types2 = _find_all_named_types(s) + else: + types2 = _normalize_types(types) + _repair_qual_names(types2) + + # Map opening brace positions to named type index. + # Named type opening brace is at (body_start - 1). + lbrace_to_type: dict[int, int] = {} + for idx, t in enumerate(types2): + lbrace_to_type[t.body_start - 1] = idx + + # Precompute "inner-type" comment payload per named type idx. + # We strip the top-level declaring type (first segment) and keep the rest. + inner_comment_by_idx: list[str | None] = [None] * len(types2) + for idx, t in enumerate(types2): + if t.parent != -1: + q = t.qual_name or t.name + dot = q.find(".") + inner_comment_by_idx[idx] = q[dot + 1:] if dot != -1 else t.name + + # Scanner state + NORMAL, LINE, BLOCK, STRING, CHAR, TEXT = 0, 1, 2, 3, 4, 5 + state = NORMAL + + out: list[str] = [] + + # Brace stack kinds: 0 = other, 1 = named type body, 2 = anonymous class body + brace_stack: list[tuple[int, int]] = [] + brace_depth = 0 + + # Track parentheses and whether they belong to a `new ... ( ... )` object creation + paren_objcreate_stack: list[bool] = [] + in_new_expr = False # True from seeing token "new" until we consume its '(' or cancel + + expecting_anon_class_brace = False + class_bodies: list[_ClassBodyFrame] = [] + + modifiers = { + "public", "protected", "private", "abstract", "final", "static", "native", + "synchronized", "strictfp", "default", "transient", "volatile", "sealed", "non-sealed", + } + ctrl_like = { + "if", "for", "while", "switch", "catch", "do", "try", "return", "throw", "new", + "case", "assert", "break", "continue", "else", "finally", "yield", + "this", "super", + } + + def _current_named_type_name() -> str | None: + if not class_bodies: + return None + idx = class_bodies[-1].named_type_idx + if idx < 0: + return None + return types2[idx].name + + def _current_inner_comment() -> str | None: + if not class_bodies: + return None + idx = class_bodies[-1].named_type_idx + if idx < 0: + return None + return inner_comment_by_idx[idx] + + def _is_constructor_allowed(method_name: str) -> bool: + tn = _current_named_type_name() + return tn is not None and method_name == tn + + def _prev_non_ws(pos: int, floor: int) -> int: + j = pos + while j >= floor and s[j].isspace(): + j -= 1 + return j + + def _read_prev_identifier(pos: int, floor: int) -> tuple[str, int, int]: + j = _prev_non_ws(pos, floor) + if j < floor: + return "", -1, -1 + if not _is_id_part(s[j]): + return "", -1, -1 + k = j + while k >= floor and _is_id_part(s[k]): + k -= 1 + return s[k + 1: j + 1], k + 1, j + 1 + + def _has_return_type_before_name(name_start: int, floor: int) -> bool: + j = name_start - 1 + while j >= floor: + while j >= floor and s[j].isspace(): + j -= 1 + if j < floor: + break + if not _is_id_part(s[j]): + j -= 1 + continue + + tok, ts, _ = _read_prev_identifier(j, floor) + if not tok: + break + + # skip annotation identifiers (@Something) + p = _prev_non_ws(ts - 1, floor) + if p >= floor and s[p] == "@": + j = p - 1 + continue + + if tok in modifiers: + j = ts - 1 + continue + + # ✅ IMPORTANT: keywords like "new"/"return"/"if"/... are NOT return types + if tok in ctrl_like: + j = ts - 1 + continue + + return True + return False + + + def _try_parse_method_at_lparen(lparen: int) -> tuple[int, int] | None: + frame = class_bodies[-1] + member_floor = frame.member_start + + # method/ctor name token before '(' + j = _prev_non_ws(lparen - 1, member_floor) + if j < member_floor or not _is_id_part(s[j]): + return None + + k = j + while k >= member_floor and _is_id_part(s[k]): + k -= 1 + name_start = k + 1 + name = s[name_start: j + 1] + if not name or name in ctrl_like: + return None + + # ------------------------------------------------------------ + # ✅ NEW: Reject annotation argument parens: "@Something(...)". + # This prevents "@SuppressWarnings(...)" from being mis-parsed as a method. + # ------------------------------------------------------------ + p0 = _prev_non_ws(name_start - 1, member_floor) + if p0 >= member_floor and s[p0] == "@": + return None + + # Reject qualified/call expressions: ".name(" or "::name(" + if p0 >= member_floor and s[p0] in (".", ":"): + return None + + # Compute the "real signature start" (skip ws/comments + annotation-only lines) + sig0 = _skip_ws_and_comments(s, member_floor, end) + sig0 = _skip_annotation_only_lines(s, sig0, min(lparen, end)) + + # ------------------------------------------------------------ + # ✅ Field initializer guard: + # If there's an '=' before this '(' in the same member signature, + # we're in an initializer, not a method/ctor declaration. + # ------------------------------------------------------------ + if sig0 < lparen and s.find("=", sig0, lparen) != -1: + if _signature_has_initializer_assignment(sig0, lparen): + return None + + # Reject type declarations that have (...) headers (e.g., "record R(...){...}") + prev_tok, _, _ = _read_prev_identifier(name_start - 1, member_floor) + if prev_tok in ("class", "interface", "enum", "record"): + return None + + # Reject "new Name(" (constructor call) vs declaration + if prev_tok == "new": + return None + + rparen = _match_balanced_paren(s, lparen, end) + if rparen == -1: + return None + + # Find '{' starting the body (skip ws/comments; allow throws etc) + i2 = _skip_ws_and_comments(s, rparen + 1, end) + if i2 >= end: + return None + if i2 + 1 < end and s[i2: i2 + 2] == "->": + return None + if s[i2] == ";": + return None + + # Scan forward until '{' or ';' or '->', respecting strings/comments. + NORMAL2, LINE2, BLOCK2, STRING2, CHAR2, TEXT2 = 0, 1, 2, 3, 4, 5 + st2 = NORMAL2 + j2 = i2 + body_lbrace = -1 + + while j2 < end: + c2 = s[j2] + if st2 == NORMAL2: + if c2 == "/" and j2 + 1 < end: + n2 = s[j2 + 1] + if n2 == "/": + st2 = LINE2 + j2 += 2 + continue + if n2 == "*": + st2 = BLOCK2 + j2 += 2 + continue + if c2 == '"': + if j2 + 2 < end and s[j2: j2 + 3] == '"""': + st2 = TEXT2 + j2 += 3 + continue + st2 = STRING2 + j2 += 1 + continue + if c2 == "'": + st2 = CHAR2 + j2 += 1 + continue + + if j2 + 1 < end and s[j2: j2 + 2] == "->": + return None + if c2 == ";": + return None + if c2 == "{": + body_lbrace = j2 + break + + elif st2 == LINE2: + if c2 == "\n": + st2 = NORMAL2 + elif st2 == BLOCK2: + if c2 == "*" and j2 + 1 < end and s[j2 + 1] == "/": + st2 = NORMAL2 + j2 += 2 + continue + elif st2 == STRING2: + if c2 == "\\": + j2 += 2 + continue + if c2 == '"': + st2 = NORMAL2 + elif st2 == CHAR2: + if c2 == "\\": + j2 += 2 + continue + if c2 == "'": + st2 = NORMAL2 + else: # TEXT2 + if j2 + 2 < end and s[j2: j2 + 3] == '"""': + st2 = NORMAL2 + j2 += 3 + continue + + j2 += 1 + + if body_lbrace == -1: + return None + + # ------------------------------------------------------------ + # ✅ IMPORTANT CHANGE: use sig0 as the "floor" for return-type detection. + # This prevents Javadoc/comments/previous-members from faking a return type. + # ------------------------------------------------------------ + if frame.named_type_idx >= 0: + if not _has_return_type_before_name(name_start, sig0): + if not _is_constructor_allowed(name): + return None + else: + if not _has_return_type_before_name(name_start, sig0): + return None + + rbrace = _match_balanced_brace(s, body_lbrace, end) + if rbrace == -1: + return None + method_end = rbrace + 1 + + # Signature start excluding annotation-only lines + sig_start = _skip_annotation_only_lines(s, sig0, min(body_lbrace, end)) + return sig_start, method_end + + i = start + while i < end: + c = s[i] + + if state == NORMAL: + # comment/string/text transitions + if c == "/" and i + 1 < end: + n2 = s[i + 1] + if n2 == "/": + state = LINE + i += 2 + continue + if n2 == "*": + state = BLOCK + i += 2 + continue + + if c == '"': + if i + 2 < end and s[i: i + 3] == '"""': + state = TEXT + i += 3 + continue + state = STRING + i += 1 + continue + + if c == "'": + state = CHAR + i += 1 + continue + + # identifiers: detect 'new' + if _is_id_start(c): + j = i + 1 + while j < end and _is_id_part(s[j]): + j += 1 + tok = s[i:j] + if tok == "new": + in_new_expr = True + i = j + continue + + # cancel `new` expectation on array creation or obvious expression terminators + if in_new_expr and c in ("[", "{", ";"): + in_new_expr = False + + # Cancel expecting anon brace if we hit meaningful char that's not '{' + if expecting_anon_class_brace and not c.isspace() and c != "{": + expecting_anon_class_brace = False + + if c == "(": + paren_objcreate_stack.append(in_new_expr) + in_new_expr = False # consumed the object-creation marker for this '(' + + # method parsing at class member level + if class_bodies: + frame = class_bodies[-1] + # Only extract methods for *named* types; skip anonymous-class bodies (new X() { ... }). + if frame.named_type_idx >= 0 and brace_depth == frame.body_depth and frame.enum_constants_done: + parsed = _try_parse_method_at_lparen(i) + if parsed is not None: + sig_start, method_end = parsed + method_src = s[sig_start:method_end] + + # Strip comments/javadocs/annotations/package/imports FIRST + method_src = _strip_java_noise(method_src) + + # Then append the inner-type marker so it survives stripping + inner = _current_inner_comment() + if inner: + method_src = method_src.rstrip() + f"\n/* inner-type: {inner} */" + + out.append(method_src) + frame.member_start = method_end + i = method_end + continue + + i += 1 + continue + + if c == ")": + if paren_objcreate_stack and paren_objcreate_stack.pop(): + expecting_anon_class_brace = True + i += 1 + continue + + if c == "{": + brace_depth += 1 + + t_idx = lbrace_to_type.get(i, -1) + if t_idx != -1: + brace_stack.append((i, 1)) + is_enum = (types2[t_idx].kind == "enum") + class_bodies.append(_ClassBodyFrame(t_idx, brace_depth, i + 1, is_enum)) + expecting_anon_class_brace = False + in_new_expr = False + i += 1 + continue + + if expecting_anon_class_brace: + brace_stack.append((i, 2)) + class_bodies.append(_ClassBodyFrame(-1, brace_depth, i + 1, False)) + expecting_anon_class_brace = False + in_new_expr = False + i += 1 + continue + + brace_stack.append((i, 0)) + in_new_expr = False + i += 1 + continue + + if c == "}": + if brace_stack: + _, kind = brace_stack.pop() + + # if closing a block directly inside a class body, advance member_start + if class_bodies: + frame = class_bodies[-1] + if kind == 0 and brace_depth == frame.body_depth + 1: + frame.member_start = i + 1 + + if kind == 1: + # closing named type: pop frame, and advance parent's member_start + if class_bodies and class_bodies[-1].named_type_idx != -1: + class_bodies.pop() + if class_bodies: + class_bodies[-1].member_start = i + 1 + elif kind == 2: + # closing anonymous class: pop frame only + if class_bodies and class_bodies[-1].named_type_idx == -1: + class_bodies.pop() + + brace_depth = max(0, brace_depth - 1) + expecting_anon_class_brace = False + in_new_expr = False + i += 1 + continue + + if c == ";" and class_bodies: + frame = class_bodies[-1] + if brace_depth == frame.body_depth: + # Enum constant section ends at the first semicolon at enum-body depth. + if frame.is_enum and not frame.enum_constants_done: + frame.enum_constants_done = True + frame.member_start = i + 1 + i += 1 + continue + + i += 1 + continue + + if state == LINE: + if c == "\n": + state = NORMAL + i += 1 + continue + + if state == BLOCK: + if c == "*" and i + 1 < end and s[i + 1] == "/": + state = NORMAL + i += 2 + continue + i += 1 + continue + + if state == STRING: + if c == "\\": + i += 2 + continue + if c == '"': + state = NORMAL + i += 1 + continue + + if state == CHAR: + if c == "\\": + i += 2 + continue + if c == "'": + state = NORMAL + i += 1 + continue + + # TEXT + if i + 2 < end and s[i: i + 3] == '"""': + state = NORMAL + i += 3 + continue + i += 1 + + return out + +# ============================== public API ==================================== + +def extract_methods(java_source: str) -> List[str]: + """ + Returns a list of Java method/constructor bodies/signatures (with bodies) plus all arrow-lambda + forms found in the file. Original comments/Javadoc are excluded and method-level annotations are + stripped during extraction. For methods/constructors declared inside a *nested named type*, a small synthetic + trailing comment is appended: + /* inner-type: */ + (Top-level type members are unchanged.) + Order: methods first, then lambdas. Optimized for large files by pre-indexing + comments/strings and delimiter pairs (O(1) matching for (), {}). + + Additional filtering: + - Exclude only the *standard* Object overrides: + * boolean equals(Object) + * int hashCode() + * String toString() + (Overloads are kept.) + """ + def _is_ident_char(ch: str) -> bool: + return ch.isalnum() or ch in ('_', '$') + + def _read_ident_fwd(s: str, pos: int, end: int) -> Tuple[int, str]: + if pos >= end or not _is_ident_char(s[pos]): + return pos, "" + start = pos + pos += 1 + while pos < end and _is_ident_char(s[pos]): + pos += 1 + return pos, s[start:pos] + + def _read_qualified_ident_fwd(s: str, pos: int, end: int) -> Tuple[int, str]: + pos, seg = _read_ident_fwd(s, pos, end) + if not seg: + return pos, "" + parts = [seg] + while pos < end and s[pos] == '.': + dot = pos + pos += 1 + pos2, seg2 = _read_ident_fwd(s, pos, end) + if not seg2: + # not actually qualified; stop at dot + return dot, ".".join(parts) + parts.append(seg2) + pos = pos2 + return pos, ".".join(parts) + + def _skip_ws(s: str, pos: int, end: int) -> int: + while pos < end and s[pos].isspace(): + pos += 1 + return pos + + def _has_top_level_comma(params: str) -> bool: + # track generic nesting so commas inside Map don't look like param separators + depth = 0 + for ch in params: + if ch == '<': + depth += 1 + elif ch == '>' and depth > 0: + depth -= 1 + elif ch == ',' and depth == 0: + return True + return False + + def _standard_override_kind(method_text: str) -> str: + """ + Return "equals" / "hashCode" / "toString" if this is exactly the standard Object override, + else "". + + Assumptions: method_text is already stripped of comments/Javadoc/annotations by extraction. + """ + open_paren = method_text.find('(') + if open_paren == -1: + return "" + + # Find the first '{' after '('; that's the method/ctor body start in extracted text. + brace = method_text.find('{', open_paren) + if brace == -1: + return "" + + # Close paren must be before the body. + close_paren = method_text.rfind(')', open_paren, brace) + if close_paren == -1: + return "" + + # Method name: scan left from '(' + j = open_paren - 1 + while j >= 0 and method_text[j].isspace(): + j -= 1 + name_end = j + 1 + while j >= 0 and _is_ident_char(method_text[j]): + j -= 1 + name_start = j + 1 + method_name = method_text[name_start:name_end] + if method_name not in ("equals", "hashCode", "toString"): + return "" + + # Return type token: scan left from method_name_start + k = name_start - 1 + while k >= 0 and method_text[k].isspace(): + k -= 1 + rt_end = k + 1 + while k >= 0 and (method_text[k].isalnum() or method_text[k] in ('_', '$', '.')): + k -= 1 + return_type = method_text[k + 1:rt_end] + + params = method_text[open_paren + 1:close_paren] + + # ---- hashCode() ---- + if method_name == "hashCode": + if return_type != "int": + return "" + if params.strip(): + return "" + return "hashCode" + + # ---- toString() ---- + if method_name == "toString": + if return_type not in ("String", "java.lang.String"): + return "" + if params.strip(): + return "" + return "toString" + + # ---- equals(Object) ---- + if method_name == "equals": + if return_type != "boolean": + return "" + p = params.strip() + if not p: + return "" + if _has_top_level_comma(p): + return "" + + # parse: [final] (Object|java.lang.Object) + tmp = p + pos = 0 + end = len(tmp) + + pos = _skip_ws(tmp, pos, end) + pos2, maybe_final = _read_ident_fwd(tmp, pos, end) + if maybe_final == "final": + pos = _skip_ws(tmp, pos2, end) + + pos, type_name = _read_qualified_ident_fwd(tmp, pos, end) + if type_name not in ("Object", "java.lang.Object"): + return "" + + pos = _skip_ws(tmp, pos, end) + + # Reject arrays/varargs/generic suffixes on the parameter type (not the standard override) + if pos < end and (tmp[pos] == '[' or tmp[pos] == '<' or tmp.startswith("...", pos)): + return "" + + pos, param_name = _read_ident_fwd(tmp, pos, end) + if not param_name: + return "" + + # nothing else meaningful should follow in a normal single-param decl + if tmp[_skip_ws(tmp, pos, end):].strip(): + return "" + + return "equals" + + return "" + + _populate_delim_index_cache(java_source) + try: + methods = _extract_methods_anywhere(java_source, 0, len(java_source)) + + # Filter only the standard Object overrides (keep overloads). + filtered: List[str] = [m for m in methods if not _standard_override_kind(m)] + + lambdas = _extract_lambdas_in_range(java_source, 0, len(java_source)) + return filtered + lambdas + finally: + _clear_delim_index_cache() + +def extract_inner_classes(src: str) -> list[str]: + """ + Extract Java *named* inner classes/interfaces/enums/records (with full bodies) + from a compilation unit text. + + Changes vs prior behavior: + - Does NOT prefix extracted inner types with the file's package/import block. + - Strips from each extracted inner type snippet: + * // line comments + * /* ... */ block comments (incl. Javadocs) + * annotations (e.g. @SuppressWarnings(...), @Nullable, @A.B) + while preserving strings/chars/text blocks. + + Notes: + - Anonymous inner classes are NOT returned here (TODO remains). + - Performance: single forward scan + O(k) balanced-brace matches for each found inner type. + Stripping is O(m) per extracted snippet and is skipped when markers are absent. + + Bugfix: + - Avoids false positives from identifiers like `record` used as a variable name + (e.g. `Object record = ...`) by only treating `class|interface|enum|record` + as declaration keywords at a declaration-start position (after `{`, `}`, `;`, start, + or after modifiers/annotations), and by cancelling pending candidates on `=` / `;` + (and on `(` for non-record kinds). + - Fixes slice start for inner types so we never “pull in” the previous statement line + (e.g. enum constants ending with `;`) when computing the snippet start. + """ + s = src + n = len(s) + if n == 0: + return [] + + # Quick reject + if ("class" not in s and "interface" not in s and "enum" not in s and "record" not in s): + return [] + + # ---------- local helpers ---------- + def _is_ident_start(ch: str) -> bool: + return ch.isalpha() or ch in "_$" + + def _is_ident_part(ch: str) -> bool: + return ch.isalnum() or ch in "_$" + + def _match_balanced_brace(lbrace: int, lim: int) -> int: + NORMAL, LINE, BLOCK, STRING, CHAR, TEXT = 0, 1, 2, 3, 4, 5 + st = NORMAL + i = lbrace + 1 + depth = 1 + + while i < lim: + c = s[i] + if st == NORMAL: + if c == "/" and i + 1 < lim: + n2 = s[i + 1] + if n2 == "/": + st = LINE + i += 2 + continue + if n2 == "*": + st = BLOCK + i += 2 + continue + if c == '"': + if i + 2 < lim and s[i:i + 3] == '"""': + st = TEXT + i += 3 + continue + st = STRING + i += 1 + continue + if c == "'": + st = CHAR + i += 1 + continue + + if c == "{": + depth += 1 + elif c == "}": + depth -= 1 + if depth == 0: + return i + + elif st == LINE: + if c == "\n": + st = NORMAL + elif st == BLOCK: + if c == "*" and i + 1 < lim and s[i + 1] == "/": + st = NORMAL + i += 2 + continue + elif st == STRING: + if c == "\\": + i += 2 + continue + if c == '"': + st = NORMAL + elif st == CHAR: + if c == "\\": + i += 2 + continue + if c == "'": + st = NORMAL + else: # TEXT + if i + 2 < lim and s[i:i + 3] == '"""': + st = NORMAL + i += 3 + continue + + i += 1 + + return -1 + + def _line_is_annotation_only(line_start: int, line_end: int) -> bool: + """ + True if the line contains only annotations (possibly multiple) plus whitespace/comments. + """ + i = line_start + while i < line_end and s[i].isspace(): + i += 1 + if i >= line_end or s[i] != "@": + return False + + # Parse one-or-more annotations: @Name or @pkg.Name or @Name(...) + while True: + if i >= line_end or s[i] != "@": + break + i += 1 + if i >= line_end or not _is_ident_start(s[i]): + return False + i += 1 + while i < line_end and (_is_ident_part(s[i]) or s[i] == "."): + i += 1 + + i = _skip_ws_comments(s, i) + if i < line_end and s[i] == "(": + r = _match_balanced_parens(s, i) + if r == -1: + return False + i = r + 1 + + i = _skip_ws_comments(s, i) + if i < line_end and s[i] == "@": + continue + break + + i = _skip_ws_comments(s, i) + return i >= line_end + + def _consume_annotations(i: int) -> int: + """ + Consume one-or-more annotations starting at s[i] == '@' and return the first index + after them (skipping whitespace/comments between annotations). + """ + while i < n and s[i] == "@": + i += 1 + if i >= n or not _is_ident_start(s[i]): + return i + + # qualified name + i += 1 + while i < n and (_is_ident_part(s[i]) or s[i] == "."): + i += 1 + + i = _skip_ws_comments(s, i) + if i < n and s[i] == "(": + r = _match_balanced_parens(s, i) + if r == -1: + return i + i = r + 1 + + i = _skip_ws_comments(s, i) + return i + + def _sig_start_for_type(kind_pos: int) -> int: + """ + Start at the beginning of the line containing the kind keyword, and (optionally) + include immediately preceding annotation-only lines. Critically, NEVER walk into + the previous statement line (e.g., enum constants ending with ';'). + """ + # Anchor to the declaration line that actually contains `class|interface|enum|record`. + sig_line_start = s.rfind("\n", 0, kind_pos) + 1 + sig_start = sig_line_start + + # Now look upward for annotation-only lines directly above. + k = sig_line_start + # Trim back over whitespace/newlines to the end of the previous non-empty content. + while k > 0 and s[k - 1].isspace(): + k -= 1 + + while k > 0: + line_start = s.rfind("\n", 0, k - 1) + 1 + if _line_is_annotation_only(line_start, k): + sig_start = line_start + k = line_start + while k > 0 and s[k - 1].isspace(): + k -= 1 + continue + break + + return sig_start + + # ---------- scan ---------- + NORMAL, LINE, BLOCK, STRING, CHAR, TEXT = 0, 1, 2, 3, 4, 5 + state = NORMAL + + inner_classes: list[str] = [] + + class_kinds = {"class", "interface", "enum", "record"} + class_mods = { + "public", "protected", "private", + "abstract", "final", "static", "sealed", "strictfp", + } + + brace_depth = 0 + named_type_depth_stack: list[int] = [] + + pending_kind: str | None = None + pending_kind_pos: int = -1 + pending_name: str | None = None + + # Only True at a plausible declaration start position. + can_start_decl = True + + i = 0 + while i < n: + c = s[i] + + if state == NORMAL: + # comments + if c == "/" and i + 1 < n: + n2 = s[i + 1] + if n2 == "/": + state = LINE + i += 2 + continue + if n2 == "*": + state = BLOCK + i += 2 + continue + + # string/text/char + if c == '"': + if i + 2 < n and s[i:i + 3] == '"""': + state = TEXT + i += 3 + continue + state = STRING + i += 1 + continue + if c == "'": + state = CHAR + i += 1 + continue + + # ---- cancel/advance on delimiters that cannot belong to a type header ---- + if c == ";": + pending_kind = None + pending_kind_pos = -1 + pending_name = None + can_start_decl = True + i += 1 + continue + + if c == "=": + pending_kind = None + pending_kind_pos = -1 + pending_name = None + can_start_decl = False + i += 1 + continue + + if c == "(" and pending_kind is not None and pending_kind != "record": + # `class|interface|enum` headers never contain '(' + pending_kind = None + pending_kind_pos = -1 + pending_name = None + can_start_decl = False + i += 1 + continue + + # annotations at decl-start: consume and remain in decl-start mode + if c == "@" and can_start_decl and pending_kind is None: + # Special-case: allow "@interface Foo { ... }" as an inner "interface" decl. + if (i + 10 <= n and s[i + 1:i + 10] == "interface" and + (i + 10 == n or not _is_ident_part(s[i + 10]))): + i += 1 # next loop sees "interface" + continue + + i = _consume_annotations(i) + can_start_decl = True + continue + + # identifier tokenizing + if _is_ident_start(c): + j = i + 1 + while j < n and _is_ident_part(s[j]): + j += 1 + tok = s[i:j] + + # After kind keyword, accept very next identifier as the type name. + if pending_kind is not None and pending_name is None: + pending_name = tok + can_start_decl = False + i = j + continue + + # Only recognize kind keywords at declaration starts. + if pending_kind is None and can_start_decl: + if tok in class_mods: + can_start_decl = True + elif tok in class_kinds: + pending_kind = tok + pending_kind_pos = i + pending_name = None + can_start_decl = False + else: + can_start_decl = False + + i = j + continue + + # Otherwise we're in code/header; not a new decl start. + can_start_decl = False + i = j + continue + + # brace open + if c == "{": + brace_depth += 1 + can_start_decl = True + + if pending_kind is not None and pending_name is not None: + is_inner = bool(named_type_depth_stack) + + # FIXED: anchor to the declaration line; do not walk into previous statements. + sig_start = _sig_start_for_type(pending_kind_pos) + + body_close = _match_balanced_brace(i, n) + if body_close != -1 and is_inner: + raw = s[sig_start:body_close + 1] + + cleaned = raw + if "/" in cleaned and (("//" in cleaned) or ("/*" in cleaned)): + cleaned = _strip_comments_keep_strings_range(cleaned, 0, len(cleaned)) + if "@" in cleaned: + cleaned = _strip_annotations_in_range(cleaned, 0, len(cleaned)) + + cleaned = cleaned.strip() + if cleaned: + inner_classes.append(cleaned) + + named_type_depth_stack.append(brace_depth) + + pending_kind = None + pending_kind_pos = -1 + pending_name = None + + i += 1 + continue + + pending_kind = None + pending_kind_pos = -1 + pending_name = None + + i += 1 + continue + + # brace close + if c == "}": + if brace_depth > 0: + if named_type_depth_stack and named_type_depth_stack[-1] == brace_depth: + named_type_depth_stack.pop() + brace_depth -= 1 + + pending_kind = None + pending_kind_pos = -1 + pending_name = None + + can_start_decl = True + i += 1 + continue + + # any other non-ws char likely means we're in an expression/statement + if not c.isspace(): + can_start_decl = False + + i += 1 + continue + + # ---------- non-NORMAL states ---------- + if state == LINE: + if c == "\n": + state = NORMAL + i += 1 + continue + + if state == BLOCK: + if c == "*" and i + 1 < n and s[i + 1] == "/": + state = NORMAL + i += 2 + continue + i += 1 + continue + + if state == STRING: + if c == "\\": + i += 2 + continue + if c == '"': + state = NORMAL + i += 1 + continue + + if state == CHAR: + if c == "\\": + i += 2 + continue + if c == "'": + state = NORMAL + i += 1 + continue + + # TEXT + if i + 2 < n and s[i:i + 3] == '"""': + state = NORMAL + i += 3 + continue + i += 1 + + return inner_classes \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/java_utils.py b/src/exploit_iq_commons/utils/java_utils.py new file mode 100644 index 000000000..229bf7dab --- /dev/null +++ b/src/exploit_iq_commons/utils/java_utils.py @@ -0,0 +1,2968 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import re +from dataclasses import dataclass +from functools import lru_cache +from typing import Collection, Dict, List, Tuple, Optional, Any, Set +from langchain_core.documents import Document +from tqdm import tqdm + +JAVA_ANNOTATION_SYMBOL = '@' +dummy_package_name = "dummy:dummy:1.0.0" +_BRACE_NORMAL_SPECIAL_RE = re.compile(r'[/"\'{}-]') +JAVA_METHOD_CLASS_KINDS = ("class", "interface", "enum", "record") +JAVA_METHOD_CLASS_MODS = { + "public", "protected", "private", "abstract", "final", "static", "sealed", "strictfp" +} +JAVA_METHOD_CTRL_WORDS = { + "if", "for", "while", "switch", "catch", "synchronized", "return", "new", "case" +} +JAVA_METHOD_METH_MODS = { + "public", "protected", "private", "abstract", "static", "final", "synchronized", + "native", "strictfp", "default", "transient" +} +JAVA_METHOD_PRIM_TYPES = { + "void", "int", "long", "double", "float", "short", "byte", "boolean", "char" +} +JAVA_METHOD_WS_CHARS = set(" \t\r\n\f\v") + +_ANNOTATION_RE = re.compile(r'@\w+(?:\s*\([^()]*\))?\s*') +_FINAL_RE = re.compile(r'(?.*?)(?P\s+)(?P[A-Za-z_$][\w$]*)(?P(?:\s*\[\s*\])*)\s*$', + re.DOTALL, +) +_ARRAY_BRACKETS_RE = re.compile(r'\[\s*\]') +_TRAILING_NAME_RE = re.compile(r'([A-Za-z_$][\w$]*)\s*$') + +# Precompiled regexes used by strip_java_generics +_GENERIC_SPACE_RE = re.compile(r'\s+') +_PUNCT_SPACE_RE = re.compile(r'\s*([\[\]\(\),.])\s*') +_VARARGS_SPACE_RE = re.compile(r'\s*(\.\.\.)') + +# Accepts: +# - groupId: dot-separated segments; each segment alnum, may contain '-' or '_' inside +# - artifactId: alnum, may contain '.', '_' or '-' inside +# - version: alnum start, may contain '.', '_' or '-' (e.g., 1.2.3, 1.0.0-SNAPSHOT) +_GAV_RE = re.compile( + r""" + ^(?P + (?:[A-Za-z0-9]+(?:[A-Za-z0-9_-]*[A-Za-z0-9])?) + (?:\.(?:[A-Za-z0-9]+(?:[A-Za-z0-9_-]*[A-Za-z0-9])?))* + ) + : + (?P[A-Za-z0-9]+(?:[A-Za-z0-9._-]*[A-Za-z0-9])?) + : + (?P[A-Za-z0-9][A-Za-z0-9._-]*) + $ + """, + re.X, +) + +# Maven coordinate constants for depgraph parsing +MAVEN_SCOPES: frozenset[str] = frozenset({ + "compile", "runtime", "test", "provided", "system", "import" +}) + +MAVEN_PACKAGING_TYPES: frozenset[str] = frozenset({ + "jar", "war", "pom", "bundle", "maven-plugin", "ear", "ejb", "rar", "zip", "test-jar" +}) + +# Regex for parsing depgraph tree indentation +_DEPGRAPH_INDENT_RE = re.compile(r"^(?P(?:\| | )*)(?P[+\\]-\s)?(?P.+)$") +_DEPGRAPH_TOKEN_RE = re.compile(r"^\(?([^\s\)]+)\)?") + + +@dataclass(frozen=True, slots=True) +class DepgraphCoordinate: + """Parsed Maven coordinate from depgraph-maven-plugin output.""" + depth: int + group_id: str + artifact_id: str + version: str + + @property + def gav(self) -> str: + """Return groupId:artifactId:version string.""" + return f"{self.group_id}:{self.artifact_id}:{self.version}" + + +def parse_depgraph_line(line: str) -> Optional[DepgraphCoordinate]: + """ + Parse one line from depgraph-maven-plugin graphFormat=text output. + + Handles formats like: + - org.apache.activemq:artemis-server:2.28.0:jar:compile + - +- org.apache.activemq:artemis-core-client:2.28.0:jar:compile + - | +- io.netty:netty-buffer:4.1.86.Final:jar:compile + - (com.google.guava:guava:31.1-jre:jar:compile) + + Args: + line: A single line from dependency_tree.txt + + Returns: + DepgraphCoordinate with depth and (groupId, artifactId, version), + or None if line is not a valid dependency entry. + """ + raw = (line or "").rstrip("\n") + if not raw.strip(): + return None + + # Mid-line [INFO] injection cannot be safely recovered + if "[INFO]" in raw and not raw.lstrip().startswith("[INFO]"): + return None + + s = raw.lstrip() + + # Strip Maven log prefix if present + if s.startswith("[INFO]"): + s = s[6:].lstrip() + + # Skip headers and build noise + if ( + not s + or "Dependency graph:" in s + or s.startswith(("---", "BUILD", "Reactor Summary", "Total time", "Finished at", "Scanning")) + or s.startswith("[") + or ":" not in s + ): + return None + + # Parse depgraph indentation: "| " or " " blocks + optional "+- " or "\- " + m = _DEPGRAPH_INDENT_RE.match(s) + if not m: + return None + + indent_str = m.group("indent") or "" + depth = (len(indent_str) // 3) + (1 if m.group("branch") else 0) + rest = m.group("rest").strip() + + # Extract first token (coordinate), handling optional parentheses for omitted deps + m2 = _DEPGRAPH_TOKEN_RE.match(rest) + if not m2: + return None + + token = m2.group(1) + parts = token.split(":") + + # Strip scope if present at end + if parts and parts[-1] in MAVEN_SCOPES: + parts = parts[:-1] + + if len(parts) < 3: + return None + + group, artifact = parts[0], parts[1] + + def looks_like_version(v: str) -> bool: + return any(ch.isdigit() for ch in v) + + version: Optional[str] = None + + # depgraph: group:artifact:version:type + if len(parts) >= 4 and parts[-1] in MAVEN_PACKAGING_TYPES and looks_like_version(parts[2]): + version = parts[2] + # depgraph (rare): group:artifact:version:type:classifier (5 parts after scope removal) + elif len(parts) >= 5 and parts[-2] in MAVEN_PACKAGING_TYPES and looks_like_version(parts[2]): + version = parts[2] + # Standard 3-part: group:artifact:version + elif len(parts) == 3 and looks_like_version(parts[2]): + version = parts[2] + else: + # Fallback: try last part, then index 2 + if looks_like_version(parts[-1]): + version = parts[-1] + elif looks_like_version(parts[2]): + version = parts[2] + else: + return None + + return DepgraphCoordinate(depth=depth, group_id=group, artifact_id=artifact, version=version) + + +# Matches real Java type headers like "class Foo", not ".class" +_JAVA_TYPE_HEADER_RE = re.compile(r'(? str: + i = s.rfind(':') + if i == -1: + return s + return s[:i] + ':jar' + s[i:] + + +def convert_from_maven_artifact(package_name: str) -> str: + s = package_name + + # 1) Remove substring up to and including the first ':' + first = s.find(":") + if first != -1: + s = s[first + 1:] + + # 2) Replace the last ':' (if any) with '-' + last = s.rfind(":") + if last != -1: + s = s[:last] + "-" + s[last + 1:] + + return s + +def _convert_to_maven_artifact(package_name: str) -> str: + last_index = package_name.rfind("-") + + if last_index != -1: # Check if : exists and it should + # Construct the new string + new_package_name = ( + package_name[:last_index] + + ":" + + package_name[last_index + len("-"):] + ) + return new_package_name + + return package_name + +@lru_cache(maxsize=65536) +def extract_jar_name(source: str) -> str: + parts = source.split('/') + + if len(parts) > 1: + return _convert_to_maven_artifact(parts[1].rstrip("-sources")) + + return "" + +def is_maven_gav(text: str) -> bool: + """ + Return True iff `s` is a Maven GAV in the form GroupId:ArtifactId:Version. + Examples of valid inputs: + - "commons-beanutils:commons-beanutils:1.9.4" + - "org.apache.commons:commons-lang3:3.14.0" + - "com.example:my-lib:1.0.0-SNAPSHOT" + """ + text = text.strip() + if ":" not in text: # quick reject + return False + return _GAV_RE.match(text) is not None + +def collect_fields_from_types( + src: str, + include_nested: bool = True, + include_anonymous: bool = True, +) -> Dict[str, List[Tuple[str, str]]]: + """ + Faster single-pass walker: + - Masks comments and string/char/text-blocks once. + - Precomputes global () and {} match maps (O(1) lookups afterward). + - Finds named types and (optionally) anonymous classes. + - For each non-enum type, collects ONLY field declarations (non-primitive, non-wrapper). + - Recurse into nested/local types without re-scanning the whole file. + Returns: { class_simple_name_or_@MD5 : [(fieldName, fieldType), ...] } + """ + + s = src + n = len(s) + out: Dict[str, List[Tuple[str, str]]] = {} + + startswith = s.startswith + find = s.find + + WRAPS = {"Boolean","Byte","Character","Short","Integer","Long","Float","Double","Void"} + MODS = { + "public","protected","private","abstract","static","final", + "transient","volatile","strictfp","synchronized","default","native" + } + MRK_SET = frozenset("(){}=;") + + # ---------- 0) Mask comments & strings with spaces (same length) ---------- + def mask_code(src: str) -> str: + m = list(src) + i = 0; N = len(src) + + def zap(a: int, b: int): + if a < b: m[a:b] = ' ' * (b - a) + + while i < N: + ch = src[i] + # line comment + if ch == '/' and i + 1 < N and src[i+1] == '/': + j = find('\n', i + 2) + if j == -1: j = N + zap(i, j); i = j; continue + # block comment + if ch == '/' and i + 1 < N and src[i+1] == '*': + j = find('*/', i + 2) + j = N if j == -1 else j + 2 + zap(i, j); i = j; continue + # text block + if startswith('"""', i): + j = i + 3 + while True: + k = find('"""', j) + if k == -1: + zap(i, N); i = N; break + # check escapes before """ + t = k - 1; back = 0 + while t >= 0 and src[t] == '\\': + back += 1; t -= 1 + j = k + 3 + if back % 2 == 0: + zap(i, j); i = j; break + continue + # string literal + if ch == '"': + j = i + 1 + while j < N: + c = src[j]; j += 1 + if c == '\\': j += 1 + elif c == '"': break + zap(i, j); i = j; continue + # char literal + if ch == "'": + j = i + 1 + while j < N: + c = src[j]; j += 1 + if c == '\\': j += 1 + elif c == "'": break + zap(i, j); i = j; continue + i += 1 + return ''.join(m) + + m = mask_code(s) + + # ---------- 1) Precompute () and {} match maps (O(n)) ---------- + # This removes repeated O(k) walks for every match_brace/paren call. + paren_match: Dict[int, int] = {} + brace_match: Dict[int, int] = {} + ps = [] + bs = [] + for idx, ch in enumerate(m): + if ch == '(': + ps.append(idx) + elif ch == ')': + if ps: + j = ps.pop() + paren_match[j] = idx + elif ch == '{': + bs.append(idx) + elif ch == '}': + if bs: + j = bs.pop() + brace_match[j] = idx + + # ---------- tiny lexing helpers (ASCII fast-path) ---------- + def is_id_start(ch: str) -> bool: + o = ord(ch) + if o < 128: + return ch == '_' or ch == '$' or (65 <= o <= 90) or (97 <= o <= 122) + return ch.isalpha() + + def is_id_part(ch: str) -> bool: + o = ord(ch) + if o < 128: + return (ch == '_' or ch == '$' or (48 <= o <= 57) or + (65 <= o <= 90) or (97 <= o <= 122)) + return ch.isalnum() + + def skip_ws(i: int) -> int: + while i < n and m[i] <= ' ': + i += 1 + return i + + # O(1) using precomputed maps; fall back (very rare) if unmatched. + def match_paren(i: int) -> int: + j = paren_match.get(i) + if j is not None: return j + # fallback (degenerate/patched sources) + depth = 1; i += 1 + while i < n and depth > 0: + ch = m[i] + if ch == '(': + depth += 1 + elif ch == ')': + depth -= 1 + i += 1 + return i - 1 + + def match_brace(i: int) -> int: + j = brace_match.get(i) + if j is not None: return j + depth = 1; i += 1 + while i < n and depth > 0: + ch = m[i] + if ch == '{': depth += 1 + elif ch == '}': depth -= 1 + i += 1 + return i - 1 + + def match_angles(i: int) -> int: + depth = 1; i += 1 + while i < n and depth > 0: + ch = m[i] + if ch == '<': depth += 1 + elif ch == '>': depth -= 1 + i += 1 + return i - 1 + + def read_ident(i: int, end: int): + if i < end and is_id_start(s[i]): + j = i + 1 + while j < end and is_id_part(s[j]): j += 1 + return s[i:j], j + return None, i + + def collapse_ws(t: str) -> str: + return " ".join(t.split()) + + def base_type_of(typ: str) -> str: + if not typ: return "" + depth = 0; out = [] + for ch in typ: + if ch == '<': depth += 1 + elif ch == '>': depth = depth-1 if depth else 0 + elif depth == 0: out.append(ch) + t = "".join(out).strip() + if t.endswith("..."): t = t[:-3].rstrip() + while t.endswith("[]"): t = t[:-2].rstrip() + return collapse_ws(t) + + def keep_type(typ: str) -> bool: + if not typ: return False + base = base_type_of(typ) + if not base: return False + simple = base.split(".")[-1] + if simple.lower() in JAVA_METHOD_PRIM_TYPES: return False + if simple in WRAPS: return False + return True + + def parse_field_header(header: str) -> List[Tuple[str, str]]: + i, L = 0, len(header) + + def ws(k): + while k < L and header[k].isspace(): k += 1 + return k + + def rid(k): + if k < L and is_id_start(header[k]): + j = k + 1 + while j < L and is_id_part(header[j]): j += 1 + return header[k:j], j + return None, k + + def paren(k): + depth = 1; k += 1 + while k < L and depth > 0: + c = header[k] + if c == '(': + depth += 1 + elif c == ')': + depth -= 1 + elif c in ('"', "'"): + k += 1 + while k < L: + d = header[k]; k += 1 + if d == '\\': k += 1 + elif d == '"': break + k += 1 + return k - 1 + + def angles(k): + depth = 1; k += 1 + while k < L and depth > 0: + c = header[k] + if c == '<': depth += 1 + elif c == '>': depth -= 1 + k += 1 + return k - 1 + + # drop leading annotations on type + i = ws(i) + while i < L and header[i] == JAVA_ANNOTATION_SYMBOL: + i += 1 + _, i = rid(i); i = ws(i) + while i < L and header[i] == '.': + _, i = rid(i + 1); i = ws(i) + if i < L and header[i] == '(': + i = paren(i) + 1 + i = ws(i) + + # read base type + generics + []/... + type_start = i + _, j = rid(i) + if j == i: + return [] + i = ws(j) + if i < L and header[i] == '<': + i = angles(i) + 1; i = ws(i) + while i < L and header[i] == '.': + _, i = rid(i + 1); i = ws(i) + if i < L and header[i] == '<': + i = angles(i) + 1; i = ws(i) + while True: + if header.startswith('[]', i): i += 2; i = ws(i); continue + if header.startswith('...', i): i += 3; i = ws(i); continue + break + base = header[type_start:i].strip() + + out_fields: List[Tuple[str, str]] = [] + + def var_and_suffix(k): + k = ws(k) + while k < L and header[k] == JAVA_ANNOTATION_SYMBOL: + k += 1 + _, k = rid(k); k = ws(k) + while k < L and header[k] == '.': + _, k = rid(k + 1); k = ws(k) + if k < L and header[k] == '(': + k = paren(k) + 1 + k = ws(k) + name, j2 = rid(k) + if not name: return None, "", k + k = ws(j2) + suf = "" + while header.startswith('[]', k): + suf += "[]"; k += 2; k = ws(k) + return name, suf, k + + i = ws(i) + v, suf, i2 = var_and_suffix(i) + if not v: return [] + out_fields.append((v, (base + suf).strip())) + i = i2 + while i < L and header[i] == ',': + i += 1 + v, suf, i2 = var_and_suffix(i) + if not v: break + out_fields.append((v, (base + suf).strip())) + i = i2 + return out_fields + + def next_marker(i: int, end: int, ms=MRK_SET): + while i < end: + i = skip_ws(i) + if i >= end: break + c = m[i] + if c in ms: return i, c + i += 1 + return -1, '' + + def scan_to_toplevel_semicolon(p: int, end: int) -> int: + par=brc=brk=0 + while p < end: + ch = m[p] + if ch == '(': + par += 1 + elif ch == ')': + if par: par -= 1 + elif ch == '{': + brc += 1 + elif ch == '}': + if brc: brc -= 1 + elif ch == '[': + brk += 1 + elif ch == ']': + if brk: brk -= 1 + elif ch == ';' and par == 0 and brc == 0 and brk == 0: + return p + p += 1 + return -1 + + # -------- scan anonymous classes inside a range -------- + def scan_anonymous_range(a: int, b: int): + if not include_anonymous: + return + i = a + while i < b: + i = skip_ws(i) + if i >= b: break + # fast token check for 'new' (masked buffer, so no comments/strings) + if m[i] != 'n': + i += 1; continue + if i + 3 <= b and m[i:i+3] == 'new' and (i == a or not is_id_part(s[i-1])): + j = i + 3 + if j < b and is_id_part(s[j]): # avoid 'newX' + i += 1; continue + + anon_start = i + k = skip_ws(j) + # type-use annotations + while k < b and s[k] == JAVA_ANNOTATION_SYMBOL: + k += 1 + nm, k = read_ident(k, b) + while k < b and s[k] == '.': + nm2, k = read_ident(k + 1, b) + if not nm2: break + k = skip_ws(k) + if k < b and m[k] == '(': + k = match_paren(k) + 1 + k = skip_ws(k) + + nm, k2 = read_ident(k, b) + if not nm: + i += 1; continue + k = k2 + while True: + k = skip_ws(k) + if k < b and m[k] == '<': + k = match_angles(k) + 1; continue + if k < b and m[k] == '.': + nm2, k2 = read_ident(k + 1, b) + if not nm2: break + k = k2; continue + break + k = skip_ws(k) + if k < b and m[k] == '(': + k = match_paren(k) + 1 + k = skip_ws(k) + if k < b and m[k] == '{': + r = match_brace(k) + segment = s[anon_start:r + 1] + md5_hex = hashlib.md5(segment.encode("utf-8")).hexdigest() + anon_name = f"@{md5_hex}" + collect_type('class', anon_name, k, r) + i = r + 1 + continue + i += 1 + + # ---------------- core: scan whole file, process types inline ---------------- + def collect_type(kind: str, name: str, lb: int, rb: int): + if kind == 'enum': + return + fields: List[Tuple[str, str]] = [] + i, end = lb + 1, rb + + while i < end: + i = skip_ws(i) + if i >= end: break + + ch = m[i] + + # initializer blocks + if ch == '{': + rblk = match_brace(i) + scan_anonymous_range(i, rblk) + i = rblk + 1 + continue + + # static { ... } + if is_id_start(s[i]): + j = i + 1 + while j < end and is_id_part(s[j]): j += 1 + if s[i:j] == 'static': + j2 = skip_ws(j) + if j2 < end and m[j2] == '{': + rblk = match_brace(j2) + scan_anonymous_range(j2, rblk) + i = rblk + 1 + continue + + # normalize: skip annotations & modifiers + start = i + tmp = start + while True: + t2 = skip_ws(tmp) + if t2 != tmp: tmp = t2; continue + if tmp < end and s[tmp] == JAVA_ANNOTATION_SYMBOL: + j = tmp + 1 + nm, j = read_ident(j, end) + while j < end and s[j] == '.': + nm2, j = read_ident(j + 1, end) + if not nm2: break + j = skip_ws(j) + if j < end and m[j] == '(': + j = match_paren(j) + 1 + tmp = j; continue + if tmp < end and is_id_start(s[tmp]): + t3 = tmp + 1 + while t3 < end and is_id_part(s[t3]): t3 += 1 + if s[tmp:t3] in MODS: + tmp = t3; continue + break + start = tmp + + # nested named type at normalized start (e.g., 'public class X { ... }') + if include_nested: + j = start + if j < end and is_id_start(s[j]): + j2 = j + 1 + while j2 < end and is_id_part(s[j2]): j2 += 1 + tok2 = s[j:j2] + if tok2 in ('class','interface','enum','record'): + m2 = skip_ws(j2) + name_start = m2 + while m2 < end and is_id_part(s[m2]): m2 += 1 + if m2 > name_start: + tname = s[name_start:m2] + m2 = skip_ws(m2) + if m2 < end and m[m2] == '<': m2 = match_angles(m2) + 1; m2 = skip_ws(m2) + if tok2 == 'record' and m2 < end and m[m2] == '(': + m2 = match_paren(m2) + 1; m2 = skip_ws(m2) + # use .find to jump to body quickly + body_open = m.find('{', m2, end) + if body_open != -1: + r2 = match_brace(body_open) + collect_type(tok2, tname, body_open, r2) + i = r2 + 1 + continue + + # decide by first structural marker + pos, mrk = next_marker(start, end, MRK_SET) + if pos == -1: break + + if mrk == '(': + # method/ctor: scan body for anonymous, then skip body + pclose = match_paren(pos) + q = skip_ws(pclose + 1) + # optional throws … + if q + 6 <= end and s[q:q+6] == 'throws' and (q + 6 == end or not is_id_part(s[q+6])): + j = skip_ws(q + 6) + while j < end and m[j] not in '{;': + if m[j] == '<': j = match_angles(j) + 1 + elif m[j] == '[' and j + 1 < end and m[j+1] == ']': j += 2 + elif is_id_start(s[j]): + k = j + 1 + while k < end and is_id_part(s[k]): k += 1 + j = k + elif m[j] in '.,': j += 1 + else: break + j = skip_ws(j) + q = skip_ws(j) + if q < end and m[q] == '{': + rbody = match_brace(q) + scan_anonymous_range(q + 1, rbody) + i = rbody + 1 + elif q < end and m[q] == ';': + i = q + 1 + else: + semi = scan_to_toplevel_semicolon(q, end) + i = semi + 1 if semi != -1 else end + continue + + if mrk == '{': + rblk = match_brace(pos) + scan_anonymous_range(pos, rblk) + i = rblk + 1 + continue + + if mrk in ('=', ';'): + if mrk == '=': + semi = scan_to_toplevel_semicolon(pos + 1, end) + if semi == -1: semi = end - 1 + scan_anonymous_range(pos + 1, semi) # anon in initializer + header = s[start:pos].strip() + for (vn, vt) in parse_field_header(header): + if keep_type(vt): + fields.append((vn, vt)) + i = semi + 1 + continue + else: # ';' + header = s[start:pos].strip() + for (vn, vt) in parse_field_header(header): + if keep_type(vt): + fields.append((vn, vt)) + i = pos + 1 + continue + + i = pos + 1 + + out[name] = fields + + # ---- top-level scan ---- + i = 0 + if ("class" not in s and "interface" not in s and "enum" not in s and "record" not in s and "new" not in s): + return out + + while i < n: + i = skip_ws(i) + if i >= n: break + if not is_id_start(s[i]): + i += 1; continue + + j = i + 1 + while j < n and is_id_part(s[j]): j += 1 + tok = s[i:j] + + if tok in ('class','interface','enum','record'): + k = skip_ws(j) + name_start = k + while k < n and is_id_part(s[k]): k += 1 + if k == name_start: i = j; continue + name = s[name_start:k] + k = skip_ws(k) + if k < n and m[k] == '<': k = match_angles(k) + 1; k = skip_ws(k) + if tok == 'record' and k < n and m[k] == '(': + k = match_paren(k) + 1; k = skip_ws(k) + body_open = m.find('{', k, n) + if body_open != -1: + rb = match_brace(body_open) + collect_type(tok, name, body_open, rb) + i = rb + 1 + continue + + if include_anonymous and tok == 'new': + anon_start = i + k = skip_ws(j) + # type-use annotations + while k < n and s[k] == JAVA_ANNOTATION_SYMBOL: + nm, k = read_ident(k + 1, n) + while k < n and s[k] == '.': + nm2, k = read_ident(k + 1, n) + if not nm2: break + k = skip_ws(k) + if k < n and m[k] == '(': + k = match_paren(k) + 1 + k = skip_ws(k) + + nm, k2 = read_ident(k, n) + if nm: + k = k2 + while True: + k = skip_ws(k) + if k < n and m[k] == '<': + k = match_angles(k) + 1; continue + if k < n and m[k] == '.': + nm2, k2 = read_ident(k + 1, n) + if not nm2: break + k = k2; continue + break + k = skip_ws(k) + if k < n and m[k] == '(': + k = match_paren(k) + 1 + k = skip_ws(k) + if k < n and m[k] == '{': + rb = match_brace(k) + segment = s[anon_start:rb + 1] + md5_hex = hashlib.md5(segment.encode("utf-8")).hexdigest() + anon_name = f"@{md5_hex}" + collect_type('class', anon_name, k, rb) + i = rb + 1 + continue + + i += 1 + + return out + +@lru_cache(maxsize=100000) +def is_java_method(source: str) -> bool: + s = source + + # avoid scanning twice for '(' and '->' + has_paren = "(" in s + has_arrow = "->" in s + if not has_paren and not has_arrow: + return False + + n = len(s) + find_in_source = s.find + startswith = s.startswith + ws = JAVA_METHOD_WS_CHARS + + # --- FAST REJECT #1: skip leading whitespace/comments and reject FreeMarker templates --- + # This avoids scanning huge .ftl-like files (your ArrowType.java template input). + j = 0 + while j < n: + ch = s[j] + if ch in ws: + j += 1 + continue + if ch == "/" and j + 1 < n: + c2 = s[j + 1] + if c2 == "/": + nl = find_in_source("\n", j + 2) + j = n if nl == -1 else nl + 1 + continue + if c2 == "*": + end = find_in_source("*/", j + 2) + j = n if end == -1 else end + 2 + continue + break + + if j < n and (s.startswith("<#", j) or s.startswith("<@", j)): + return False + + # --- FAST REJECT #2: compilation unit headers ("package"/"import") --- + # Safe for your prior examples (method/ctor/lambda snippets) and avoids full-file scans. + leading_ident_match = _LEADING_IDENT_RE.match(s, j) + if leading_ident_match and leading_ident_match.group(0) in ("package", "import"): + return False + + # Conservative “might be a top-level type decl somewhere” flag: + # Use a real "type header" detector, not substring "class" (to avoid `.class` false positives). + maybe_type_decl = bool(has_arrow and _JAVA_TYPE_HEADER_RE.search(s)) + + i = 0 + + NORMAL, LINE, BLOCK, STRING, CHAR, TEXT = 0, 1, 2, 3, 4, 5 + state = NORMAL + brace = 0 + saw_lambda = False + + def is_id_start(ch: str) -> bool: + return ch == "_" or ("A" <= ch <= "Z") or ("a" <= ch <= "z") or (ch >= "\u0080" and ch.isalpha()) + + def is_id_part(ch: str) -> bool: + return ( + ch == "_" or ch == "$" or + ("A" <= ch <= "Z") or ("a" <= ch <= "z") or ("0" <= ch <= "9") or + (ch >= "\u0080" and ch.isalnum()) + ) + + def _skip_quoted(jq: int, quote: str) -> int: + while jq < n: + end = find_in_source(quote, jq) + if end == -1: + return n + k = end - 1 + back = 0 + while k >= 0 and s[k] == "\\": + back += 1 + k -= 1 + if back % 2 == 0: + return end + 1 + jq = end + 1 + return n + + def skip_paren(jp: int) -> int: + depth = 1 + st = NORMAL + while jp < n and depth > 0: + chp = s[jp] + if st == NORMAL: + if chp == "/": + if jp + 1 < n: + c2 = s[jp + 1] + if c2 == "/": + st = LINE + jp += 2 + continue + if c2 == "*": + st = BLOCK + jp += 2 + continue + elif chp == '"': + if jp + 2 < n and s[jp + 1] == '"' and s[jp + 2] == '"': + st = TEXT + jp += 3 + continue + st = STRING + jp += 1 + continue + elif chp == "'": + st = CHAR + jp += 1 + continue + elif chp == "(": + depth += 1 + jp += 1 + continue + elif chp == ")": + depth -= 1 + jp += 1 + continue + else: + jp += 1 + continue + elif st == LINE: + nl = find_in_source("\n", jp) + jp = n if nl == -1 else nl + 1 + st = NORMAL + elif st == BLOCK: + end = find_in_source("*/", jp) + jp = n if end == -1 else end + 2 + st = NORMAL + elif st == STRING: + jp = _skip_quoted(jp, '"') + st = NORMAL + elif st == CHAR: + jp = _skip_quoted(jp, "'") + st = NORMAL + else: # TEXT + while True: + end = find_in_source('"""', jp) + if end == -1: + jp = n + break + k = end - 1 + back = 0 + while k >= 0 and s[k] == "\\": + back += 1 + k -= 1 + jp = end + 3 + if back % 2 == 0: + break + st = NORMAL + return jp + + def read_ident(jr: int) -> tuple[Optional[str], int]: + if jr >= n or not is_id_start(s[jr]): + return None, jr + k = jr + 1 + while k < n and is_id_part(s[k]): + k += 1 + return s[jr:k], k + + def read_qual_ident(jr: int) -> tuple[Optional[str], int]: + name, k = read_ident(jr) + if not name: + return None, jr + while k < n and s[k] == ".": + nm2, k2 = read_ident(k + 1) + if not nm2: + break + k = k2 + return name, k + + def skip_ws_comments(jw: int) -> int: + while jw < n: + chw = s[jw] + if chw in ws: + jw += 1 + continue + if chw == "/" and jw + 1 < n: + c2 = s[jw + 1] + if c2 == "/": + nl = find_in_source("\n", jw + 2) + jw = n if nl == -1 else nl + 1 + continue + if c2 == "*": + end = find_in_source("*/", jw + 2) + jw = n if end == -1 else end + 2 + continue + break + return jw + + def has_dot_between(jd: int, kd: int) -> bool: + jd = skip_ws_comments(jd) + while jd < kd: + chd = s[jd] + if chd == ".": + return True + if chd in ws: + jd += 1 + continue + if chd == "/" and jd + 1 < n: + c2 = s[jd + 1] + if c2 == "/": + nl = find_in_source("\n", jd + 2) + jd = n if nl == -1 else nl + 1 + continue + if c2 == "*": + end = find_in_source("*/", jd + 2) + jd = n if end == -1 else end + 2 + continue + jd += 1 + return False + + def plausible_method_decl(name_start: int) -> bool: + jx = name_start + j0 = max(s.rfind("\n", 0, jx), s.rfind("{", 0, jx), s.rfind(";", 0, jx)) + 1 + jx = skip_ws_comments(j0) + + broken = False + + while jx < name_start: + if s[jx] == JAVA_ANNOTATION_SYMBOL: + _, jx = read_qual_ident(jx + 1) + jx = skip_ws_comments(jx) + if jx < n and s[jx] == "(": + jx = skip_paren(jx + 1) + jx = skip_ws_comments(jx) + continue + + if s[jx] == "<": + depth = 1 + jx += 1 + while jx < n and depth > 0: + if s[jx] == "<": + depth += 1 + elif s[jx] == ">": + depth -= 1 + jx += 1 + jx = skip_ws_comments(jx) + continue + + tok, k = read_ident(jx) + if tok: + if tok in JAVA_METHOD_PRIM_TYPES: + return True + if tok in JAVA_METHOD_METH_MODS: + jx = skip_ws_comments(k) + continue + if tok == "non": + k2 = skip_ws_comments(k) + if k2 < n and s[k2] == "-": + k3 = skip_ws_comments(k2 + 1) + id2, j2 = read_ident(k3) + if id2 == "sealed": + jx = skip_ws_comments(j2) + continue + + jx = k + jx = skip_ws_comments(jx) + + if jx < n and s[jx] == "<": + depth = 1 + jx += 1 + while jx < n and depth > 0: + if s[jx] == "<": + depth += 1 + elif s[jx] == ">": + depth -= 1 + jx += 1 + jx = skip_ws_comments(jx) + + while jx + 1 < n and s[jx] == "[" and s[jx + 1] == "]": + jx += 2 + jx = skip_ws_comments(jx) + + tcount = 0 + while tcount < 4: + jx = skip_ws_comments(jx) + if jx < n and s[jx] == "&": + jx = skip_ws_comments(jx + 1) + nm2, k2 = read_ident(jx) + if not nm2: + break + jx = k2 + jx = skip_ws_comments(jx) + if jx < n and s[jx] == "<": + depth = 1 + jx += 1 + while jx < n and depth > 0: + if s[jx] == "<": + depth += 1 + elif s[jx] == ">": + depth -= 1 + jx += 1 + jx = skip_ws_comments(jx) + while jx + 1 < n and s[jx] == "[" and s[jx + 1] == "]": + jx += 2 + jx = skip_ws_comments(jx) + tcount += 1 + else: + break + + return True + + broken = True + break + + return not broken # constructor + + while i < n: + if state == NORMAL: + if brace > 0: + m = _BRACE_NORMAL_SPECIAL_RE.search(s, i) + if not m: + break + i = m.start() + ch = s[i] + + if ch == "-" and i + 1 < n and s[i + 1] == ">": + saw_lambda = True + if not maybe_type_decl: + return True + i += 2 + continue + + if ch == "/": + if i + 1 < n: + c2 = s[i + 1] + if c2 == "/": + state = LINE + i += 2 + continue + if c2 == "*": + state = BLOCK + i += 2 + continue + i += 1 + continue + + if ch == '"': + if i + 2 < n and s[i + 1] == '"' and s[i + 2] == '"': + state = TEXT + i += 3 + continue + state = STRING + i += 1 + continue + + if ch == "'": + state = CHAR + i += 1 + continue + + if ch == "{": + brace += 1 + i += 1 + continue + if ch == "}": + brace -= 1 + if brace < 0: + brace = 0 + i += 1 + continue + + i += 1 + continue + + ch = s[i] + + if ch == "-" and i + 1 < n and s[i + 1] == ">": + saw_lambda = True + if not maybe_type_decl: + return True + i += 2 + continue + + if ch == "/": + if i + 1 < n: + c2 = s[i + 1] + if c2 == "/": + state = LINE + i += 2 + continue + if c2 == "*": + state = BLOCK + i += 2 + continue + elif ch == '"': + if i + 2 < n and s[i + 1] == '"' and s[i + 2] == '"': + state = TEXT + i += 3 + continue + state = STRING + i += 1 + continue + elif ch == "'": + state = CHAR + i += 1 + continue + elif ch == "{": + brace += 1 + i += 1 + continue + elif ch == "}": + brace = max(0, brace - 1) + i += 1 + continue + + if ch in ws: + i += 1 + continue + + if ch == JAVA_ANNOTATION_SYMBOL: + _, i = read_qual_ident(i + 1) + i = skip_ws_comments(i) + if i < n and s[i] == "(": + i = skip_paren(i + 1) + continue + + if is_id_start(ch): + ident, j2 = read_ident(i) + + if ident in JAVA_METHOD_CLASS_MODS: + i = skip_ws_comments(j2) + continue + + if ident == "non": + k = skip_ws_comments(j2) + if k < n and s[k] == "-": + k2 = skip_ws_comments(k + 1) + id2, j3 = read_ident(k2) + if id2 == "sealed": + i = skip_ws_comments(j3) + continue + + if ident in JAVA_METHOD_CLASS_KINDS: + i2 = skip_ws_comments(j2) + name, j4 = read_ident(i2) + if name: + return False + i = j2 + continue + + i = j2 + continue + + if ch == "(": + jx = i - 1 + while jx >= 0 and s[jx] in ws: + jx -= 1 + end = jx + 1 + while jx >= 0 and is_id_part(s[jx]): + jx -= 1 + name = s[jx + 1:end] if end > (jx + 1) else None + + if name and (name not in JAVA_METHOD_CTRL_WORDS) and not has_dot_between(end, i): + k = skip_paren(i + 1) + q = skip_ws_comments(k) + + if startswith("throws", q) and (q + 6 == n or not is_id_part(s[q + 6])): + q = skip_ws_comments(q + 6) + while q < n and s[q] not in "{;": + cq = s[q] + if cq in ws or cq in ".,": # whitespace, commas, dots + q += 1 + continue + if cq == "<": + depth = 1 + q += 1 + while q < n and depth > 0: + if s[q] == "<": + depth += 1 + elif s[q] == ">": + depth -= 1 + q += 1 + continue + if is_id_start(cq): + _, q = read_ident(q) + continue + if cq == "[" and q + 1 < n and s[q + 1] == "]": + q += 2 + continue + break + q = skip_ws_comments(q) + + if q < n and s[q] in "{;": + if plausible_method_decl(jx + 1): + return True + + i += 1 + continue + + i += 1 + continue + + elif state == LINE: + nl = find_in_source("\n", i) + i = n if nl == -1 else nl + 1 + state = NORMAL + continue + + elif state == BLOCK: + end = find_in_source("*/", i) + i = n if end == -1 else end + 2 + state = NORMAL + continue + + elif state == STRING: + i = _skip_quoted(i, '"') + state = NORMAL + continue + + elif state == CHAR: + i = _skip_quoted(i, "'") + state = NORMAL + continue + + else: # TEXT + while True: + end = find_in_source('"""', i) + if end == -1: + i = n + break + k = end - 1 + back = 0 + while k >= 0 and s[k] == "\\": + back += 1 + k -= 1 + i = end + 3 + if back % 2 == 0: + break + state = NORMAL + continue + + return saw_lambda + +def is_java_type(source: str) -> bool: + """ + Extremely fast: single pass, early exit. + Detects top-level class/interface/enum/record only. + Avoids misclassifying method snippets. + """ + s = source + n = len(s) + i = 0 + + if ("class" not in s and "interface" not in s and + "enum" not in s and "record" not in s and "(" not in s): + return False + + NORMAL, LINE, BLOCK, STRING, CHAR, TEXT = 0, 1, 2, 3, 4, 5 + state = NORMAL + brace = 0 + + ws = " \t\r\n\f\v" + class_kinds = JAVA_METHOD_CLASS_KINDS + class_mods = JAVA_METHOD_CLASS_MODS + meth_ctrl = JAVA_METHOD_CTRL_WORDS + + def is_ws(ch): return ch in ws + def is_id_start(ch): return ch == '_' or ch == '$' or ('A' <= ch <= 'Z') or ('a' <= ch <= 'z') or (ch >= '\u0080' and ch.isalpha()) + def is_id_part(ch): return ch == '_' or ch == '$' or ('A' <= ch <= 'Z') or ('a' <= ch <= 'z') or ('0' <= ch <= '9') or (ch >= '\u0080' and ch.isalnum()) + + def read_ident(j: int): + if j >= n or not is_id_start(s[j]): return None, j + k = j + 1 + while k < n and is_id_part(s[k]): k += 1 + return s[j:k], k + + def skip_ws_comments(j: int) -> int: + while j < n: + ch = s[j] + if is_ws(ch): + j += 1 + continue + if ch == '/' and j + 1 < n: + c2 = s[j + 1] + if c2 == '/': + nl = s.find('\n', j + 2) + j = n if nl == -1 else nl + 1 + continue + if c2 == '*': + end = s.find("*/", j + 2) + j = n if end == -1 else end + 2 + continue + break + return j + + def skip_paren_balanced(j: int) -> int: + depth = 1 + st = NORMAL + while j < n and depth > 0: + ch = s[j] + if st == NORMAL: + if ch == '/': + # FIX: if it's not // or /*, it's an operator; advance! + if j + 1 < n and s[j+1] == '/': + nl = s.find('\n', j + 2) + j = n if nl == -1 else nl + 1 + continue + if j + 1 < n and s[j+1] == '*': + end = s.find('*/', j + 2) + j = n if end == -1 else end + 2 + continue + j += 1 + continue + + elif ch == '"': + if j + 2 < n and s[j+1] == '"' and s[j+2] == '"': + st = TEXT; j += 3; continue + st = STRING; j += 1; continue + elif ch == "'": + st = CHAR; j += 1; continue + elif ch == '(': + depth += 1; j += 1; continue + elif ch == ')': + depth -= 1; j += 1; continue + else: + j += 1; continue + + elif st == STRING: + while j < n: + c2 = s[j]; j += 1 + if c2 == '\\': j += 1 + elif c2 == '"': break + st = NORMAL + + elif st == CHAR: + while j < n: + c2 = s[j]; j += 1 + if c2 == '\\': j += 1 + elif c2 == "'": break + st = NORMAL + + else: # TEXT + while True: + end = s.find('"""', j) + if end == -1: j = n; break + k = end - 1; back = 0 + while k >= 0 and s[k] == '\\': back += 1; k -= 1 + j = end + 3 + if back % 2 == 0: break + st = NORMAL + + return j + + def skip_angles_balanced(j: int) -> int: + depth = 1; j += 1 + while j < n and depth > 0: + ch = s[j] + if ch == '<': depth += 1 + elif ch == '>': depth -= 1 + elif ch == '/': + if j + 1 < n and s[j+1] == '/': + nl = s.find('\n', j + 2) + j = n if nl == -1 else nl + 1 + continue + if j + 1 < n and s[j+1] == '*': + end = s.find('*/', j + 2) + j = n if end == -1 else end + 2 + continue + j += 1 + return j + + def has_dot_between(a: int, b: int) -> bool: + j = a + while j < b: + ch = s[j] + if ch == '.': return True + if is_ws(ch): + j += 1; continue + if ch == '/' and j + 1 < n: + if s[j+1] == '/': + nl = s.find('\n', j + 2) + j = n if nl == -1 else nl + 1 + continue + if s[j+1] == '*': + end = s.find('*/', j + 2) + j = n if end == -1 else end + 2 + continue + j += 1 + return False + + # NEW: skip a single qualified type (with optional annotations, generics, array suffix) + def skip_one_type(p: int) -> int: + p = skip_ws_comments(p) + # annotations on the type + while p < n and s[p] == JAVA_ANNOTATION_SYMBOL: + _, p = read_ident(p + 1) + while p < n and s[p] == '.': + _, p = read_ident(p + 1) + p = skip_ws_comments(p) + if p < n and s[p] == '(': + p = skip_paren_balanced(p + 1) + p = skip_ws_comments(p) + # qualified name + optional generics per segment + nm, p2 = read_ident(p) + if not nm: return p + p = p2; p = skip_ws_comments(p) + if p < n and s[p] == '<': + p = skip_angles_balanced(p); p = skip_ws_comments(p) + while p < n and s[p] == '.': + nm2, p2 = read_ident(p + 1) + if not nm2: break + p = p2; p = skip_ws_comments(p) + if p < n and s[p] == '<': + p = skip_angles_balanced(p); p = skip_ws_comments(p) + # array suffix(es) + while p + 1 < n and s[p] == '[' and s[p+1] == ']': + p += 2; p = skip_ws_comments(p) + return p + + # NEW: skip comma-separated type list after extends/implements/permits + def skip_type_list(p: int) -> int: + p = skip_ws_comments(p) + while p < n: + p2 = skip_one_type(p) + if p2 == p: break + p = skip_ws_comments(p2) + if p < n and s[p] == ',': + p = skip_ws_comments(p + 1) + continue + break + return p + + while i < n: + ch = s[i] + + if state == NORMAL: + if ch == '/': + if i + 1 < n and s[i+1] == '/': + state = LINE; i += 2; continue + if i + 1 < n and s[i+1] == '*': + state = BLOCK; i += 2; continue + elif ch == '"': + if i + 2 < n and s[i+1] == '"' and s[i+2] == '"': + state = TEXT; i += 3; continue + state = STRING; i += 1; continue + elif ch == "'": + state = CHAR; i += 1; continue + elif ch == '{': + brace += 1; i += 1; continue + elif ch == '}': + brace = max(0, brace - 1); i += 1; continue + + if brace != 0: + i += 1; continue + if is_ws(ch): + i += 1; continue + + if ch == JAVA_ANNOTATION_SYMBOL: + i += 1 + _, i = read_ident(i) + while i < n and s[i] == '.': + _, i = read_ident(i + 1) + i = skip_ws_comments(i) + if i < n and s[i] == '(': + i = skip_paren_balanced(i + 1) + continue + + if is_id_start(ch): + tok, j = read_ident(i) + + # early: method/ctor at top level => not a class + k = skip_ws_comments(j) + if k < n and s[k] == '(' and tok not in meth_ctrl and not has_dot_between(j, k): + r = skip_paren_balanced(k + 1) + q = skip_ws_comments(r) + if q + 6 <= n and s[q:q+6] == 'throws' and (q + 6 == n or not is_id_part(s[q+6])): + q = skip_ws_comments(q + 6) + while q < n and s[q] not in '{;': + if is_ws(s[q]) or s[q] in '.,': q += 1; continue + if s[q] == '<': + q = skip_angles_balanced(q); q = skip_ws_comments(q); continue + if is_id_start(s[q]): + _, q = read_ident(q); q = skip_ws_comments(q); continue + if s[q] == '[' and q + 1 < n and s[q+1] == ']': + q += 2; continue + break + q = skip_ws_comments(q) + if q < n and s[q] in '{;': + return False + + # class modifiers (incl. "non-sealed") + if tok in class_mods: + i = skip_ws_comments(j) + if tok == "non" and i < n and s[i] == '-': + i = skip_ws_comments(i + 1) + t2, j2 = read_ident(i) + if t2 == "sealed": + i = skip_ws_comments(j2) + continue + + if tok in class_kinds: + p = skip_ws_comments(j) + name, p2 = read_ident(p) + if not name: + i = j; continue + p = skip_ws_comments(p2) + if tok == 'record' and p < n and s[p] == '(': + p = skip_paren_balanced(p + 1) + p = skip_ws_comments(p) + if p < n and s[p] == '<': + p = skip_angles_balanced(p) + p = skip_ws_comments(p) + + # allow extends/implements/permits with full type lists + while p < n: + if s.startswith("extends", p) and (p+7 == n or not is_id_part(s[p+7])): + p = skip_type_list(skip_ws_comments(p + 7)); continue + if s.startswith("implements", p) and (p+10 == n or not is_id_part(s[p+10])): + p = skip_type_list(skip_ws_comments(p + 10)); continue + if s.startswith("permits", p) and (p+7 == n or not is_id_part(s[p+7])): + p = skip_type_list(skip_ws_comments(p + 7)); continue + # rare: stray type annotation chunk before the brace + if s[p] == JAVA_ANNOTATION_SYMBOL: + p = skip_ws_comments(p + 1) + _, p = read_ident(p) + while p < n and s[p] == '.': + _, p = read_ident(p + 1) + p = skip_ws_comments(p) + if p < n and s[p] == '(': + p = skip_paren_balanced(p + 1) + p = skip_ws_comments(p); continue + if s[p] == '<': # stray generics chunk + p = skip_angles_balanced(p); p = skip_ws_comments(p); continue + break + + if p < n and s[p] == '{': + return True + + i = j + continue + + i = j + continue + + i += 1 + continue + + elif state == LINE: + nl = s.find('\n', i) + i = n if nl == -1 else nl + 1 + state = NORMAL + continue + + elif state == BLOCK: + end = s.find('*/', i) + i = n if end == -1 else end + 2 + state = NORMAL + continue + + elif state == STRING: + while i < n: + c2 = s[i]; i += 1 + if c2 == '\\': i += 1 + elif c2 == '"': break + state = NORMAL + continue + + elif state == CHAR: + while i < n: + c2 = s[i]; i += 1 + if c2 == '\\': i += 1 + elif c2 == "'": break + state = NORMAL + continue + + else: # TEXT + while True: + end = s.find('"""', i) + if end == -1: i = n; break + k = end - 1; back = 0 + while k >= 0 and s[k] == '\\': back += 1; k -= 1 + i = end + 3 + if back % 2 == 0: break + state = NORMAL + continue + + return False + +def create_inheritance_map(full_source_documents: Collection[Document]) -> dict[Tuple[str, str], List[Tuple[str, str]]]: + """ + Build a jar-aware inheritance/implements map. + + Key: (fqcn, source_path) + Value: ordered, deduped list of (fqcn, source_path) including: + - self + - ancestors (class chain or interface parents) + - implemented interfaces (direct, via ancestors, and via super-interfaces) + - transitive subclasses / implementers (only for types present in the set) + + Jar-aware selection rule for related types: + - If an FQCN has a variant in the same jar as the declaring type, pick that. + - Else if it has exactly one variant globally, pick that. + - Else (multiple variants across jars and none in this jar), skip wiring to avoid ambiguity. + + External/unseen types (e.g., JDK classes/interfaces) are INCLUDED in the + upward/interface lists as placeholders with the current type's source path: + (fqcn_or_token, current_source_of_key) + Placeholders are NOT used to wire reverse edges (subclasses/implementers). + + Parsing guarantees: + - Robust to comments and annotations. + - Explicit-imports-first resolution; star imports allowed. + - Order-preserving, no duplicates. + - Critically, generic type-parameter bounds are ignored: + e.g., in `interface X extends A, B { ... }`, only `A, B` are + treated as super-interfaces. Any `extends` tokens inside the `<...>` generic + parameter list are NOT misinterpreted as inheritance clauses. + - As a final safety net, any residual tokens that still contain the word + `extends` (indicative of generic bounds accidentally leaking through) are dropped. + """ + + # ---------------- helpers ---------------- + + def _balance_body_end(s: str, open_brace_idx: int) -> int: + """Return index just after the matching '}' for body that starts at open_brace_idx.""" + depth = 1 + i = open_brace_idx + 1 + n = len(s) + while i < n and depth: + ch = s[i] + if ch == '{': + depth += 1 + elif ch == '}': + depth -= 1 + i += 1 + return i + + def _find_all_type_decls(src: str) -> List[Dict]: + """ + Return ordered list of type decl dicts with fields: + kind, name, header (text up to '{'), hstart, bstart, bend, enclosing (names list). + Uses a single pass nesting stack to compute 'enclosing' chain. + """ + s = strip_comments_preserving_newlines(src) + decls: List[Dict] = [] + + # First pass: collect raw declarations with their body ranges + for m in TYPE_DECL_RE.finditer(s): + kind, name = m.group(1), m.group(2) + brace = s.find("{", m.end()) + if brace == -1: + continue # malformed; ignore + bend = _balance_body_end(s, brace) + header_text = s[m.start():brace] + decls.append({ + "kind": kind, + "name": name, + "header": re.sub(r"\s+", " ", header_text).strip(), + "hstart": m.start(), + "bstart": brace, + "bend": bend, + }) + + if not decls: + return [] + + # Second pass: assign enclosing chain using a simple interval stack + decls.sort(key=lambda d: d["hstart"]) + stack: List[Dict] = [] + for d in decls: + while stack and d["hstart"] >= stack[-1]["bend"]: + stack.pop() + d["enclosing"] = [x["name"] for x in stack] + stack.append(d) + + return decls + def strip_comments_preserving_newlines(s: str) -> str: + """Remove /* ... */ and // ... comments, preserving newlines for positions.""" + def _blk(m): + t = m.group(0) + return re.sub(r"[^\n]", " ", t) + s2 = re.sub(r"/\*.*?\*/", _blk, s, flags=re.S) + s2 = re.sub(r"//[^\n]*", "", s2) + return s2 + + def find_header_slice_no_comments(src: str) -> str | None: + """ + Strip comments; find first real type header; return text from that point + up to the first '{' that follows it. + """ + s = strip_comments_preserving_newlines(src) + NAME_KIND_RE = re.compile( + r"(?m)^\s*" + r"(?:@[A-Za-z_$][\w$]*(?:\s*\([^()]*\))?\s*)*" # annotations (line-wise) + r"(?:(?:public|protected|private|abstract|final|static|sealed|non-sealed|strictfp)\s+)*" + r"\b(class|interface|enum|record)\b" + ) + m = NAME_KIND_RE.search(s) + if not m: + return None + brace_pos = s.find("{", m.end()) + return s[m.start():].strip() if brace_pos == -1 else s[m.start():brace_pos].strip() + + def before_brace(s: str) -> str: + i = s.find("{") + return s if i == -1 else s[:i] + + def drop_leading_annotations(s: str) -> str: + i = 0 + n = len(s) + while True: + while i < n and s[i].isspace(): + i += 1 + if i < n and s[i] == JAVA_ANNOTATION_SYMBOL: + i += 1 + while i < n and (s[i].isalnum() or s[i] in ('_', '$', '.')): + i += 1 + if i < n and s[i] == '(': + paren_depth = 1 + i += 1 + while i < n and paren_depth > 0: + ch = s[i] + if ch == '(': + paren_depth += 1 + elif ch == ')': + paren_depth -= 1 + i += 1 + continue + break + return s[i:] + + def base_token(t: str) -> str: + """Strip generics and [] to get a core type token.""" + t = (t or "").strip() + out, generics_depth = [], 0 + for ch in t: + if ch == '<': + generics_depth += 1 + elif ch == '>': + generics_depth = max(0, generics_depth - 1) + elif generics_depth == 0: + out.append(ch) + return ''.join(out).replace("[]", "").strip() + + def simple_name_from_fqcn(fqcn: str) -> str: + return fqcn.split('.')[-1] if fqcn else fqcn + + def clean_type_token(tok: str) -> str: + """ + Defensive cleanup: drop tokens that look like generic bounds fragments, + e.g., 'T extends Foo' or 'B extends SdkBuilder'. + """ + t = tok.strip() + if not t: + return "" + # If a bare 'extends' remains in a token here, it can only be a generic bound. + if re.search(r"\bextends\b", t): + return "" + return t + + def tup_dedup_keep_order(pairs: List[Tuple[str, str]]) -> List[Tuple[str, str]]: + """ + Remove duplicate (name, source) pairs while preserving the order + of their first appearance. + """ + seen: set[Tuple[str, str]] = set() + out: List[Tuple[str, str]] = [] + for p in pairs: + if p and p not in seen: + seen.add(p) + out.append(p) + return out + + def meta_source(doc) -> str: + try: + meta = getattr(doc, "metadata", None) or {} + return meta.get("source") or "" + except Exception: + return "" + + def simple_name_from_filename(doc) -> str | None: + src = meta_source(doc) + m = re.search(r"([A-Za-z_$][\w$]*)\.java$", src) + return m.group(1) if m else None + + def split_type_list_strict(segment: str) -> List[str]: + """Split by commas outside <...> and (...).""" + if not segment: + return [] + items, cur = [], [] + generics_depth = 0 + paren_depth = 0 + for ch in segment: + if ch == '<': + generics_depth += 1 + elif ch == '>': + generics_depth = max(0, generics_depth - 1) + elif ch == '(': + paren_depth += 1 + elif ch == ')': + paren_depth = max(0, paren_depth - 1) + if ch == ',' and generics_depth == 0 and paren_depth == 0: + tok = ''.join(cur).strip() + if tok: + items.append(tok) + cur = [] + else: + cur.append(ch) + last = ''.join(cur).strip() + if last: + items.append(last) + return items + + # Clause extractor that starts scanning AFTER the 'kind name' match. + name_kind_re = re.compile(r"""\b(class|interface|enum|record)\b\s+([A-Za-z_$][\w$]*)""", re.X) + + def extract_clauses_after_kindname(header: str) -> Tuple[str | None, str | None]: + """ + Structured scan for 'extends' and 'implements' clauses while ignoring + anything inside generic parameter lists after the type name. + """ + m = name_kind_re.search(header) + if not m: + return None, None + scan_start = m.end() # start right after 'class|interface|enum|record ' + + def is_word_at(pos: int, word: str) -> bool: + if pos < 0 or pos + len(word) > len(header): + return False + if header[pos:pos+len(word)] != word: + return False + left_ok = pos == 0 or not (header[pos-1].isalnum() or header[pos-1] == '_') + right_ok = pos+len(word) == len(header) or not (header[pos+len(word)].isalnum() or header[pos+len(word)] == '_') + return left_ok and right_ok + + generics_depth = 0 + paren_depth = 0 + ext_start = impl_start = None + i = scan_start + while i < len(header): + ch = header[i] + if ch == '<': + generics_depth += 1 + elif ch == '>': + generics_depth = max(0, generics_depth - 1) + elif ch == '(': + paren_depth += 1 + elif ch == ')': + paren_depth = max(0, paren_depth - 1) + elif generics_depth == 0 and paren_depth == 0: + if ext_start is None and is_word_at(i, "extends"): + j = i + len("extends") + while j < len(header) and header[j].isspace(): + j += 1 + ext_start = j + i = j + continue + if impl_start is None and is_word_at(i, "implements"): + j = i + len("implements") + while j < len(header) and header[j].isspace(): + j += 1 + impl_start = j + i = j + continue + i += 1 + + def segment_until_next(start: int) -> str | None: + if start is None: + return None + seg_generics_depth = 0 + seg_paren_depth = 0 + k = start + while k < len(header): + ch = header[k] + if ch == '<': + seg_generics_depth += 1 + elif ch == '>': + seg_generics_depth = max(0, seg_generics_depth - 1) + elif ch == '(': + seg_paren_depth += 1 + elif ch == ')': + seg_paren_depth = max(0, seg_paren_depth - 1) + elif seg_generics_depth == 0 and seg_paren_depth == 0: + if is_word_at(k, "extends") or is_word_at(k, "implements"): + break + k += 1 + seg = header[start:k].strip().rstrip(",").strip() + return seg or None + + ext_seg = segment_until_next(ext_start) + impl_seg = segment_until_next(impl_start) + + return ext_seg, impl_seg + + # Inline-friendly (work on full source) + pkg_re = re.compile(r"\bpackage\s+([A-Za-z_][\w.]*)\s*;") + import_re = re.compile(r"\bimport\s+(?!static\b)\s*([A-Za-z_][\w.]*?(?:\.\*|))\s*;") + + # ---- jar id extraction ---- + # Captures like: 'dependencies-sources/-sources' + jar_dir_re = re.compile(r"(?:^|/)(dependencies-sources/[^/]+)(?:/|$)") + + def source_jar_id(path: str) -> str: + """Extract a stable jar id directory from a source path.""" + if not path: + return "" + m = jar_dir_re.search(path) + if m: + return m.group(1) + parts = [p for p in path.split('/') if p] + return parts[0] if parts else "" + + # ---------------- first pass: collect per-FQCN variants ---------------- + variants_by_fq: Dict[str, List[Dict]] = {} + known_fqcns: set[str] = set() + + for doc in tqdm(full_source_documents, total=len(full_source_documents), desc="Building types inheritance map..."): + src_all = doc.page_content or "" + + # Parse package/imports once per compilation unit + pkg = (pkg_re.search(src_all) or [None, ""])[1] + imps = import_re.findall(src_all) + src_path = meta_source(doc) + file_simple = simple_name_from_filename(doc) # e.g., "FormEncodedDataDefinition" + s_nc = strip_comments_preserving_newlines(src_all) + + # Discover ALL type declarations (top-level + nested) + decls = _find_all_type_decls(src_all) + + # Fallback: no decls found → keep legacy behavior (top-level by filename) + if not decls: + if not file_simple: + continue + kind = "class" + extends_names: List[str] = [] + implements_names: List[str] = [] + declared_simple = file_simple + fqcn = f"{pkg}.{declared_simple}" if pkg else declared_simple + + rec = { + "fqcn": fqcn, + "simple": declared_simple, + "pkg": pkg, + "imports": imps, + "kind": kind, + "extends_raw": extends_names, + "implements_raw": implements_names, + "source": src_path, + "jar": source_jar_id(src_path), + } + variants_by_fq.setdefault(fqcn, []).append(rec) + known_fqcns.add(fqcn) + continue + + # If we're processing a *slice* that only contains an inner type, we may not see the outer. + # Detect presence of the outer by name in this source (to avoid mislabeling top-level siblings). + has_outer_decl = bool(file_simple and re.search( + rf"\b(class|interface|enum|record)\b\s+{re.escape(file_simple)}\b", s_nc)) + + # Build records for every discovered declaration + for d in decls: + kind = d["kind"] + declared_simple = d["name"] + header_text = d["header"] + + # Extract extends/implements from the specific header + ext_seg, impl_seg = extract_clauses_after_kindname(header_text) + # LAST RESORT (classes only): scan for "class ... extends ..." in de-commented source slice + if kind == "class" and (not ext_seg) and ("extends" in s_nc): + m_ext2 = re.search( + rf"\bclass\s+{re.escape(declared_simple)}\b[^{{;]*?\bextends\b\s+([^{{;]+)", + s_nc + ) + if m_ext2: + ext_seg = m_ext2.group(1).strip() + + extends_names = split_type_list_strict(ext_seg) if ext_seg else [] + implements_names = split_type_list_strict(impl_seg) if impl_seg else [] + + # Defensive cleanup + extends_names = [clean_type_token(x) for x in extends_names if clean_type_token(x)] + implements_names = [clean_type_token(x) for x in implements_names if clean_type_token(x)] + + # Compute FQCN with correct nesting + if d["enclosing"]: + # Full chain from discovered parents: pkg.Outer.Inner.Current + prefix = ".".join(d["enclosing"]) + fqcn = f"{pkg}.{prefix}.{declared_simple}" if pkg else f"{prefix}.{declared_simple}" + else: + # No enclosing type discovered. + # If this looks like an inner slice (no outer decl visible in this source) + # and filename suggests the real outer, prefix with file_simple. + if file_simple and declared_simple != file_simple and not has_outer_decl: + outer_fq = f"{pkg}.{file_simple}" if pkg else file_simple + fqcn = f"{outer_fq}.{declared_simple}" + else: + fqcn = f"{pkg}.{declared_simple}" if pkg else declared_simple + + rec = { + "fqcn": fqcn, + "simple": declared_simple, + "pkg": pkg, + "imports": imps, + "kind": kind, + "extends_raw": extends_names, + "implements_raw": implements_names, + "source": src_path, + "jar": source_jar_id(src_path), + } + variants_by_fq.setdefault(fqcn, []).append(rec) + known_fqcns.add(fqcn) + + # simple -> known FQCNs + by_simple: Dict[str, List[str]] = {} + for fqcn in known_fqcns: + by_simple.setdefault(simple_name_from_fqcn(fqcn), []).append(fqcn) + + def make_resolver(pkg: str, imports: List[str]): + """ + Build a simple type resolver for a compilation unit. + Resolution precedence: + 1) explicit imports + 2) same package + 3) star-import packages (unique) + 4) globally unique simple name + Also handles partially-qualified nested types like + 'FormParserFactory.ParserDefinition' → 'io.undertow.server.handlers.form.FormParserFactory.ParserDefinition'. + """ + explicit: Dict[str, str] = {} + star_import_packages: List[str] = [] + for imp in imports: + if imp.endswith(".*"): + star_import_packages.append(imp[:-2]) # package of a star import + else: + explicit[simple_name_from_fqcn(imp)] = imp # simple -> explicit FQCN + + def _resolve_head(head_simple: str) -> str | None: + """Resolve a top-level simple name to FQCN using the precedence above.""" + if head_simple in explicit: + return explicit[head_simple] + if pkg: + candidate = f"{pkg}.{head_simple}" + if candidate in known_fqcns: + return candidate + # Star imports – accept only a unique hit + candidates: List[str] = [] + for fq_top in by_simple.get(head_simple, []): + if any(fq_top.startswith(base + ".") for base in star_import_packages): + candidates.append(fq_top) + if len(candidates) == 1: + return candidates[0] + # Globally unique simple name across all known types + all_candidates = by_simple.get(head_simple, []) + if len(all_candidates) == 1: + return all_candidates[0] + return None + + def resolve(token: str) -> str: + tok = base_token(token).strip() + if not tok: + return "" # empty token + + # Handle partially-qualified nested types, e.g., "FormParserFactory.ParserDefinition" + if "." in tok: + head, rest = tok.split(".", 1) + # If already looks fully-qualified for a known top-level type in this package, + # prefer composing with the package (avoids leaving it partially-qualified). + fq_head = _resolve_head(head) + if fq_head: + return fq_head + "." + rest + # If we can't resolve the head, leave as-is (may already be a fully-qualified package path) + return tok + + # Simple (non-dotted) type name + fq_simple = _resolve_head(tok) + return fq_simple if fq_simple else tok + + return resolve + + # resolve parents/impls per variant + for fqcn, var_list in variants_by_fq.items(): + for rec in var_list: + resolve = make_resolver(rec["pkg"], rec["imports"]) + if rec["kind"] == "record": + parents = ["java.lang.Record"] + elif rec["kind"] == "class": + parents = [resolve(x) for x in (rec.get("extends_raw") or [])][:1] + elif rec["kind"] == "enum": + parents = [] + else: # interface + parents = [resolve(x) for x in (rec.get("extends_raw") or [])] + impls = [resolve(x) for x in (rec.get("implements_raw") or [])] + rec["parents"] = parents + rec["impls"] = impls + + # index variants + def vid_of(rec: Dict) -> Tuple[str, str]: + return rec["fqcn"], rec["source"] + + variants: List[Dict] = [rec for lst in variants_by_fq.values() for rec in lst] + rec_by_vid: Dict[Tuple[str, str], Dict] = {vid_of(rec): rec for rec in variants} + rec_by_fq_and_jar: Dict[Tuple[str, str], Dict] = {} + for rec in variants: + rec_by_fq_and_jar[(rec["fqcn"], rec["jar"])] = rec + + def pick_variant_for_jar(type_fqcn: str, jar_id: str) -> Dict | None: + """Pick the variant in the same jar if present; else unique globally; else None.""" + if not type_fqcn: + return None + same = rec_by_fq_and_jar.get((type_fqcn, jar_id)) + if same: + return same + lst = variants_by_fq.get(type_fqcn) or [] + if len(lst) == 1: + return lst[0] + return None # ambiguous across jars, and not in this jar + + # reverse extends (jar-aware) + rev_extends: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + rev_iface_extends: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + + def is_interface_record(type_rec: Dict) -> bool: + return type_rec["kind"] == "interface" + + for rec in variants: + jar_id = rec["jar"] + this_vid = vid_of(rec) + for parent_fqcn in rec.get("parents", []): + if not parent_fqcn: + continue + parent_rec = pick_variant_for_jar(parent_fqcn, jar_id) + if not parent_rec: + continue + parent_vid = vid_of(parent_rec) + if is_interface_record(rec) and is_interface_record(parent_rec): + rev_iface_extends.setdefault(parent_vid, []).append(this_vid) + else: + rev_extends.setdefault(parent_vid, []).append(this_vid) + + # --------- upward chain & iface collectors with cycle guards --------- + extends_chain_pairs_memo: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + + def extends_chain_pairs(start_vid: Tuple[str, str]) -> List[Tuple[str, str]]: + """ + Upward chain for classes (linear) and interface parents (DFS), + jar-aware; placeholders where parents are unseen. + Returns list of (fqcn, source). + """ + if start_vid in extends_chain_pairs_memo: + return extends_chain_pairs_memo[start_vid] + + cur = rec_by_vid.get(start_vid) + if not cur: + return [] + jar_id = cur["jar"] + chain: List[Tuple[str, str]] = [] + + if is_interface_record(cur): + seen_vids = set() + + def dfs_iface(v: Tuple[str, str]): + r = rec_by_vid.get(v) + if not r: + return + for parent_fqcn in r.get("parents", []): + if not parent_fqcn: + continue + parent_rec = pick_variant_for_jar(parent_fqcn, jar_id) + if parent_rec: + parent_vid = vid_of(parent_rec) + if parent_vid in seen_vids: + continue + seen_vids.add(parent_vid) + if is_interface_record(parent_rec): + dfs_iface(parent_vid) + chain.append((parent_rec["fqcn"], parent_rec["source"])) + else: + # Keep the best-available token (may be FQCN or simple if unresolved) + chain.append((parent_fqcn, cur["source"])) + + dfs_iface(start_vid) + out = tup_dedup_keep_order(chain) + extends_chain_pairs_memo[start_vid] = out + return out + + # class/record/enum: linear with visited guard + cur_vid = start_vid + visited_vids = set() + while True: + r = rec_by_vid.get(cur_vid) + if not r: + break + ps = r.get("parents", []) + if not ps: + break + parent_fqcn = ps[0] + parent_rec = pick_variant_for_jar(parent_fqcn, jar_id) if parent_fqcn else None + if parent_rec: + parent_vid = vid_of(parent_rec) + if parent_vid in visited_vids or parent_vid == cur_vid: + break + visited_vids.add(parent_vid) + chain.append((parent_rec["fqcn"], parent_rec["source"])) + if not is_interface_record(parent_rec): + cur_vid = parent_vid + else: + break + else: + chain.append((parent_fqcn, r["source"])) + break + + out = tup_dedup_keep_order(chain) + extends_chain_pairs_memo[start_vid] = out + return out + + # vids only (memoized) for reverse implementers + iface_closure_vids_memo: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + + def all_interfaces_vids(start_vid: Tuple[str, str]) -> List[Tuple[str, str]]: + """ + For wiring reverse implementers: return interface vids reachable from a class + (direct implements + via ancestors + super-interfaces). Jar-aware. + """ + if start_vid in iface_closure_vids_memo: + return iface_closure_vids_memo[start_vid] + + rec = rec_by_vid.get(start_vid) + if not rec: + iface_closure_vids_memo[start_vid] = [] + return [] + jar_id = rec["jar"] + ordered_vids: List[Tuple[str, str]] = [] + added, visiting = set(), set() + + def add_vid(v: Tuple[str, str]): + if v and v not in added: + added.add(v) + ordered_vids.append(v) + + def dfs_iface_vid(v: Tuple[str, str]): + if not v or v in visiting or v in added: + return + r = rec_by_vid.get(v) + if not r: + return + if not is_interface_record(r): + add_vid(v) + return + visiting.add(v) + for parent_fqcn in r.get("parents", []): + parent_rec = pick_variant_for_jar(parent_fqcn, jar_id) + if parent_rec: + dfs_iface_vid(vid_of(parent_rec)) + visiting.remove(v) + add_vid(v) + + # direct implements + for iface_fqcn in rec.get("impls", []): + iface_rec = pick_variant_for_jar(iface_fqcn, jar_id) + if iface_rec: + dfs_iface_vid(vid_of(iface_rec)) + + # via class ancestors (with visited guard on upward chain) + cur_r = rec + seen_up = set() + while True: + ps = cur_r.get("parents", []) + if not ps: + break + parent_fqcn = ps[0] + parent_rec = pick_variant_for_jar(parent_fqcn, jar_id) + if not parent_rec or is_interface_record(parent_rec): + break + parent_vid = vid_of(parent_rec) + if parent_vid in seen_up: + break + seen_up.add(parent_vid) + for iface_fqcn in parent_rec.get("impls", []): + iface2_rec = pick_variant_for_jar(iface_fqcn, jar_id) + if iface2_rec: + dfs_iface_vid(vid_of(iface2_rec)) + cur_r = parent_rec + + # if start is an interface, include its super-interfaces + if is_interface_record(rec): + for parent_fqcn in rec.get("parents", []): + parent_rec = pick_variant_for_jar(parent_fqcn, jar_id) + if parent_rec: + dfs_iface_vid(vid_of(parent_rec)) + + iface_closure_vids_memo[start_vid] = ordered_vids + return ordered_vids + + # --- helper: transitive super-interfaces as PAIRS, jar-aware, memoized --- + iface_super_pairs_memo: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + + def iface_super_pairs(iface_vid: Tuple[str, str]) -> List[Tuple[str, str]]: + """ + For an interface variant vid, return all transitive super-interfaces + as (fqcn, source) pairs, jar-aware. Memoized. + """ + if iface_vid in iface_super_pairs_memo: + return iface_super_pairs_memo[iface_vid] + + rec = rec_by_vid.get(iface_vid) + if not rec or rec["kind"] != "interface": + iface_super_pairs_memo[iface_vid] = [] + return [] + + jar_id = rec["jar"] + cur_src = rec["source"] + + ordered: List[Tuple[str, str]] = [] + added: Set[Tuple[str, str]] = set() + visiting: Set[Tuple[str, str]] = set() + + def add_pair(p: Tuple[str, str]): + if p and p not in added: + added.add(p) + ordered.append(p) + + def dfs_iface_vid(v: Tuple[str, str]): + if v in visiting: + return + r = rec_by_vid.get(v) + if not r or r["kind"] != "interface": + return + visiting.add(v) + for parent_fqcn in r.get("parents", []): + parent_rec = pick_variant_for_jar(parent_fqcn, jar_id) + if parent_rec: + pk = (parent_rec["fqcn"], parent_rec["source"]) + dfs_iface_vid(pk) + add_pair((parent_rec["fqcn"], parent_rec["source"])) + else: + # unknown external interface -> placeholder bound to current interface's source + add_pair((parent_fqcn, cur_src)) + visiting.remove(v) + + dfs_iface_vid(iface_vid) + out = tup_dedup_keep_order(ordered) + iface_super_pairs_memo[iface_vid] = out + return out + + # pairs version (placeholders for externals) with visited guard + all_ifaces_pairs_memo: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + + def all_interfaces_pairs(start_vid: Tuple[str, str]) -> List[Tuple[str, str]]: + """ + For a class/record/enum: direct interfaces + super-interfaces of those interfaces + + interfaces from ancestor classes (and their super-interfaces). + For an interface: its super-interfaces. + Jar-aware; placeholders only for interfaces not present in provided set. + Returns (fqcn, source) pairs. + """ + if start_vid in all_ifaces_pairs_memo: + return all_ifaces_pairs_memo[start_vid] + + rec = rec_by_vid.get(start_vid) + if not rec: + all_ifaces_pairs_memo[start_vid] = [] + return [] + + jar_id = rec["jar"] + cur_src = rec["source"] + + ordered: List[Tuple[str, str]] = [] + added: Set[Tuple[str, str]] = set() + + def add_pair(p: Tuple[str, str]): + if p and p not in added: + added.add(p) + ordered.append(p) + + # Helper: add one interface (jar-aware) and all its super-interfaces (pairs) + def add_iface_and_supers(iface_fqcn: str): + iface_rec = pick_variant_for_jar(iface_fqcn, jar_id) + if iface_rec: + ik = (iface_rec["fqcn"], iface_rec["source"]) + add_pair((iface_rec["fqcn"], iface_rec["source"])) + for sup in iface_super_pairs(ik): + add_pair(sup) + else: + # external/unseen -> placeholder (cannot know its supers) + add_pair((iface_fqcn, cur_src)) + + # Direct implements on this type + for iface_fqcn in rec.get("impls", []): + add_iface_and_supers(iface_fqcn) + + # Interfaces via ancestor classes (jar-aware). Guard against cycles. + cur_r = rec + seen_up: Set[Tuple[str, str]] = set() + while True: + ps = cur_r.get("parents", []) + if not ps: + break + parent_fqcn = ps[0] + parent_rec = pick_variant_for_jar(parent_fqcn, jar_id) + if not parent_rec or is_interface_record(parent_rec): + break + parent_vid = (parent_rec["fqcn"], parent_rec["source"]) + if parent_vid in seen_up: + break + seen_up.add(parent_vid) + for iface_fqcn in parent_rec.get("impls", []): + add_iface_and_supers(iface_fqcn) + cur_r = parent_rec + + # If starting node itself is an interface, include its super-interfaces. + if is_interface_record(rec): + for sup in iface_super_pairs(start_vid): + add_pair(sup) + + out = ordered + all_ifaces_pairs_memo[start_vid] = out + return out + + # reverse: interface implementers (vids only, memoized closures) + rev_impl_all: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + for vid, r in rec_by_vid.items(): + if is_interface_record(r): + continue + for i_vid in all_interfaces_vids(vid): + i_rec = rec_by_vid.get(i_vid) + if i_rec and is_interface_record(i_rec): + rev_impl_all.setdefault(i_vid, []).append(vid) + + def all_subclasses_variant(parent_vid: Tuple[str, str]) -> List[Tuple[str, str]]: + out: List[Tuple[str, str]] = [] + seen = set() + q = list(rev_extends.get(parent_vid, [])) + while q: + x = q.pop(0) + if x in seen: + continue + seen.add(x) + out.append(x) + q.extend(rev_extends.get(x, []) or []) + return out + + def all_subinterfaces_variant(parent_vid: Tuple[str, str]) -> List[Tuple[str, str]]: + out: List[Tuple[str, str]] = [] + seen = set() + q = list(rev_iface_extends.get(parent_vid, [])) + while q: + x = q.pop(0) + if x in seen: + continue + seen.add(x) + out.append(x) + q.extend(rev_iface_extends.get(x, []) or []) + return out + + def all_implementers_of_interface_variant(iface_vid: Tuple[str, str]) -> List[Tuple[str, str]]: + impls: List[Tuple[str, str]] = [] + impls.extend(rev_impl_all.get(iface_vid, [])) + for sub_i in all_subinterfaces_variant(iface_vid): + impls.extend(rev_impl_all.get(sub_i, [])) + # stable dedup + seen = set() + out: List[Tuple[str, str]] = [] + for v in impls: + if v and v not in seen: + seen.add(v) + out.append(v) + return out + + # ---------------- final map ---------------- + out: Dict[Tuple[str, str], List[Tuple[str, str]]] = {} + + for vid, rec in rec_by_vid.items(): + self_fqcn = rec["fqcn"] + source = rec.get("source", "") + + up_chain_pairs = extends_chain_pairs(vid) # may include placeholders + iface_pairs = all_interfaces_pairs(vid) # may include placeholders + + down_pairs: List[Tuple[str, str]] = [] + if is_interface_record(rec): + impl_vids = all_implementers_of_interface_variant(vid) # vids only + down_pairs.extend( + (rec_by_vid[v]["fqcn"], rec_by_vid[v]["source"]) for v in impl_vids + ) + else: + subs_vids = all_subclasses_variant(vid) + down_pairs.extend( + (rec_by_vid[v]["fqcn"], rec_by_vid[v]["source"]) for v in subs_vids + ) + + vals = tup_dedup_keep_order([(self_fqcn, source)] + up_chain_pairs + iface_pairs + down_pairs) + out[(self_fqcn, source)] = vals + + return out + +@lru_cache(maxsize=100000) +def extract_method_name_with_params(java_src: str) -> str | None: + """ + Return 'methodName()' from a Java method header string. + Return 'lambda' ONLY if the snippet itself starts with a lambda expression like: + - (a, b) -> ... + - x -> ... + Assumptions: + - The string begins at the actual method header OR a top-level lambda. + - No comments/javadocs/annotations are present at the very beginning (before a lambda or header). + """ + s = java_src.lstrip() + n = len(s) + if n == 0: + return None + + # ---------- helpers ---------- + def is_id_start(ch: str) -> bool: + return ch.isalpha() or ch in "_$" + + def is_id_part(ch: str) -> bool: + return ch.isalnum() or ch in "_$" + + def skip_ws(i: int) -> int: + _s = s + _n = n + while i < _n and _s[i].isspace(): + i += 1 + return i + + def match_paren_close(start: int) -> int: + """Given s[start] == '(', return index of its matching ')', else -1.""" + depth = 0 + i2 = start + _s = s + _n = n + while i2 < _n: + ch = _s[i2] + if ch == '(': + depth += 1 + elif ch == ')': + depth -= 1 + if depth == 0: + return i2 + i2 += 1 + return -1 + + # ---------- detect lambda ONLY if it is at the beginning ---------- + i0 = skip_ws(0) + if i0 < n: + ch0 = s[i0] + if ch0 == '(': # (params) -> ... + end = match_paren_close(i0) + if end != -1: + j = skip_ws(end + 1) + if j + 1 < n and s[j] == '-' and s[j + 1] == '>': + return "lambda" + elif is_id_start(ch0): # x -> ... + j = i0 + 1 + while j < n and is_id_part(s[j]): + j += 1 + k = skip_ws(j) + if k + 1 < n and s[k] == '-' and s[k + 1] == '>': + return "lambda" + + # ---------- regular method header parsing ---------- + open_idx = s.find('(', i0) + if open_idx == -1: + return None + + # Backtrack to method name just before '(' + i = open_idx - 1 + _s = s # local alias + while i >= 0 and _s[i].isspace(): + i -= 1 + if i < 0 or not is_id_part(_s[i]): + return None + start = i + while start - 1 >= 0 and is_id_part(_s[start - 1]): + start -= 1 + name = _s[start:i + 1] + + # Find matching ')' + close_idx = match_paren_close(open_idx) + if close_idx == -1: + return None + + raw_params = _s[open_idx + 1:close_idx] + + # ---- split ONLY on top-level commas between parameters ---- + if not raw_params: + return f"{name}()" + + # Fast path: no commas at all -> single parameter, so no need for deep scan + if ',' not in raw_params: + parts = [raw_params] + else: + parts: list[str] = [] + depth_p = depth_b = depth_a = 0 # () [] <> + in_str = in_chr = False + token_start = 0 + append_part = parts.append + + for idx, ch in enumerate(raw_params): + if in_str: + if ch == '\\': + continue + elif ch == '"': + in_str = False + continue + if in_chr: + if ch == '\\': + continue + elif ch == "'": + in_chr = False + continue + + if ch == '"': + in_str = True + continue + if ch == "'": + in_chr = True + continue + + if ch == '(': + depth_p += 1 + elif ch == ')': + depth_p -= 1 + elif ch == '[': + depth_b += 1 + elif ch == ']': + depth_b -= 1 + elif ch == '<': + depth_a += 1 + elif ch == '>' and depth_a > 0: + depth_a -= 1 + elif ch == ',' and depth_p == 0 and depth_b == 0 and depth_a == 0: + append_part(raw_params[token_start:idx]) + token_start = idx + 1 + parts.append(raw_params[token_start:]) + + # ---- per-parameter cleanup (preserve intra-type spacing; normalize name-side arrays) ---- + cleaned: list[str] = [] + cleaned_append = cleaned.append + ann_sub = _ANNOTATION_RE.sub + final_sub = _FINAL_RE.sub + param_main_match = _PARAM_MAIN_RE.match + trailing_name_search = _TRAILING_NAME_RE.search + array_brackets_findall = _ARRAY_BRACKETS_RE.findall + + for p in parts: + # Trim edges; preserve interior spacing + p = p.strip() + if not p: + continue + + # Remove annotations like @Foo or @Foo(bar) anywhere in the parameter + # (consume following whitespace so we don't leave double spaces). + p = ann_sub('', p) + + # Remove standalone 'final' tokens (do not collapse other spacing) + p = final_sub('', p).strip() + if not p: + continue + + # Split into type-part, name, and any array brackets stuck on the *name* side. + # tpart can contain generics and [] with arbitrary spacing; keep as-is. + m = param_main_match(p) + + if not m: + # Fallback: ensure there's at least one space before the trailing identifier + mname = trailing_name_search(p) + if mname: + pname = mname.group(1) + tpart = p[:mname.start()] + if len(tpart) == 0 or not tpart[-1].isspace(): + p = f"{tpart} {pname}" + else: + p = f"{tpart}{pname}" + cleaned_append(p) + continue + + tpart = m.group('t') + pname = m.group('name') + name_arr = m.group('name_arr') or '' + + if name_arr.strip(): + # Move name-side brackets to the type side, canonicalized as '[]' per dim. + dims = len(array_brackets_findall(name_arr)) + t_clean = tpart.rstrip() # drop trailing space before appending + t_with_arr = t_clean + '[]' * dims + p = f"{t_with_arr} {pname}" + else: + # Keep original spacing around tpart; ensure at least one space before name + if len(tpart) == 0 or not tpart[-1].isspace(): + p = f"{tpart} {pname}" + else: + p = f"{tpart}{pname}" + + cleaned_append(p) + + params_norm = ', '.join(filter(None, cleaned)) + return f"{name}({params_norm})" + +def find_function(caller_src:str, method_name:str, documents_of_functions) -> Document | None: + """Find a function document by source path and method name. + + documents_of_functions can be either: + - list[Document]: iterates all documents (original behavior) + - dict[str, list[Document]]: O(1) lookup by source path (from _source_to_fn_docs) + """ + # TODO make it smarter, also use the method parameters to check if this is the actual method + if isinstance(documents_of_functions, dict): + candidates = documents_of_functions.get(caller_src, []) + else: + candidates = (doc for doc in documents_of_functions if caller_src == doc.metadata["source"]) + + for doc in candidates: + extracted_method_name = extract_method_name_with_params(doc.page_content) + if extracted_method_name and method_name in extracted_method_name: + return doc + +def get_target_class_names(inheritance_list: List[Tuple[str, str]]) -> frozenset[str]: + return frozenset([class_name for (class_name, source) in inheritance_list]) + +@lru_cache(maxsize=50000) +def strip_java_generics(type_str: str) -> str: + """ + Remove all balanced <...> generic sections from a Java type string. + + Examples: + "List" -> "List" + "Map>" -> "Map" + "Map>[]" -> "Map[]" + "Map.Entry" -> "Map.Entry" + "List..." -> "List..." + "java.util.Map " -> "java.util.Map" + "@Nonnull List<@A ? super T>" -> "@Nonnull List" + """ + s = type_str.strip() + + # Fast path: no generic markers at all, just normalize whitespace/punctuation + if '<' not in s and '>' not in s: + res = _GENERIC_SPACE_RE.sub(' ', s).strip() + res = _PUNCT_SPACE_RE.sub(r'\1', res) + res = _VARARGS_SPACE_RE.sub(r'\1', res) + return res + + # Slow path: remove balanced <...> sections + out = [] + depth = 0 + for ch in s: + if ch == '<': + depth += 1 + continue + if ch == '>': + if depth > 0: + depth -= 1 + # ignore stray '>' if any + continue + if depth == 0: + out.append(ch) + + # Join and normalize spaces around punctuation commonly found in types + res = ''.join(out) + + # Collapse multiple spaces to one + res = _GENERIC_SPACE_RE.sub(' ', res).strip() + + # Remove spaces around punctuation like dots, brackets, commas, and varargs + res = _PUNCT_SPACE_RE.sub(r'\1', res) # "Map []" -> "Map[]", "Map . Entry" -> "Map.Entry" + res = _VARARGS_SPACE_RE.sub(r'\1', res) # "List ..." -> "List..." + + return res + +def extract_fqcn(text: str) -> str: + """ + Convert a source file path to an FQCN. + Supported forms: + 1) dependencies-sources/--sources//.java + 2) .../src/main/java//.java + 3) /.java (no prefix) + + Examples: + 'dependencies-sources/hibernate-core-6.6.13.Final-sources/org/hibernate/type/descriptor/java/ArrayJavaType.java' + -> 'org.hibernate.type.descriptor.java.ArrayJavaType' + 'org/hibernate/type/descriptor/java/ArrayJavaType.java' + -> 'org.hibernate.type.descriptor.java.ArrayJavaType' + '.../src/main/java/org/keycloak/.../VerifyEmailActionTokenHandler.java' + -> 'org.keycloak....VerifyEmailActionTokenHandler' + """ + p = (text or "").replace("\\", "/").strip() + + # Prefer the standard Maven/Gradle source-root marker when present. + src_marker = "/src/main/java/" + if src_marker in p: + tail = p.split(src_marker, 1)[1] + else: + # Otherwise, if there's a '-sources/' prefix, drop everything up to and including + # the last occurrence (handles dependencies-sources/...-sources/...). + marker = "-sources/" + cut = p.rfind(marker) + tail = p[cut + len(marker):] if cut != -1 else p + + if tail.startswith("/"): + tail = tail[1:] + + # Drop the .java suffix (case-sensitive per your examples) + if tail.endswith(".java"): + tail = tail[:-5] + + return tail.replace("/", ".") \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/javascript_extended_segmenter.py b/src/exploit_iq_commons/utils/javascript_extended_segmenter.py new file mode 100644 index 000000000..28bb9211f --- /dev/null +++ b/src/exploit_iq_commons/utils/javascript_extended_segmenter.py @@ -0,0 +1,276 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import threading +from typing import List + +from tree_sitter_languages import get_language +from langchain_community.document_loaders.parsers.language.tree_sitter_segmenter import TreeSitterSegmenter + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) + +_JS_EXCLUDE_DIRS: frozenset[str] = frozenset({ + "dist", "build/static", "coverage", ".nyc_output", +}) + +_JS_EXCLUDE_SUFFIXES: tuple[str, ...] = ( + ".min.js", ".min.css", ".bundle.js", +) + +# Tree-sitter query matching JavaScript constructs that the agent tools +# need for function extraction and call-chain analysis. +# +# Named exports (export function foo, export class Bar, export const x = () => {}) +# are captured by the inner declaration patterns — the export keyword is NOT +# included in those chunks. Only anonymous default exports need explicit +# export_statement patterns since they have no matching inner declaration. +JS_CHUNK_QUERY = """ +[ + (function_declaration) @function + (generator_function_declaration) @generator + + (class_declaration) @class + + (lexical_declaration + (variable_declarator + value: [(arrow_function) (function)])) @func_expr + + (variable_declaration + (variable_declarator + value: [(arrow_function) (function)])) @var_func_expr + + (variable_declaration + (variable_declarator + value: (call_expression + arguments: (arguments [(function) (arrow_function)])))) @wrapped_func + + (lexical_declaration + (variable_declarator + value: (call_expression + arguments: (arguments [(function) (arrow_function)])))) @wrapped_func_const + + (lexical_declaration + (variable_declarator + value: (object + [(method_definition) (pair value: [(function) (arrow_function)])]))) @obj_with_methods + + (variable_declaration + (variable_declarator + value: (object + [(method_definition) (pair value: [(function) (arrow_function)])]))) @var_obj_with_methods + + (export_statement (function)) @export_default_func + (export_statement (arrow_function)) @export_default_arrow + (export_statement (class)) @export_default_class + + (expression_statement + (assignment_expression + right: [(function) (arrow_function)])) @assign_func +] +""" + + +class ExtendedJavaScriptSegmenter(TreeSitterSegmenter): + """JavaScript segmenter using tree-sitter for fast, modern-syntax-aware parsing. + + Handles optional chaining, nullish coalescing, top-level await, and all + other modern JS syntax that esprima-python cannot parse. Parses each + file once and caches the tree for reuse across extract/simplify calls. + """ + + # Class-level caches — initialized on first instantiation, shared across + # all instances. Double-checked locking for thread safety. + _js_language = None + _js_chunk_query = None + _js_error_query = None + _init_lock = threading.Lock() + + @staticmethod + def should_skip(path: str) -> bool: + """Return True if the file at *path* is a JS build artifact that should not be indexed.""" + parts = path.replace("\\", "/").split("/") + if "node_modules" in parts[:-1]: + return False + if path.endswith(_JS_EXCLUDE_SUFFIXES): + return True + # Only check root-level directories — deeper dist/build are legitimate (e.g. node_modules/pkg/dist/) + if len(parts) > 1: + if len(parts) > 2 and f"{parts[0]}/{parts[1]}" in _JS_EXCLUDE_DIRS: + return True + if parts[0] in _JS_EXCLUDE_DIRS: + return True + return False + + def __init__(self, code: str): + super().__init__(code) + cls = type(self) + if cls._js_language is None: + with cls._init_lock: + if cls._js_language is None: + lang = get_language("javascript") + cls._js_chunk_query = lang.query(JS_CHUNK_QUERY) + cls._js_error_query = lang.query("(ERROR) @error") + cls._js_language = lang + self._cached_tree = None + + if code.startswith("#!"): + logger.warning("File contains a shebang line. Skipping parsing.") + self.skip_file = True + else: + self.skip_file = False + + # -- TreeSitterSegmenter contract ---------------------------------------- + + def get_language(self): + return self._js_language + + def get_chunk_query(self) -> str: + return JS_CHUNK_QUERY + + def make_line_comment(self, text: str) -> str: + return f"// {text}" + + # -- Cached tree --------------------------------------------------------- + + def _get_tree(self): + if self._cached_tree is None: + from tree_sitter import Parser + parser = Parser() + parser.set_language(self._js_language) + self._cached_tree = parser.parse(bytes(self.code, "utf-8")) + return self._cached_tree + + # -- Public API ---------------------------------------------------------- + # Overrides of TreeSitterSegmenter methods to use cached tree/query and + # to extend extract_functions_classes with class/object method extraction. + + def is_valid(self) -> bool: + if self.skip_file: + return False + return len(self._js_error_query.captures(self._get_tree().root_node)) == 0 + + def extract_functions_classes(self) -> List[str]: + if self.skip_file: + return [] + + tree = self._get_tree() + captures = self._js_chunk_query.captures(tree.root_node) + + processed_lines = set() + chunks: List[str] = [] + + for node, _name in captures: + start_line = node.start_point[0] + end_line = node.end_point[0] + lines = range(start_line, end_line + 1) + + if any(line in processed_lines for line in lines): + continue + + processed_lines.update(lines) + chunks.append(node.text.decode("utf-8")) + + chunks.extend(self._extract_class_methods(tree)) + chunks.extend(self._extract_object_methods(tree)) + + return chunks + + def simplify_code(self) -> str: + if self.skip_file: + return self.code + + tree = self._get_tree() + processed_lines: set = set() + + simplified_lines = self.source_lines[:] + for node, _name in self._js_chunk_query.captures(tree.root_node): + start_line = node.start_point[0] + end_line = node.end_point[0] + + lines = range(start_line, end_line + 1) + if any(line in processed_lines for line in lines): + continue + + simplified_lines[start_line] = self.make_line_comment( + f"Code for: {self.source_lines[start_line]}" + ) + for line_num in range(start_line + 1, end_line + 1): + simplified_lines[line_num] = None # type: ignore[call-overload] + + processed_lines.update(lines) + + return "\n".join(line for line in simplified_lines if line is not None) + + # -- Private helpers ----------------------------------------------------- + + def _extract_class_methods(self, tree) -> List[str]: + """Extract individual methods from class declarations, annotated with class name.""" + methods: List[str] = [] + for node_type in ("class_declaration", "class"): + for node in self._walk_type(tree.root_node, node_type): + class_name = self._child_text(node, "identifier") or "AnonymousClass" + body = self._child_by_type(node, "class_body") + if body is None: + continue + for method_node in body.children: + if method_node.type == "method_definition": + method_code = method_node.text.decode("utf-8") + methods.append(f"{method_code}\n//(class: {class_name})") + return methods + + def _extract_object_methods(self, tree) -> List[str]: + """Extract individual methods from object literals in variable declarations.""" + methods: List[str] = [] + for decl_type in ("lexical_declaration", "variable_declaration"): + for decl_node in self._walk_type(tree.root_node, decl_type): + for declarator in decl_node.children: + if declarator.type != "variable_declarator": + continue + obj_name = self._child_text(declarator, "identifier") or "AnonymousObject" + obj_node = self._child_by_type(declarator, "object") + if obj_node is None: + continue + for prop in obj_node.children: + if prop.type == "method_definition": + methods.append(f"{prop.text.decode('utf-8')}\n//(class: {obj_name})") + elif prop.type == "pair": + value = self._child_by_type(prop, "function") or self._child_by_type(prop, "arrow_function") + if value is not None: + methods.append(f"{prop.text.decode('utf-8')}\n//(class: {obj_name})") + return methods + + @staticmethod + def _walk_type(root, node_type: str): + """Yield top-level children matching node_type, including inside export_statement.""" + for child in root.children: + if child.type == node_type: + yield child + elif child.type == "export_statement": + for grandchild in child.children: + if grandchild.type == node_type: + yield grandchild + + @staticmethod + def _child_by_type(node, child_type: str): + for child in node.children: + if child.type == child_type: + return child + return None + + @staticmethod + def _child_text(node, child_type: str) -> str | None: + child = ExtendedJavaScriptSegmenter._child_by_type(node, child_type) + return child.text.decode("utf-8") if child is not None else None \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/python_segmenters_with_classes_methods.py b/src/exploit_iq_commons/utils/python_segmenters_with_classes_methods.py new file mode 100644 index 000000000..f37a3cc6e --- /dev/null +++ b/src/exploit_iq_commons/utils/python_segmenters_with_classes_methods.py @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ast +from typing import List +import re + +from langchain_community.document_loaders.parsers.language.python import PythonSegmenter + + +def parse_all_classes_methods(code: str) -> list[str]: + methods = [] + tree = ast.parse(code) + lines = code.splitlines(keepends=True) + for node in tree.body: + if isinstance(node, ast.ClassDef): + class_name = node.name + for item in node.body: + if isinstance(item, ast.FunctionDef): + method_start_line_index = item.lineno - 1 + method_end_line_index = item.end_lineno + method_lines = lines[method_start_line_index:method_end_line_index] + if method_lines: + method_indentation_level = len(method_lines[0]) - len(method_lines[0].lstrip()) + + # Remove the method's specific indentation from all its lines + # This "dedents" the method content to column 0 + dedented_method_lines = [] + for line in method_lines: + if line.startswith(' ' * method_indentation_level): + dedented_method_lines.append(line[method_indentation_level:]) + else: + dedented_method_lines.append(line) + methods.append("".join(dedented_method_lines).strip()) + for i , method in enumerate(methods): + methods[i] = f'{method}\n#(class: {class_name})' + + return methods + +# Compiled once for speed +PY2_PRINT_REGEX = re.compile(r'^\s*print (?![\(])', re.MULTILINE) +PY2_EXCEPT_REGEX = re.compile(r'^\s*except\s+(?:[\w.]+|\([^)]+\))\s*,\s*\w+\s*:', re.MULTILINE) # matches: except Exception, e: +PY2_RAW_INPUT_REGEX = re.compile(r'\braw_input\s*\(', re.MULTILINE) # matches: raw_input( +PY2_RAISE_REGEX = re.compile(r'^\s*raise\s+\w+\s*,', re.MULTILINE) # matches: raise Exception, + + +def is_python2_code(code: str) -> bool: + """Check if the code is Python 2 code. + Args: + code: The code to check. + Returns: + True if the code is Python 2 code, False otherwise. + Checks: + 1. Check Shebang (Instant) + 2. Check for 'print "string"' + 3. Check for old exception syntax 'except Exception, e:' + 4. Check for 'raw_input()' function + 5. Check for old-style 'raise Exception, "Error"' syntax + """ + if code.startswith("#!") and "python2" in code[:50]: + return True + if PY2_PRINT_REGEX.search(code): + return True + if PY2_EXCEPT_REGEX.search(code): + return True + if PY2_RAW_INPUT_REGEX.search(code): + return True + if PY2_RAISE_REGEX.search(code): + return True + + return False + +class PythonSegmenterWithClassesMethods(PythonSegmenter): + + def __init__(self, code: str): + super().__init__(code) + self._code = code + + def extract_functions_classes(self) -> List[str]: + if is_python2_code(self._code): + return [] + + function_classes = super().extract_functions_classes() + classes_methods = [] + for func in function_classes: + if func.startswith('class '): + classes_methods.extend(parse_all_classes_methods(func)) + function_classes.extend(classes_methods) + return function_classes \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/source_code_git_loader.py b/src/exploit_iq_commons/utils/source_code_git_loader.py new file mode 100644 index 000000000..0d2e6f1d7 --- /dev/null +++ b/src/exploit_iq_commons/utils/source_code_git_loader.py @@ -0,0 +1,543 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import contextlib +import os +import tempfile +import typing +import urllib.parse +from pathlib import Path + +from git import Blob as GitBlob +from git import Repo +from git.exc import GitCommandError +from langchain_community.document_loaders.blob_loaders.schema import BlobLoader +from langchain_core.document_loaders.blob_loaders import Blob +from tqdm import tqdm + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.credential_client import ( + AES_256_KEY_SIZE_BYTES, + AuthenticationError, + CredentialNotFoundError, + _credential_id_ctx, + fetch_and_decrypt_credential, +) +from exploit_iq_commons.utils.dep_tree import INSTALLED_PACKAGES_FILE, TRANSITIVE_ENV_NAME +from exploit_iq_commons.utils.transitive_code_searcher_tool import ( + TransitiveCodeSearcher, +) + +# Maximum number of .py files a site-packages package may contain before it +# is excluded from automatic indexing. +_SITE_PKG_MAX_PY_FILES: int = 150 + +# Directory-name suffixes/names to skip when scanning site-packages. +_SITE_PKG_SKIP_SUFFIXES: frozenset[str] = frozenset({".dist-info", ".egg-info"}) +_SITE_PKG_SKIP_DIRS: frozenset[str] = frozenset({ + "__pycache__", + "ansible_collections", # excluded: exceeds file-count threshold by a large margin + "tests", + "test", + "docs", + "doc", +}) + +# Patterns always excluded from code indexing regardless of include/exclude config. +# These are internal directories or files that contain binary/metadata content +# and should never be searched or indexed. +_ALWAYS_EXCLUDED_PATTERNS: tuple[str, ...] = ( + ".git/**/*", # Files inside .git/ + ".git/**", # Directories inside .git/ + ".git", # The .git directory itself +) + +PathLike = typing.Union[str, os.PathLike] + + +logger = LoggingFactory.get_agent_logger(__name__) + + +class SourceCodeGitLoader(BlobLoader): + """ + Load `Git` repository files. + + The Repository can be local on disk available at `repo_path`, + or remote at `clone_url` that will be cloned to `repo_path`. + Currently, supports only text files. + + Each document represents one file in the repository. The `path` points to + the local Git repository, and the `ref` specifies the git reference to load + files from. By default, it loads from the `main` branch. + + If `credential_id` and `jwt_token` are provided, the loader will fetch + encrypted credentials from the backend and use them for cloning private + repositories. Credentials are decrypted in memory and cleaned up immediately + after the clone operation. + """ + + def __init__( + self, + repo_path: PathLike, + clone_url: str | None = None, + ref: typing.Optional[str] = "main", + include: typing.Optional[typing.Iterable[str]] = None, + exclude: typing.Optional[typing.Iterable[str]] = None, + ): + """ + Initialize the Git loader. + + Parameters + ---------- + repo_path : PathLike + Path to the local Git repository. + clone_url : str | None, optional + URL to the remote Git repository, by default None + ref : typing.Optional[str], optional + Commit ID to load files from, by default "main" + include : typing.Optional[typing.Iterable[str]], optional + A list of file patterns to include. Uses the glob syntax, by default None + exclude : typing.Optional[typing.Iterable[str]], optional + A list of file patterns to exclude. Uses the glob syntax, by default None + + Credential for private repository authentication is read from the current + async context (set via ``credential_context()`` in cve_generate_vdbs). + """ + + self.repo_path = Path(repo_path) + self.clone_url = clone_url + self.ref = ref + + self.include = include + self.exclude = exclude + + self._repo: Repo | None = None + + def _get_credential_env_vars(self) -> tuple[str, str]: + """ + Read and validate required environment variables for credential operations. + + Returns + ------- + tuple[str, str] + A tuple of (backend_url, encryption_key). + + Raises + ------ + ValueError + If CLIENT_BACKEND_URL or CREDENTIAL_ENCRYPTION_KEY is missing or empty. + ValueError + If CREDENTIAL_ENCRYPTION_KEY is shorter than 32 bytes when UTF-8 encoded. + + """ + backend_url = os.environ.get("CLIENT_BACKEND_URL", "").strip() + if not backend_url: + raise ValueError("CLIENT_BACKEND_URL is required") + encryption_key = os.environ.get("CREDENTIAL_ENCRYPTION_KEY", "").strip() + if not encryption_key: + raise ValueError("CREDENTIAL_ENCRYPTION_KEY is required") + key_bytes = encryption_key.encode("utf-8") + if len(key_bytes) < AES_256_KEY_SIZE_BYTES: + raise ValueError( + f"CREDENTIAL_ENCRYPTION_KEY must be at least {AES_256_KEY_SIZE_BYTES} bytes " + f"when UTF-8 encoded, got {len(key_bytes)} bytes." + ) + return backend_url, encryption_key + + def load_repo(self): + """ + Load the Git repository and return the GitPython `Repo` object. + + Returns + ------- + Repo + GitPython `Repo` object representing the Git repository. + + Raises + ------ + ValueError + If the repository path does not exist. + ValueError + If a different repository is already cloned at the path. + """ + + if (self._repo is not None): + return self._repo + + credential_id = _credential_id_ctx.get() + credential_clone = False + auth_mode = "private" if credential_id else "public" + logger.info( + "load_repo: url='%s' ref='%s' path='%s' auth=%s", + self.clone_url, + self.ref, + self.repo_path, + auth_mode, + ) + + if not os.path.exists(self.repo_path) and self.clone_url is None: + raise ValueError(f"Path {self.repo_path} does not exist") + elif self.clone_url: + # If the repo_path already contains a git repository, verify that it's the + # same repository as the one we're trying to clone. + if os.path.isdir(os.path.join(self.repo_path, ".git")): + repo = Repo(self.repo_path) + # If the existing repository is not the same as the one we're trying to + # clone, raise an error. + if repo.remotes.origin.url != self.clone_url: + raise ValueError("A different repository is already cloned at this path.") + + logger.info("Updating existing Git repo for URL '%s' @ '%s' auth=%s", self.clone_url, self.ref, auth_mode) + # Ensure existing shallow clones have a widened fetch refspec + # so that branch names are resolvable (APPENG-4896). + repo.git.config("remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*") + else: + logger.info("Cloning repository from URL: '%s' @ '%s' auth=%s", self.clone_url, self.ref, auth_mode) + + if credential_id: + repo = self._clone_with_credential(credential_id) + credential_clone = True + else: + # Create a shallow clone of the repository without checking out. + # GIT_TERMINAL_PROMPT=0: never block on username/password prompts. + + clone_env = {"GIT_TERMINAL_PROMPT": "0"} + try: + repo = Repo.clone_from( + self.clone_url, + self.repo_path, + depth=1, + no_checkout=True, + env=clone_env, + ) + except GitCommandError as e: + logger.error("Failed to clone repository '%s': %s", self.clone_url, e) + raise ValueError( + f"Failed to clone repository '{self.clone_url}'" + ) from e + # depth=1 implies --single-branch, which narrows the fetch + # refspec to only the default branch. Widen it so that + # fetching any branch creates a remote-tracking ref and + # `git checkout ` succeeds (APPENG-4896). + repo.git.config("remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*") + + # Set repo as git safe directory to avoid errors if directory ownership is changed + # https://git-scm.com/docs/git-config#Documentation/git-config.txt-safedirectory + if self.clone_url: + with repo.config_writer(config_level="global") as config: + config.add_value("safe", "directory", str(self.repo_path.absolute())) + + else: + repo = Repo(self.repo_path) + logger.info("Using existing Git repo at path: '%s' @ '%s'", self.repo_path, self.ref) + + # Fetch the ref (skip if already cloned with credentials) + if not credential_clone: + if credential_id: + self._fetch_authenticated(repo, credential_id) + else: + self._fetch_public(repo) + + repo.git.checkout(self.ref, "--force") + + logger.info("Loaded Git repository at path: '%s' @ '%s'", self.repo_path, self.ref) + TransitiveCodeSearcher.download_dependencies(self.repo_path) + self._repo = repo + + return repo + + def _fetch_public(self, repo: Repo) -> None: + """Fetch from public repository.""" + self._do_fetch(repo) + + def _fetch_authenticated(self, repo: Repo, credential_id: str) -> None: + """Fetch with authentication (PAT or SSH).""" + backend_url, encryption_key = self._get_credential_env_vars() + + try: + cred = fetch_and_decrypt_credential( + credential_id=credential_id, + backend_url=backend_url, + encryption_key=encryption_key, + ) + except CredentialNotFoundError: + logger.warning("Credential expired or consumed: %s, falling back to public fetch", credential_id) + self._do_fetch(repo) + return + except AuthenticationError: + logger.exception("Authentication failed for credential: %s", credential_id) + raise + + try: + if cred["credential_type"] == "PAT": + logger.info("Using PAT auth for fetch: username=%s", cred["username"]) + self._do_fetch_with_pat(repo, cred["secret_value"], cred["username"]) + elif cred["credential_type"] == "SSH_KEY": + logger.info("Using SSH key auth for fetch") + self._do_fetch_with_ssh_key(repo, cred["secret_value"]) + else: + raise ValueError(f"Unsupported credential type: {cred['credential_type']}") + finally: + del cred + + def _do_fetch(self, repo: Repo) -> None: + """Core fetch logic: fetch ref (branch, tag, or commit SHA). + + Tries the tag-specific refspec first because it creates a local + tag ref that ``git checkout`` can resolve. A generic fetch is + tried second (handles branches and commit SHAs). The two fetches + must NOT both run on a shallow clone — consecutive shallow fetches + trigger ``fatal: shallow file has changed since we read it``. + """ + # Try as a tag first — creates a local ref under refs/tags/ + try: + repo.git.fetch("origin", f"refs/tags/{self.ref}:refs/tags/{self.ref}", "--depth=1", "--force") + logger.info("Fetched ref '%s' as tag", self.ref) + return + except GitCommandError as e: + logger.warning("Ref '%s' is not a tag, retrying as branch/commit SHA: %s", self.ref, e) + + # Not a tag — try as a branch or commit SHA + try: + repo.git.fetch("origin", self.ref, "--depth=1", "--force") + logger.info("Fetched ref '%s' as branch/commit", self.ref) + return + except GitCommandError as e: + logger.warning("Ref '%s' not found on remote, checking if it exists locally: %s", self.ref, e) + + # Neither fetch worked — check if ref already exists locally + try: + repo.commit(self.ref) + logger.info("Ref '%s' already exists locally", self.ref) + except GitCommandError: + raise GitCommandError( + ["git", "fetch"], 128, + f"Could not fetch ref '{self.ref}' from origin (tried as tag, branch, and commit)" + ) from None + + def _do_fetch_with_pat(self, repo: Repo, pat: str, username: str | None) -> None: + """Fetch with PAT: temporarily update URL, fetch, restore URL.""" + original_url = repo.remotes.origin.url + parsed = urllib.parse.urlparse(original_url) + user_part = f"{username}:{pat}" if username else f"token:{pat}" + auth_url = parsed._replace(netloc=f"{user_part}@{parsed.netloc}").geturl() + + try: + repo.remotes.origin.set_url(auth_url) + self._do_fetch(repo) + finally: + repo.remotes.origin.set_url(original_url) + del auth_url, pat + + + def _do_fetch_with_ssh_key(self, repo: Repo, key: str) -> None: + """Fetch with SSH key: write temp file, set env var, fetch, cleanup.""" + ssh_key_path = None + try: + with tempfile.NamedTemporaryFile(mode="w", suffix=".pem", delete=False, dir="/tmp") as f: + f.write(key) + ssh_key_path = f.name + Path(ssh_key_path).chmod(0o600) + git_ssh_cmd = f"ssh -i {ssh_key_path} -o StrictHostKeyChecking=no -o BatchMode=yes" + with repo.git.custom_environment(GIT_SSH_COMMAND=git_ssh_cmd): + self._do_fetch(repo) + finally: + if ssh_key_path and Path(ssh_key_path).exists(): + Path(ssh_key_path).unlink() + del key + + def _clone_with_credential(self, credential_id: str) -> Repo: + """Clone with authenticated credential (PAT or SSH key).""" + backend_url, encryption_key = self._get_credential_env_vars() + + cred = fetch_and_decrypt_credential( + credential_id=credential_id, + backend_url=backend_url, + encryption_key=encryption_key, + ) + + try: + if cred["credential_type"] == "PAT": + logger.info("Using PAT auth for clone: url=%s username=%s", self.clone_url, cred["username"]) + return self._clone_with_pat(cred["secret_value"], cred["username"]) + if cred["credential_type"] == "SSH_KEY": + logger.info("Using SSH key auth for clone: url=%s", self.clone_url) + return self._clone_with_ssh_key(cred["secret_value"]) + raise ValueError(f"Unsupported credential type: {cred['credential_type']}") + finally: + del cred + + def _clone_with_pat(self, pat: str, username: str | None) -> Repo: + """Clone using PAT: embed in URL, clone, reset URL to clean version.""" + parsed = urllib.parse.urlparse(self.clone_url) + user_part = f"{username}:{pat}" if username else f"token:{pat}" + auth_url = parsed._replace(netloc=f"{user_part}@{parsed.netloc}").geturl() + + repo = None + try: + repo = Repo.clone_from(auth_url, self.repo_path, depth=1, no_checkout=True) + repo.git.config("remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*") + self._do_fetch(repo) + return repo + finally: + if repo: + repo.remotes.origin.set_url(self.clone_url) + del auth_url, pat + + + def _clone_with_ssh_key(self, key: str) -> Repo: + """Clone using SSH key: convert URL to SSH format, write temp key file.""" + ssh_url = self._https_to_ssh_url(self.clone_url) + ssh_key_path = None + try: + with tempfile.NamedTemporaryFile(mode="w", suffix=".pem", delete=False, dir="/tmp") as f: + f.write(key) + ssh_key_path = f.name + Path(ssh_key_path).chmod(0o600) + git_ssh_cmd = f"ssh -i {ssh_key_path} -o StrictHostKeyChecking=no -o BatchMode=yes" + + repo = Repo.clone_from( + ssh_url, self.repo_path, depth=1, no_checkout=True, + env={"GIT_SSH_COMMAND": git_ssh_cmd}, + ) + repo.git.config("remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*") + with repo.git.custom_environment(GIT_SSH_COMMAND=git_ssh_cmd): + self._do_fetch(repo) + return repo + finally: + if ssh_key_path and Path(ssh_key_path).exists(): + Path(ssh_key_path).unlink() + del key + + + @staticmethod + def _https_to_ssh_url(https_url: str) -> str: + """Convert HTTPS URL to SSH format (required for SSH key auth).""" + if not https_url.startswith("https://"): + return https_url + parsed = urllib.parse.urlparse(https_url) + path = parsed.path.lstrip("/") + if parsed.port and parsed.port != 443: + return f"ssh://git@{parsed.hostname}:{parsed.port}/{path}" + return f"git@{parsed.hostname}:{path}" + + def yield_blobs(self) -> typing.Iterator[Blob]: + """ + Yield the blobs from the Git repository. One blob will be generated for each file in the repo which passes the + include and exclude filters. + + Returns + ------- + typing.Iterator[Blob] + An iterator of `Blob` objects representing the files in the repository. + + Yields + ------ + Iterator[typing.Iterator[Blob]] + An iterator of `Blob` objects representing the files in the repository. + """ + + repo = self.load_repo() + + logger.info("Scanning documents for Git repository at path: '%s'", self.repo_path) + + all_files_in_repo = [str(item.path) for item in repo.tree().traverse() if isinstance(item, GitBlob)] + + base_path = Path(self.repo_path) + + include_files: set[str] = set() + exclude_files: set[str] = set() + + for inc in self.include or ["**/*"]: + include_files = include_files.union(set(str(x.relative_to(base_path)) for x in base_path.glob(inc))) + + # Combine user-provided excludes with always-excluded internal patterns + all_excludes = list(self.exclude or []) + list(_ALWAYS_EXCLUDED_PATTERNS) + for exc in all_excludes: + exclude_files = exclude_files.union(set(str(x.relative_to(base_path)) for x in base_path.glob(exc))) + + # Always include installed_packages.txt when present so that Code + # Keyword Search can answer "is package X installed?" for transitive deps. + installed_pkg_file = base_path / INSTALLED_PACKAGES_FILE + if installed_pkg_file.is_file(): + include_files.add(INSTALLED_PACKAGES_FILE) + logger.debug("Including %s in document index", INSTALLED_PACKAGES_FILE) + + # Include Python source from site-packages so that CCA and Code Keyword + # Search can trace transitive call chains across package boundaries. + # Packages exceeding _SITE_PKG_MAX_PY_FILES .py files and known noisy + # directories are excluded to bound indexing cost. + self._add_site_packages_blobs(base_path, include_files) + + # Take the include files and remove the exclude files. + final_files = include_files - exclude_files + + logger.info("Processing %d files in the Git repository at path: '%s'", len(final_files), self.repo_path) + + for f in tqdm(final_files): + abs_file_path = base_path / f + if abs_file_path.is_file(): + try: + metadata = { + "source": f, + "file_path": f, + "file_name": abs_file_path.name, + "file_type": abs_file_path.suffix, + } + yield Blob.from_path(abs_file_path, metadata=metadata) + except Exception as e: + logger.warning("Failed to read blob for '%s'. Ignoring this file. Error: %s", abs_file_path, e) + else: + logger.debug("Skipping path as it is a directory, not a file: '%s'", abs_file_path) + + @staticmethod + def _add_site_packages_blobs(base_path: Path, include_files: set[str]) -> None: + """Add Python source files from transitive_env site-packages to include_files. + + Only packages with at most ``_SITE_PKG_MAX_PY_FILES`` .py files are + indexed. Known heavy or noisy directories are skipped. + Files inside ``__pycache__`` sub-directories are always excluded. + """ + added_pkgs: list[str] = [] + skipped_pkgs: list[str] = [] + + for sp_dir in base_path.glob(f"{TRANSITIVE_ENV_NAME}/lib/*/site-packages"): + if not sp_dir.is_dir(): + continue + for pkg_dir in sp_dir.iterdir(): + if not pkg_dir.is_dir(): + continue + # Skip metadata directories and known noisy dirs + if any(pkg_dir.name.endswith(sfx) for sfx in _SITE_PKG_SKIP_SUFFIXES): + continue + if pkg_dir.name in _SITE_PKG_SKIP_DIRS: + continue + py_files = [ + f for f in pkg_dir.rglob("*.py") + if "__pycache__" not in f.parts + ] + if len(py_files) <= _SITE_PKG_MAX_PY_FILES: + for f in py_files: + include_files.add(str(f.relative_to(base_path))) + added_pkgs.append(pkg_dir.name) + else: + skipped_pkgs.append(f"{pkg_dir.name}({len(py_files)} files)") + + if added_pkgs: + logger.info("Indexed %d site-packages package(s) for transitive analysis: %s", + len(added_pkgs), ", ".join(added_pkgs)) + if skipped_pkgs: + logger.info("Skipped %d oversized site-packages package(s): %s", + len(skipped_pkgs), ", ".join(skipped_pkgs)) diff --git a/src/exploit_iq_commons/utils/source_rpm_downloader.py b/src/exploit_iq_commons/utils/source_rpm_downloader.py new file mode 100644 index 000000000..01fa0a240 --- /dev/null +++ b/src/exploit_iq_commons/utils/source_rpm_downloader.py @@ -0,0 +1,725 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from pathlib import Path +import gzip +import xml.etree.ElementTree as ET +import subprocess +from concurrent.futures import ThreadPoolExecutor, as_completed +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry +from dataclasses import dataclass +from typing import List, Dict, Any, Optional +import threading +from exploit_iq_commons.data_models.info import SBOMPackage +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) + + +@dataclass +class RepoUrl: + url: str + platform: str # e.g., 'el9', 'el10' + + +def extract_archives_in_folder(folder: str): + """ + Extracts all .tar.* files in the given folder to subdirectories using bsdtar. + Supports .tar.gz, .tgz, .tar.xz, .txz, .tar.bz2, .tbz2, .tar.lzma, .tlz, and .tar. + + :param folder: Path to the folder to scan. + """ + shorthand_extensions = ('.tar', '.tgz', '.txz', '.tbz', '.tbz2', '.tlz') + + folder_path = Path(folder) + + for file in folder_path.iterdir(): + if not file.is_file(): + continue + + name_lower = file.name.lower() + is_tarball = '.tar.' in name_lower or name_lower.endswith(shorthand_extensions) + if not is_tarball: + continue + + suffix = ''.join(file.suffixes[-2:]) if len(file.suffixes) >= 2 else file.suffix + full_suffix = suffix if suffix.startswith('.tar') else file.suffix + + # Determine output directory name + output_dir = file.with_suffix('').with_suffix('').stem if full_suffix.startswith('.tar') else file.stem + output_path = folder_path / output_dir + + logger.debug(f"Extracting: {file.name} → {output_path}") + if output_path.exists(): + logger.debug(f"Output path already exists: {output_path}") + return + output_path.mkdir(exist_ok=True) + + try: + result = subprocess.run( + ['bsdtar', '-xf', str(file), '-C', str(output_path)], + capture_output=True, + text=True + ) + if result.returncode != 0: + logger.error(f"Failed to extract {file.name}: {result.stderr.strip()}") + except Exception as e: + logger.error(f"Could not run extraction for {file.name}: {e}") + +class RPMDependencyManager: + """ + Encapsulates RPM download, dependency tree building, pruning, and output writing logic. + Can be used from both scripts and library code. + + Implements singleton pattern to ensure single instance across the application. + """ + _instance = None + _lock = threading.Lock() + + def __new__(cls): + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self): + if not hasattr(self, '_initialized'): + # Initialize only once + self.base_urls = [ + RepoUrl(url="https://vault.centos.org/8-stream/BaseOS/Source/", platform="el8"), + RepoUrl(url="https://vault.centos.org/8-stream/AppStream/Source/", platform="el8"), + RepoUrl(url="https://mirror.stream.centos.org/9-stream/BaseOS/source/tree/", platform="el9"), + RepoUrl(url="https://mirror.stream.centos.org/9-stream/AppStream/source/tree/", platform="el9"), + RepoUrl(url="https://mirror.stream.centos.org/10-stream/BaseOS/source/tree/", platform="el10"), + RepoUrl(url="https://mirror.stream.centos.org/10-stream/AppStream/source/tree/", platform="el10"), + RepoUrl(url="https://mirror.stream.centos.org/11-stream/BaseOS/source/tree/", platform="el11"), + RepoUrl(url="https://mirror.stream.centos.org/11-stream/AppStream/source/tree/", platform="el11"), + RepoUrl(url="https://mirror.stream.centos.org/12-stream/BaseOS/source/tree/", platform="el12"), + RepoUrl(url="https://mirror.stream.centos.org/12-stream/AppStream/source/tree/", platform="el12") + ] + # Default RPM cache directory + self.rpm_cache_dir = "/morpheus-data/rpms/" + # Unit Test mode toggle (default disabled) + self._unit_test_mode: bool = False + self._initialized = True + self._sbom: list[SBOMPackage] = [] + logger.info(f"RPMDependencyManager singleton initialized with default cache dir: {self.rpm_cache_dir}") + + @classmethod + def get_instance(cls): + """Get the singleton instance of RPMDependencyManager""" + return cls() + + def set_rpm_cache_dir(self, cache_dir: str): + """Set the RPM cache directory for all future operations""" + self.rpm_cache_dir = cache_dir + + # Ensure the cache directory exists + cache_path = Path(cache_dir) + if not cache_path.exists(): + try: + cache_path.mkdir(parents=True, exist_ok=True) + logger.info("Created RPM cache directory: %s", cache_dir) + except Exception as e: + logger.error("Failed to create RPM cache directory %s: %s", cache_dir, e) + # Fall back to default directory if creation fails + self.rpm_cache_dir = "/morpheus-data/rpms/" + logger.warning("Falling back to default cache directory: %s", self.rpm_cache_dir) + else: + logger.info("RPM cache directory updated to: %s", cache_dir) + + def get_rpm_cache_dir(self) -> str: + """Get the current RPM cache directory""" + return self.rpm_cache_dir + + def enableUnitTestMode(self) -> None: + """Enable Unit Test mode to relax version matching during RPM URL resolution.""" + self._unit_test_mode = True + + def disableUnitTestMode(self) -> None: + """Disable Unit Test mode and restore strict version matching.""" + self._unit_test_mode = False + + def isUnitTestMode(self) -> bool: + """Return whether Unit Test mode is enabled.""" + return self._unit_test_mode + + @property + def sbom(self) -> list[SBOMPackage]: + """Get the SBOM list""" + return self._sbom + + @sbom.setter + def sbom(self, value: list[SBOMPackage]) -> None: + """Set the SBOM list""" + if not isinstance(value, list): + raise TypeError("sbom must be a list") + self._sbom = value + + @property + def container_image(self) -> str: + """Get the container image string""" + return getattr(self, '_container_image', '') + + @container_image.setter + def container_image(self, value: str) -> None: + """Set the container image string""" + if not isinstance(value, str): + raise TypeError("container_image must be a string") + self._container_image = value + + def threaded_rpm_download(self, sbom, project_name, download_path, max_workers=1): + """ + Download RPMs in parallel using threads. Each SBOM entry is downloaded in a separate thread. + """ + logger.info(f"Starting threaded RPM download with {max_workers} workers...") + # Use singleton's cache directory instead of hardcoded value + rpm_cache_dir = self.rpm_cache_dir + downloader = SourceRPMDownloader( + project_name, sbom, self.base_urls, + download_path, rpm_cache_dir, max_workers, + unit_test_mode=self._unit_test_mode + ) + downloader.run() + + +class SourceRPMDownloader: + def __init__(self, prj_name, sbom_info, repo_urls: List[RepoUrl], work_dir="ext_libs", rpm_cache_dir=None, max_workers=1, unit_test_mode: bool = False): + self.sbom_packages = sbom_info + self.repo_urls = repo_urls # Now a list of RepoUrl objects + self.work_dir = Path(work_dir) + # Unit Test mode: when enabled, ignore exact version during URL resolution + self.unit_test_mode = unit_test_mode + + # Set up RPM cache directory + if rpm_cache_dir is None: + # No cache directory specified - disable caching + self.rpm_cache_dir = None + logger.info("RPM caching disabled (no cache directory specified)") + else: + self.rpm_cache_dir = Path(rpm_cache_dir) + # Ensure cache directory exists + self.rpm_cache_dir.mkdir(parents=True, exist_ok=True) + logger.debug(f"RPM cache directory: {self.rpm_cache_dir}") + + # Build cache index from existing files + self.cache_index = self._build_cache_index() + self.cache_hit =0 + self.cache_miss =0 + self.repo_metadata = {} + self.max_workers = max_workers + self.project_name = prj_name + # Thread-safe session with retries + self.session = requests.Session() + retries = Retry( + total=3, backoff_factor=0.3, status_forcelist=[500, 502, 503, 504], allowed_methods=["GET"] + ) + adapter = HTTPAdapter(max_retries=retries) + self.session.mount("http://", adapter) + self.session.mount("https://", adapter) + + def _parse_rpm_filename(self, filename: str) -> Optional[Dict[str, str]]: + """ + Parse RPM filename to extract package info. + Args: + filename: RPM filename string + + Returns: + Optional[Dict[str, str]]: Parsed package info dictionary with keys: + 'name', 'version', 'release', 'platform', 'filename', or None if invalid + """ + if not filename.endswith('.src.rpm'): + return None + + #filesystem-3.16-5.el9.src.rpm + #fonts-rpm-macros-2.0.5-7.el9.1.src.rpm + #libsemanage-3.6-5.el9_6.src.rpm + #python-setuptools-53.0.0-13.el9_6.1.src.rpm + + + # Remove .src.rpm extension + base_name = filename[:-8] + + + # Split by dashes to get parts + parts = base_name.split('-') + if len(parts) < 3: + return None + #fonts,rpm,macros,2.0.5,7.el9.1 + if len(parts) == 3: + package_name = parts[0] + elif len(parts) > 3: + # Build package name from all parts except last 2 + package_name_parts = [] + for i in range(len(parts) - 2): + package_name_parts.append(parts[i]) + package_name = '-'.join(package_name_parts) + # Last 2 elements: version and platform + version_part = parts[-2] + full_platform = parts[-1] + + #cases + #5.el9 + #7.el9.1 + #5.el9_6 + #13.el9_6.1 + platform = None + if 'el' in full_platform: + # Extract base platform by removing _ and . and finding el followed by digits + import re + match = re.search(r'el\d+', full_platform) + if match: + platform = match.group() + else: + platform = full_platform + + return { + 'name': package_name, + 'version': version_part, + 'release': full_platform, + 'platform': platform, + 'filename': filename + } + + def _build_cache_index(self): + """ + Build cache index from existing RPM files in cache directory. + + Returns: + dict: Cache index with key format: (package_name, version, platform) + """ + if self.rpm_cache_dir is None: + return {} + + cache_index = {} + + for rpm_file in self.rpm_cache_dir.glob("*.src.rpm"): + parsed = self._parse_rpm_filename(rpm_file.name) + if parsed: + # Key: (package_name, version, platform) + key = (parsed['name'], parsed['version'], parsed['platform']) + cache_index[key] = { + 'filepath': rpm_file, + 'release': parsed['release'], + 'filename': parsed['filename'] + } + #logger.debug(f"Cached RPM: {parsed['name']}-{parsed['version']}-{parsed['platform']}") + + logger.info(f"Cache index built: {len(cache_index)} RPMs found") + return cache_index + + def _get_cache_path(self, package: Dict[str, Any]) -> Path: + """ + Generate cache file path for a package. + + Args: + package: Package dictionary containing name, version, rel information + + Returns: + Path: Full path to cached RPM file + """ + # Create cache filename: package-version-release.src.rpm + cache_filename = f"{package['name']}-{package['version']}-{package['rel']}.src.rpm" + return self.rpm_cache_dir / cache_filename + + def _find_cached_rpm(self, package: Dict[str, Any], platform_version: str) -> Optional[Path]: + """ + Find cached RPM using the cache index. + + Args: + package: Package dictionary containing name, version, rel information + platform_version: Platform version string (e.g., 'el9', 'el10') + + Returns: + Optional[Path]: Path to cached RPM file if found, None otherwise + """ + if self.rpm_cache_dir is None or not self.cache_index: + return None + + package_name = package['name'] + package_version = package['version'] + + # Exact match with platform version + key = (package_name, package_version, platform_version) + if key in self.cache_index: + return self.cache_index[key]['filepath'] + return None + + def _is_cached(self, package, platform_version): + """ + Check if RPM is already cached. + + Args: + package: Package dictionary with name, version, rel + platform_version: Platform version (e.g., 'el9', 'el10') + + Returns: + bool: True if RPM exists in cache + """ + return self._find_cached_rpm(package, platform_version) is not None + + def _download_to_cache(self, package: Dict[str, Any], download_url: str) -> bool: + """ + Download RPM and store in cache. + + Args: + package: Package dictionary containing package information + download_url: URL string to download RPM from + + Returns: + bool: True if download and cache successful, False otherwise + """ + if self.rpm_cache_dir is None: + return False + cache_path = self._get_cache_path(package) + try: + logger.info(f"Downloading {package['name']} to cache: {cache_path}") + data = self.session.get(download_url, timeout=120).content + cache_path.write_bytes(data) + #logger.info(f"Cached {package['name']} ({len(data)} bytes)") + #add file to cache + parsed = self._parse_rpm_filename(cache_path.name) + if parsed: + key = (parsed['name'], parsed['version'], parsed['platform']) + self.cache_index[key] = { + 'filepath': cache_path, + 'release': parsed['release'], + 'filename': parsed['filename'] + } + return True + except Exception as e: + logger.error(f"Failed to download {package['name']} to cache: {e}") + return False + + def _copy_from_cache(self, package: Dict[str, Any], work_dir: Path, platform_version: str) -> Optional[Path]: + """ + Copy RPM from cache to work directory. + + Args: + package: Package dictionary containing package information + work_dir: Target work directory (Path object) + platform_version: Platform version string (e.g., 'el9', 'el10') + + Returns: + Optional[Path]: Path to copied RPM file in work directory, or None if failed + """ + if self.rpm_cache_dir is None: + return None + + # Find the cached RPM (might have different release than expected) + cache_path = self._find_cached_rpm(package, platform_version) + if cache_path is None: + logger.error(f"No cached RPM found for {package['name']}") + return None + + # work_dir is already the package directory, so just use the filename + work_path = work_dir / cache_path.name + + try: + import shutil + # Ensure the target directory exists + work_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(cache_path, work_path) + #logger.info(f"Copied {package['name']} from cache to {work_path}") + return work_path + except Exception as e: + logger.error(f"Failed to copy {package['name']} from cache: {e}") + return None + + def parse_sbom(self): + logger.info("Parsing SBOM input...") + packages = [] + filter_pkgs = ["glibc"] + standard_pkg = None + filter_pkgs.append(self.project_name) + platform_version = None + for pkg in self.sbom_packages: + versions = pkg.version.split('-') + version_no_epoch = versions[0].split(':', 1)[-1] + rel = versions[1] if len(versions) > 1 else '' + # Try to extract el9/el10 from release string + if not platform_version and 'el' in rel: + for part in rel.split('.'): + if part.startswith('el') and part[2:].isdigit(): + platform_version = part + break + skip_pkg = False + for f_pkg in filter_pkgs: + if f_pkg in pkg.name : + skip_pkg = True + if f_pkg == pkg.name: + standard_pkg = {'short_name': pkg.name,'name': pkg.name,'version': version_no_epoch,'rel': rel} + break + if skip_pkg: + continue + + packages.append({ + 'short_name': pkg.name, + 'name': pkg.name, + 'version': version_no_epoch, + 'rel': rel + }) + if standard_pkg : + packages.append(standard_pkg) + logger.info(f"Found {len(packages)} packages in SBOM, platform: {platform_version}") + return packages, platform_version + + @staticmethod + def extract_src_rpm(rpm_path: Path, extract_dir: Path): + #logger.info(f" Extracting {rpm_path.name} to {extract_dir} ...") + extract_dir.mkdir(parents=True, exist_ok=True) + try: + base_rpm_path = os.path.basename(rpm_path) + # Use bsdtar to extract the RPM + result = subprocess.run([ + "bsdtar", "-xf",base_rpm_path + ], cwd=extract_dir, capture_output=True) + if result.returncode == 0: + # After extracting, extract any tar files in the folder + extract_archives_in_folder(str(extract_dir)) + else: + logger.error(f" Extraction failed with return code {result.returncode}: {result.stderr.decode().strip()}") + except Exception as e: + logger.error(f" Extraction error: {e}") + + def _get_repo_metadata(self, base_url: str): + if base_url in self.repo_metadata: + return self.repo_metadata[base_url] + logger.info(f"Fetching metadata for repo: {base_url}") + try: + repomd_url = base_url.rstrip('/') + "/repodata/repomd.xml" + response = self.session.get(repomd_url, timeout=30) + response.raise_for_status() + root = ET.fromstring(response.content) + primary_href = next( + data.find("{*}location").attrib["href"] + for data in root.findall(".//{*}data") if data.attrib.get("type") == "primary" + ) + primary_url = base_url.rstrip('/') + "/" + primary_href + response = self.session.get(primary_url, timeout=60) + response.raise_for_status() + xml_bytes = gzip.decompress(response.content) + tree = ET.fromstring(xml_bytes) + self.repo_metadata[base_url] = tree + + ######## + #filename ="repo_meta.xml" + #root2 = ET.fromstring(xml_bytes) + #tree2 = ET.ElementTree(root2) + #tree2.write(filename, encoding="utf-8", xml_declaration=True) + #logger.debug(f"XML string saved to {filename} successfully!") + return tree + except Exception as e: + logger.error(f" Could not fetch metadata from {base_url}: {e}") + self.repo_metadata[base_url] = None + return None + + def find_package_url(self, package, repos): + """ + For a given package and list of repos, return (download_url, repo) if found, else (None, None). + When unit_test_mode is enabled, relax match to name + src arch only (ignore exact version). + """ + package_name = package['name'] + package_version = package['version'] + for repo in repos: + base_url = repo.url + metadata = self._get_repo_metadata(base_url) + if not metadata: + continue + # attempt match + try: + if self.unit_test_mode: + pkg_elem = next( + pkg for pkg in metadata.findall(".//{*}package") + if (pkg.find("{*}name") is not None and pkg.find("{*}name").text == package_name) + and (pkg.find("{*}arch") is not None and pkg.find("{*}arch").text == 'src') + ) + else: + pkg_elem = next( + pkg for pkg in metadata.findall(".//{*}package") + if (pkg.find("{*}name") is not None and pkg.find("{*}name").text == package_name) + and (pkg.find("{*}arch") is not None and pkg.find("{*}arch").text == 'src') + and (pkg.find("{*}version") is not None and pkg.find("{*}version").attrib.get("ver") == package_version) + ) + except StopIteration: + continue + href = pkg_elem.find("{*}location").attrib.get("href") + rpm_url = f"{base_url.rstrip('/')}/{href}" + return rpm_url, repo + logger.debug(f"could not find in repo {base_url} : {package_name} - version {package_version}") + return None, None + + def _process_package_from_cache(self, package: Dict[str, Any], package_dir: Path, platform_version: str, can_pkg_download: bool) -> bool: + """ + Try to process package from cache. + + Args: + package: Package dictionary + package_dir: Target package directory + platform_version: Platform version + can_pkg_download: Whether the package can be downloaded if not in cache + + Returns: + bool: True if successfully processed from cache, False otherwise + """ + if self.rpm_cache_dir is None or not platform_version: + return False + + try: + # Ensure we have the RPM in cache (download if needed) + if not self._is_cached(package, platform_version): + self.cache_miss += 1 + if can_pkg_download: + if not self._download_to_cache(package, package['download_url']): + logger.error(f"Failed to download {package['short_name']} to cache") + return False + else: + return False + else: + self.cache_hit += 1 + + # Copy from cache and extract + rpm_path = self._copy_from_cache(package, package_dir, platform_version) + if rpm_path: + self.extract_src_rpm(rpm_path, package_dir) + return True + else: + logger.error(f"Failed to copy {package['short_name']} from cache") + return False + + except Exception as e: + logger.error(f"Failed to process {package['short_name']} from cache: {e}") + return False + + def run(self): + logger.info("Starting Source RPM Download and Extraction Process") + # Track processing statistics + stats = { + 'already_extracted': 0, + 'cache_processed': 0, + 'downloaded': 0, + 'no_download_url': 0, + 'failed': 0, + 'found' : 0 + } + packages, platform_version = self.parse_sbom() + if platform_version: + filtered_repos = [repo for repo in self.repo_urls if platform_version == repo.platform] + if not filtered_repos: + logger.warning(f"No repositories found for platform {platform_version}. Using all repos.") + filtered_repos = self.repo_urls + self.repo_urls = filtered_repos + logger.info(f"Filtered repo URLs to: {[repo.url for repo in self.repo_urls]}") + else: + logger.warning("Could not determine platform version from SBOM; using all repo URLs.") + filtered_repos = self.repo_urls + + # Step 1: Find download URLs for all packages + for pkg in packages: + download_url, repo = self.find_package_url(pkg, filtered_repos) + pkg['download_url'] = download_url + pkg['repo'] = repo.url if repo else None + if download_url : + stats['found'] +=1 + + # Step 2: Download and extract in parallel only for found packages + successful_downloads = 0 + total_packages = len(packages) + not_found = 0 + + def process_package(package): + + can_pkg_download = True + if not package['download_url']: + can_pkg_download = False + + short_name = package['short_name'] + package_dir = self.work_dir / short_name + + try: + # Check if already extracted in work directory + if os.path.exists(package_dir): + return "already_extracted" + + # Try cache first, if successful return + if self._process_package_from_cache(package, package_dir, platform_version, can_pkg_download): + return "cache_processed" + + if not can_pkg_download: + return "no_download_url" + + # Original logic - download directly to work directory + data = self.session.get(package['download_url'], timeout=120).content + package_dir.mkdir(parents=True, exist_ok=True) + out_path = package_dir / Path(package['download_url']).name + out_path.write_bytes(data) + self.extract_src_rpm(out_path, package_dir) + return "downloaded" + + except Exception as e: + logger.error(f"Failed to process package {short_name}: {e}") + return "failed" + + + + # Use ThreadPoolExecutor for I/O bound operations (HTTP downloads, file I/O) + # This is more efficient than multiprocessing for this workload + with ThreadPoolExecutor(max_workers=self.max_workers) as executor: + futures = {executor.submit(process_package, pkg): pkg for pkg in packages} + for future in as_completed(futures): + result = future.result() + if result in stats: + stats[result] += 1 + if result in ['already_extracted', 'cache_processed', 'downloaded']: + successful_downloads += 1 + + + not_found = sum(1 for pkg in packages if not pkg['download_url']) + + # Step 3: logger.debug comprehensive summary report + logger.debug("\n" + "="*80) + logger.debug("RPM PROCESSING SUMMARY REPORT") + logger.debug("="*80) + + # Processing statistics + logger.debug(f"\nProcessing Results:") + logger.debug(f" • Found Packages: {stats['found']}") + logger.debug(f" • Already extracted: {stats['already_extracted']}") + logger.debug(f" • Processed from cache: {stats['cache_processed']}") + logger.debug(f" • Downloaded and processed: {stats['downloaded']}") + logger.debug(f" • No download URL available: {stats['no_download_url']}") + logger.debug(f" • Failed to process: {stats['failed']}") + + # Cache statistics + if self.rpm_cache_dir is not None: + logger.debug(f"\nCache Performance:") + logger.debug(f" • Cache hits: {self.cache_hit}") + logger.debug(f" • Cache misses: {self.cache_miss}") + if self.cache_hit + self.cache_miss > 0: + hit_rate = (self.cache_hit / (self.cache_hit + self.cache_miss)) * 100 + logger.debug(f" • Cache hit rate: {hit_rate:.1f}%") + + # Overall summary + logger.debug(f"\nOverall Summary:") + logger.debug(f" • Total packages: {total_packages}") + logger.debug(f" • Successfully processed: {successful_downloads}") + logger.debug(f" • Not found in repositories: {not_found}") + success_rate = (successful_downloads / total_packages) * 100 if total_packages > 0 else 0 + logger.debug(f" • Success rate: {success_rate:.1f}%") + + logger.debug("="*80) \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/standard_library_cache.py b/src/exploit_iq_commons/utils/standard_library_cache.py new file mode 100644 index 000000000..7435ab2dd --- /dev/null +++ b/src/exploit_iq_commons/utils/standard_library_cache.py @@ -0,0 +1,174 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Standard library cache utilities. +Loads an auto-generated JSON of standard libraries for supported ecosystems +and exposes a simple lookup with lightweight hit/miss counters. Every 5 +lookups, a short summary is logged. Designed to minimize external API usage. +""" + +from __future__ import annotations + +import json +from pathlib import Path +import threading +import pickle +import difflib +# No typing imports needed; we use builtin generics + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.dep_tree import Ecosystem + + +logger = LoggingFactory.get_agent_logger(__name__) + + +class StandardLibraryCache: + """In-memory cache for standard library package/module names per ecosystem. + Implements singleton pattern to ensure single instance across the application. + """ + _instance = None + _lock = threading.Lock() + + def __new__(cls): + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self, json_path: str | Path | None = None) -> None: + if not hasattr(self, '_initialized'): + # Initialize only once + base_path = Path(__file__).resolve().parents[1] + default_json = base_path / "data" / "standard_libs" / "standard_libraries.json" + self.json_path = Path(json_path) if json_path else default_json + + self._ecosystem_to_set: dict[str, set[str]] = {} + self._hits: int = 0 + self._misses: int = 0 + self._lookups_since_last_log: int = 0 + self._data_lock = threading.RLock() + self._initialized = True + self.cache_directory = Path(".cache/am_cache/pickle") + self.cache_file_name = "runtime_std_packages_cache.pkl" + self._load() + + @classmethod + def get_instance(cls): + """Get the singleton instance of StandardLibraryCache""" + return cls() + + def _load(self) -> None: + try: + data = json.loads(self.json_path.read_text(encoding="utf-8")) + except FileNotFoundError: + logger.warning("Standard libraries JSON not found at %s", self.json_path) + data = {} + except json.JSONDecodeError as exc: + logger.error("Failed to parse standard libraries JSON: %s", exc) + data = {} + + for eco_key in (Ecosystem.GO.value, Ecosystem.PYTHON.value, Ecosystem.JAVASCRIPT.value, Ecosystem.JAVA.value): + items: list[str] = data.get(eco_key, []) + normalized = {self._normalize_name(name) for name in items if isinstance(name, str)} + self._ecosystem_to_set[eco_key] = normalized + + @staticmethod + def _normalize_name(name: str) -> str: + # Normalize to lowercase; trim whitespace + return name.strip().lower() + + def is_standard_library(self, package_name: str, ecosystem: Ecosystem | None) -> bool: + if not package_name: + return False + if ecosystem is None: + return False + + with self._data_lock: + eco_key = ecosystem.value + candidates = self._ecosystem_to_set.get(eco_key, set()) + found = self._normalize_name(package_name) in candidates + + if found: + self._hits += 1 + else: + self._misses += 1 + logger.info("StdLibCache lookup (%s): hits=%d misses=%d (eco=%s)", + package_name, self._hits, self._misses, eco_key) + return found + + def get_close_match_list(self, package_name:str, ecosystem: Ecosystem)->list[str]: + package_name = self._normalize_name(package_name) + eco_key = ecosystem.value + pkg_close_matches = [] + with self._data_lock: + candidates = self._ecosystem_to_set.get(eco_key, set()) + pkg_close_matches = difflib.get_close_matches(package_name, candidates, n=5, cutoff=0.6) + return pkg_close_matches + + def load_from_cache(self): + try: + path = self.cache_directory / self.cache_file_name + if path.is_file(): + with open(path, 'rb') as file: + cache_ecosystem_to_set = pickle.load(file) + # Only replace if load was successful + self._ecosystem_to_set = cache_ecosystem_to_set + logger.info("Loaded runtime standard library cache from %s", path) + except Exception as e: + logger.error("Failed to load runtime standard library cache from %s: %s", path, e) + + def set_cache_directory(self, base_directory: str | Path) -> None: + """Set the cache directory and create subdirectory if it doesn't exist. + + Args: + base_directory: The base directory where the std_packages_cache subdirectory will be created + """ + base_path = Path(base_directory) + self.cache_directory = base_path / "std_packages_cache" + + # Create the subdirectory if it doesn't exist + self.cache_directory.mkdir(parents=True, exist_ok=True) + logger.info("Cache directory set to: %s", self.cache_directory) + self.load_from_cache() + + def add_to_cache(self, package_name: str, ecosystem: Ecosystem) -> None: + """Add a package name to the cache for the specified ecosystem. + + Args: + package_name: The package/module name to add to the cache + ecosystem: The ecosystem to add the package to + """ + if not package_name: + logger.warning("Cannot add empty package name to cache") + return + + eco_key = ecosystem.value + normalized_name = self._normalize_name(package_name) + with self._data_lock: + # Initialize the set for this ecosystem if it doesn't exist + if eco_key not in self._ecosystem_to_set: + self._ecosystem_to_set[eco_key] = set() + + # Add the normalized name to the set + self._ecosystem_to_set[eco_key].add(normalized_name) + logger.debug("Added '%s' to standard library cache for ecosystem '%s'", + package_name, eco_key) + path = self.cache_directory / self.cache_file_name + with open(path, 'wb') as file: + pickle.dump(self._ecosystem_to_set, file) + logger.info("Saved runtime standard library cache to %s", path) \ No newline at end of file diff --git a/src/vuln_analysis/utils/string_utils.py b/src/exploit_iq_commons/utils/string_utils.py similarity index 88% rename from src/vuln_analysis/utils/string_utils.py rename to src/exploit_iq_commons/utils/string_utils.py index 5ba6873d3..ad5061978 100644 --- a/src/vuln_analysis/utils/string_utils.py +++ b/src/exploit_iq_commons/utils/string_utils.py @@ -18,14 +18,20 @@ # Find all substrings that start and end with quotes, allowing for spaces before a comma or closing bracket re_quote_capture = re.compile( r""" - (['"]) # Opening quote - ( # Start capturing the quoted content - (?:\\.|[^\\])*? # Non-greedy match for any escaped character or non-backslash character - ) # End capturing the quoted content - \1 # Matching closing quote - (?=\s*,|\s*\]) # Lookahead for whitespace followed by a comma or closing bracket, without including it in the match + (['"]) # Group 1: Opening quote + ( # Group 2: Content + (?: + # Match anything UNLESS it is the specific outer list separator. + # Outer separator = quote + comma + space + quote + CAPITAL LETTER. + # OR it is the closing bracket at the very end of the string. + (?!\1\s*,\s*\1\s*[A-Z]|\1\s*\]\s*$) + . + )* + ) + \1 # Closing quote + (?=\s*[,\]]) # Lookahead for comma or closing bracket """, - re.VERBOSE) + re.VERBOSE | re.DOTALL) # Using the ASCII flag to stricly match numbers 0-9 and not on unicode numbers which include characters like ٤. # https://cve.mitre.org/cve/identifiers/syntaxchange.html diff --git a/src/exploit_iq_commons/utils/tests/__init__.py b/src/exploit_iq_commons/utils/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/exploit_iq_commons/utils/tests/test_python_build_tree.py b/src/exploit_iq_commons/utils/tests/test_python_build_tree.py new file mode 100644 index 000000000..5e3e5c9eb --- /dev/null +++ b/src/exploit_iq_commons/utils/tests/test_python_build_tree.py @@ -0,0 +1,116 @@ +import pytest +from unittest.mock import patch, MagicMock +from pathlib import Path + +from exploit_iq_commons.utils.dep_tree import PythonDependencyTreeBuilder + + +DEPTREE_OUTPUT = ( + 'flask==3.1.2 # flask\n' + ' werkzeug==3.1.3 # werkzeug>=3.1.0\n' + ' click==8.3.0 # click>=8.1.3\n' + 'python-dateutil==2.9.0 # python-dateutil\n' + ' six==1.17.0 # six>=1.5\n' + 'Pillow==11.2.1 # Pillow\n' +) + +REQUIREMENTS_TXT = ( + "flask==3.1.2\n" + "python-dateutil==2.9.0\n" + "Pillow==11.2.1\n" +) + + +class TestPythonBuildTree: + def setup_method(self): + self.builder = PythonDependencyTreeBuilder() + + def _mock_open_side_effect(self, *args, **kwargs): + file_path = str(args[0]) if args else "" + mock_file = MagicMock() + mock_file.__enter__ = MagicMock(return_value=mock_file) + mock_file.__exit__ = MagicMock(return_value=None) + if 'requirements.txt' in file_path: + mock_file.__iter__ = MagicMock(return_value=iter(REQUIREMENTS_TXT.splitlines(keepends=True))) + return mock_file + + def _build(self, deptree_output=DEPTREE_OUTPUT, import_mapping=None): + """Run build_tree with mocked deptree output and _find_module_dirs.""" + if import_mapping is None: + import_mapping = { + "python-dateutil": ["dateutil"], + "pillow": ["PIL"], + "flask": ["flask"], + "werkzeug": ["werkzeug"], + "click": ["click"], + "six": ["six"], + } + + fake_site_packages = Path("/fake/site-packages") + + def fake_find_module_dirs(pkg_name, site_packages): + return import_mapping.get(pkg_name, [pkg_name.replace('-', '_')]) + + with patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=deptree_output), \ + patch('builtins.open', side_effect=self._mock_open_side_effect), \ + patch.object(self.builder, '_find_site_packages', return_value=fake_site_packages), \ + patch.object(self.builder, '_find_module_dirs', side_effect=fake_find_module_dirs): + return self.builder.build_tree(Path("/fake/repo")) + + def test_no_hyphen_underscore_normalization(self): + """Hyphens should NOT be replaced with underscores in tree keys.""" + tree = self._build() + assert "python_dateutil" not in tree + assert "python-dateutil" not in tree # re-keyed to import name + + def test_import_name_rekey_dateutil(self): + """python-dateutil should be re-keyed to 'dateutil' (its import name).""" + tree = self._build() + assert "dateutil" in tree + + def test_import_name_rekey_pillow(self): + """Pillow should be re-keyed to 'pil' (PIL lowercased).""" + tree = self._build() + assert "pil" in tree + + def test_same_name_unchanged(self): + """Packages where PyPI name == import name should keep their key.""" + tree = self._build() + assert "flask" in tree + assert "werkzeug" in tree + + def test_root_project_for_direct_deps(self): + """Direct dependencies from requirements.txt should have ROOT_PROJECT as parent.""" + tree = self._build() + assert "root_project" in tree["flask"] + assert "root_project" in tree["dateutil"] + assert "root_project" in tree["pil"] + + def test_transitive_dep_no_root_project(self): + """Transitive-only deps should NOT have ROOT_PROJECT.""" + tree = self._build() + assert "root_project" not in tree.get("six", []) + assert "root_project" not in tree.get("werkzeug", []) + + def test_parent_relationships_preserved(self): + """Transitive deps should still point to correct parents.""" + tree = self._build() + # werkzeug is a dep of flask + assert "flask" in tree["werkzeug"] + # six is a dep of python-dateutil -> re-keyed to dateutil + assert "dateutil" in tree["six"] + + def test_import_to_pypi_mapping_stored(self): + """Builder should store _import_to_pypi after build_tree.""" + self._build() + assert self.builder._import_to_pypi.get("dateutil") == "python-dateutil" + assert self.builder._import_to_pypi.get("pil") == "pillow" + + def test_no_site_packages_skips_rekey(self): + """If site-packages not found, tree keeps lowercased PyPI names.""" + with patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=DEPTREE_OUTPUT), \ + patch('builtins.open', side_effect=self._mock_open_side_effect), \ + patch.object(self.builder, '_find_site_packages', return_value=None): + tree = self.builder.build_tree(Path("/fake/repo")) + assert "python-dateutil" in tree + assert "pillow" in tree \ No newline at end of file diff --git a/src/exploit_iq_commons/utils/transitive_code_searcher_tool.py b/src/exploit_iq_commons/utils/transitive_code_searcher_tool.py new file mode 100644 index 000000000..12c3455a3 --- /dev/null +++ b/src/exploit_iq_commons/utils/transitive_code_searcher_tool.py @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from pathlib import Path + +from langchain.docstore.document import Document + +from exploit_iq_commons.utils.chain_of_calls_retriever_base import ChainOfCallsRetrieverBase +from exploit_iq_commons.utils.dep_tree import ( + detect_ecosystem, + get_dependency_tree_builder, +) + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, MULTI_LINE_MESSAGE_TRUE + +logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") + + +class TransitiveCodeSearcher: + """ + Transitive code Searcher for code using a Chain Of Calls Retriever object + + Parameters + ---------- + chain_of_calls_retriever : ChainOfCallsRetrieverBase + Chain Of Calls Retriever object that knows how to search in code + + """ + + def __init__(self, chain_of_calls_retriever: ChainOfCallsRetrieverBase): + self.chain_of_calls_retriever = chain_of_calls_retriever + + @staticmethod + def download_dependencies(git_repo_path: Path) -> bool: + """ Download all dependencies according to manifest file in the Git repository + Parameters + ---------- + git_repo_path : Path + Git repository path to fetch the application manifests from + + Returns whether dependencies were downloaded or not. + """ + ecosystem = detect_ecosystem(git_repo_path) + if ecosystem is None: + logger.info(f"Didn't find manifest to install, skipping.. {git_repo_path}") + return False + + try: + logger.info(f"Started installing packages for {ecosystem.value}") + tree_builder = get_dependency_tree_builder(ecosystem) + tree_builder.install_dependencies(git_repo_path) + with open(os.path.join(git_repo_path, 'ecosystem_data.txt'), 'w') as file: + file.write(ecosystem.name) + logger.info(f"Finished installing packages for {ecosystem}") + return True + except NotImplementedError as err: + logger.info(f"Didn't install package using manifest, underlying error --> {str(err)} ") + return False + + def search(self, query: str) -> tuple[bool, list[Document]]: + """search for functions/methods in transitive packages if they're reachable from base application or not + + Parameters + ---------- + query : str + query to search, in the form of "package_name,function_name" or "package_name,class_name.function_name". + + Returns + ------- + bool + Whether the function or method in transitive package is reachable or not from base application + """ + logger.debug(f"Now using transitive code search tool with input =>{query}") + try: + call_hierarchy_list, found_path = self.chain_of_calls_retriever.get_relevant_documents(query) + except ValueError as ex: + found_path = False + logger.debug(f"error intercepted from COC Retriever =>{str(ex)}") + logger.debug(f"error intercepted from COC Retriever =>{str(ex)}", extra=MULTI_LINE_MESSAGE_TRUE) + return found_path, [] + + path_details = self.chain_of_calls_retriever.print_call_hierarchy(call_hierarchy_list) + log_message = f"\n Retriever found path={found_path}\n path size={len(call_hierarchy_list)}\n" \ + f"==============================================\nPrints Hierarchy call functions " \ + f"path\n==============================================\n" + + for node in path_details: + log_message = log_message + node + "\n" + + logger.debug(log_message) + logger.debug(log_message, extra=MULTI_LINE_MESSAGE_TRUE) + log_message_2 = "Path Contents Content: \n" + "==============================================\n" + content_of_files_in_path = f"{log_message_2}" + for i, function_method in enumerate(reversed(call_hierarchy_list)): + content_of_files_in_path = f"{content_of_files_in_path} File {i + 1}: {function_method.metadata['source']}\n" \ + f"-------------------------------------------\n{function_method.page_content}\n" + logger.debug(content_of_files_in_path) + logger.debug(content_of_files_in_path, extra=MULTI_LINE_MESSAGE_TRUE) + return found_path, call_hierarchy_list \ No newline at end of file diff --git a/src/vuln_analysis/configs/brew/external-user-profile.yml b/src/vuln_analysis/configs/brew/external-user-profile.yml new file mode 100644 index 000000000..3d0dd1984 --- /dev/null +++ b/src/vuln_analysis/configs/brew/external-user-profile.yml @@ -0,0 +1,23 @@ +# External User Profile — public Fedora Koji (no VPN / no auth) +# +# Assumptions: +# - Packages are resolved and downloaded from koji.fedoraproject.org +# - Build logs are not fetched (optional for internal; unavailable or unused for external v1) + +profile: + name: fedora-public + +hosts: + rpm: + brew_hub: https://koji.fedoraproject.org/kojihub + brew_download: https://kojipkgs.fedoraproject.org + +default_arch: x86_64 + +# Dev/local phase: false (matches internal profile). Set to true before cluster deploy. +ssl_verify: true + +build_log: + auto_fetch: false + +download_binary_rpm: false diff --git a/src/vuln_analysis/configs/brew/internal-user-profile.yml b/src/vuln_analysis/configs/brew/internal-user-profile.yml new file mode 100644 index 000000000..35df8e035 --- /dev/null +++ b/src/vuln_analysis/configs/brew/internal-user-profile.yml @@ -0,0 +1,26 @@ +# Internal User Profile — Red Hat VPN-connected environment +# +# Assumptions: +# - User is on the Red Hat VPN (can reach *.redhat.com internal hosts) +# - Build logs are available via Brew task output + +profile: + name: redhat-internal + +hosts: + rpm: + brew_hub: https://brewhub.engineering.redhat.com/brewhub + brew_download: https://download-01.beak-001.prod.iad2.dc.redhat.com/brewroot + git: + dist_git: https://pkgs.devel.redhat.com/cgit + +default_arch: x86_64 + +ssl_verify: true +# RH engineering hosts (brewhub, brew CDN) — use git-ca-bundle, not service-ca.crt +verify_path: /app/git-ca-bundle/ca-bundle.crt + +build_log: + auto_fetch: true + +download_binary_rpm: false diff --git a/src/vuln_analysis/configs/config-http-nim.yml b/src/vuln_analysis/configs/config-http-nim.yml new file mode 100644 index 000000000..23a87e85e --- /dev/null +++ b/src/vuln_analysis/configs/config-http-nim.yml @@ -0,0 +1,270 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +general: + use_uvloop: true + telemetry: + tracing: + phoenix: + _type: phoenix + endpoint: ${OTEL_TRACES_ENDPOINT:-http://localhost:6006/v1/traces} + project: cve_agent + +functions: + cve_clone_and_deps: + _type: cve_clone_and_deps + base_git_dir: .cache/am_cache/git + base_pickle_dir: .cache/am_cache/pickle + base_rpm_dir: .cache/am_cache/rpms + cve_segmentation: + _type: cve_segmentation + agent_name: cve_agent_executor + embedder_name: nim_embedder + base_git_dir: .cache/am_cache/git + base_vdb_dir: .cache/am_cache/vdb + base_code_index_dir: .cache/am_cache/code_index + base_pickle_dir: .cache/am_cache/pickle + ignore_code_embedding: true + cve_fetch_intel: + _type: cve_fetch_intel + rpm_user_type: ${RPM_USER_TYPE:-internal} + intel_plugin_config: + plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin + plugin_config: + source: Product Security research + endpoint: http://localhost:8080/api/v1/vulnerabilities/{vuln_id}/comments + cve_process_sbom: + _type: cve_process_sbom + cve_checklist: + _type: cve_checklist + llm_name: checklist_llm + Call Chain Analyzer: + _type: transitive_code_search + enable_transitive_search: true + Function Caller Finder: + _type: calling_function_name_extractor + enable_functions_usage_search: true + Function Locator: + _type: package_and_function_locator + Function Library Version Finder: + _type: calling_function_library_version_finder + Code Semantic Search: + _type: local_vdb_retriever + embedder_name: nim_embedder + llm_name: code_vdb_retriever_llm + vdb_type: code + return_source_documents: false + Docs Semantic Search: + _type: local_vdb_retriever + embedder_name: nim_embedder + llm_name: doc_vdb_retriever_llm + vdb_type: doc + return_source_documents: false + Code Keyword Search: + _type: lexical_code_search + top_k: 5 + CVE Web Search: + _type: serp_wrapper + max_retries: 5 + Container Analysis Data: + _type: container_image_analysis_data + Configuration Scanner: + _type: configuration_scanner + max_results: 15 + context_lines: 5 + Import Usage Analyzer: + _type: import_usage_analyzer + max_files: 20 + cve_agent_executor: + _type: cve_agent_executor + llm_name: cve_agent_executor_llm + tool_names: + - Code Semantic Search + - Docs Semantic Search + - Code Keyword Search + - CVE Web Search + - Call Chain Analyzer + - Function Caller Finder + - Function Locator + - Function Library Version Finder + - Configuration Scanner + - Import Usage Analyzer + max_concurrency: null + max_iterations: 10 + prompt_examples: false + replace_exceptions: true + replace_exceptions_value: "I do not have a definitive answer for this checklist item." + return_intermediate_steps: false +# transitive_search_tool_enabled: false + cve_web_search_enabled: true + uber_jar_file_threshold: 600 + verbose: false + cve_generate_cvss: + _type: cve_generate_cvss + skip: false + llm_name: generate_cvss_llm + tool_names: + - Container Image Code QA System + - Container Image Developer Guide QA System + - Lexical Search Container Image Code QA System # Uncomment to enable lexical search + - Container Image Analysis Data + max_concurrency: null + max_iterations: 10 + prompt_examples: true + replace_exceptions: false + replace_exceptions_value: "Failed to generate CVSS for this analysis." + return_intermediate_steps: false + verbose: false + cve_summarize: + _type: cve_summarize + llm_name: summarize_llm + cve_justify: + _type: cve_justify + llm_name: justify_llm + cve_generate_vex: + _type: cve_generate_vex + skip: false + vex_format: csaf + cve_http_output: + _type: cve_http_output + url: http://localhost:8080 + endpoint: /api/v1/reports + enable_mlops: ${ENABLE_MLOPS:-false} + mlops_config: + mlops_url: http://localhost:8080 + auth_type: "None" + token_path: "/var/run/secrets/kubernetes.io/serviceaccount/token" + verify_path: "/app/certs/service-ca.crt" + enable_verify: false + cve_calculate_intel_score: + _type: cve_calculate_intel_score + llm_name: intel_source_score_llm + generate_intel_score: true + intel_low_score: 51 + insist_analysis: false + cve_fetch_patches: + _type: cve_fetch_patches + +llms: + checklist_llm: + _type: nim + api_key: ${NVIDIA_API_KEY:-"EMPTY"} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CHECKLIST_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + code_vdb_retriever_llm: + _type: nim + api_key: ${NVIDIA_API_KEY:-"EMPTY"} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CODE_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + doc_vdb_retriever_llm: + _type: nim + api_key: ${NVIDIA_API_KEY:-"EMPTY"} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${DOC_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + cve_agent_executor_llm: + _type: nim + api_key: ${NVIDIA_API_KEY:-"EMPTY"} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CVE_AGENT_EXECUTOR_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + generate_cvss_llm: + _type: nim + api_key: ${NVIDIA_API_KEY:-"EMPTY"} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + summarize_llm: + _type: nim + api_key: ${NVIDIA_API_KEY:-"EMPTY"} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${SUMMARIZE_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + justify_llm: + _type: nim + api_key: ${NVIDIA_API_KEY:-"EMPTY"} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + intel_source_score_llm: + _type: nim + api_key: ${NVIDIA_API_KEY:-"EMPTY"} + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + +embedders: + nim_embedder: + _type: nim + base_url: ${NIM_EMBED_BASE_URL:-https://integrate.api.nvidia.com/v1} + model_name: ${EMBEDDER_MODEL_NAME:-nvidia/nv-embedqa-e5-v5} + truncate: END + max_batch_size: 128 + +workflow: + _type: cve_agent + cve_clone_and_deps_name: cve_clone_and_deps + cve_segmentation_name: cve_segmentation + cve_fetch_intel_name: cve_fetch_intel + cve_calculate_intel_score_name: cve_calculate_intel_score + cve_process_sbom_name: cve_process_sbom + cve_checklist_name: cve_checklist + cve_agent_executor_name: cve_agent_executor + cve_generate_cvss_name: cve_generate_cvss + cve_generate_vex_name: cve_generate_vex + cve_summarize_name: cve_summarize + cve_justify_name: cve_justify + cve_output_config_name: cve_http_output + cve_fetch_patches_name: cve_fetch_patches + +eval: + general: + output_dir: ./.tmp/eval/cve_agent + dataset: + _type: json + file_path: data/eval_datasets/eval_dataset.json + + profiler: + token_uniqueness_forecast: true + workflow_runtime_forecast: true + compute_llm_metrics: true + csv_exclude_io_text: true + prompt_caching_prefixes: + enable: true + min_frequency: 0.1 + bottleneck_analysis: + # Can also be simple_stack + enable_nested_stack: true + concurrency_spike_analysis: + enable: true + spike_threshold: 7 diff --git a/src/vuln_analysis/configs/config-http-openai.yml b/src/vuln_analysis/configs/config-http-openai.yml new file mode 100644 index 000000000..08a5b20e3 --- /dev/null +++ b/src/vuln_analysis/configs/config-http-openai.yml @@ -0,0 +1,325 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +general: + front_end: + _type: fastapi + endpoints: + - path: /health + method: GET + description: Perform a health check. + function_name: health_check + use_uvloop: true + telemetry: + tracing: + phoenix: + _type: phoenix + endpoint: ${OTEL_TRACES_ENDPOINT:-http://localhost:6006/v1/traces} + project: cve_agent + +functions: + cve_clone_and_deps: + _type: cve_clone_and_deps + base_git_dir: .cache/am_cache/git + base_pickle_dir: .cache/am_cache/pickle + base_rpm_dir: .cache/am_cache/rpms + cve_segmentation: + _type: cve_segmentation + agent_name: cve_agent_executor + embedder_name: nim_embedder + base_git_dir: .cache/am_cache/git + base_vdb_dir: .cache/am_cache/vdb + base_code_index_dir: .cache/am_cache/code_index + base_pickle_dir: .cache/am_cache/pickle + ignore_code_embedding: true + cve_fetch_intel: + _type: cve_fetch_intel + rpm_user_type: ${RPM_USER_TYPE:-internal} + intel_plugin_config: + plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin + plugin_config: + source: Product Security research + endpoint: http://localhost:8080/api/v1/vulnerabilities/{vuln_id}/comments + cve_process_sbom: + _type: cve_process_sbom + cve_verify_vuln_package: + _type: cve_verify_vuln_package + skip: false + base_git_dir: .cache/am_cache/git + llm_name: checklist_llm + cve_checklist: + _type: cve_checklist + llm_name: checklist_llm + Call Chain Analyzer: + _type: transitive_code_search + enable_transitive_search: true + Function Caller Finder: + _type: calling_function_name_extractor + enable_functions_usage_search: true + Function Locator: + _type: package_and_function_locator + Function Library Version Finder: + _type: calling_function_library_version_finder + Code Semantic Search: + _type: local_vdb_retriever + embedder_name: nim_embedder + llm_name: code_vdb_retriever_llm + vdb_type: code + return_source_documents: false + Docs Semantic Search: + _type: local_vdb_retriever + embedder_name: nim_embedder + llm_name: doc_vdb_retriever_llm + vdb_type: doc + return_source_documents: false + Code Keyword Search: + _type: lexical_code_search + top_k: 5 + Source Grep: + _type: source_grep + base_checker_dir: .cache/am_cache/checker + max_results: 50 + context_lines: 2 + CVE Web Search: + _type: serp_wrapper + max_retries: 5 + Container Analysis Data: + _type: container_image_analysis_data + Configuration Scanner: + _type: configuration_scanner + max_results: 15 + context_lines: 5 + Import Usage Analyzer: + _type: import_usage_analyzer + max_files: 20 + cve_agent_executor: + _type: cve_agent_executor + llm_name: cve_agent_executor_llm + tool_names: + - Code Semantic Search + - Docs Semantic Search + - Code Keyword Search + - CVE Web Search + - Call Chain Analyzer + - Function Caller Finder + - Function Locator + - Function Library Version Finder + - Configuration Scanner + - Import Usage Analyzer + max_concurrency: null + max_iterations: 10 + prompt_examples: false + replace_exceptions: true + replace_exceptions_value: "I do not have a definitive answer for this checklist item." + return_intermediate_steps: false +# transitive_search_tool_enabled: false + cve_web_search_enabled: true + uber_jar_file_threshold: 600 + verbose: false + cve_generate_cvss: + _type: cve_generate_cvss + skip: true + llm_name: generate_cvss_llm + tool_names: + - Code Semantic Search + - Docs Semantic Search + - Code Keyword Search + - Container Analysis Data + max_concurrency: null + max_iterations: 10 + prompt_examples: true + replace_exceptions: false + replace_exceptions_value: "Failed to generate CVSS for this analysis." + return_intermediate_steps: false + verbose: false + cve_summarize: + _type: cve_summarize + llm_name: summarize_llm + cve_justify: + _type: cve_justify + llm_name: justify_llm + cve_generate_vex: + _type: cve_generate_vex + skip: false + cve_http_output: + _type: cve_http_output + url: http://localhost:8080 + endpoint: /api/v1/reports + enable_mlops: ${ENABLE_MLOPS:-false} + mlops_config: + mlops_url: ${MLOPS_URL:-http://localhost:8080} + auth_type: ${MLOPS_AUTH_TYPE:-None} + token_path: ${MLOPS_TOKEN_PATH:-/var/run/secrets/kubernetes.io/serviceaccount/token} + verify_path: ${MLOPS_VERIFY_PATH:-/app/certs/service-ca.crt} + enable_verify: ${MLOPS_ENABLE_VERIFY:-false} + cve_calculate_intel_score: + _type: cve_calculate_intel_score + llm_name: intel_source_score_llm + generate_intel_score: true + intel_low_score: 51 + insist_analysis: false + cve_source_acquisition: + _type: cve_source_acquisition + base_git_dir: .cache/am_cache/git + base_pickle_dir: .cache/am_cache/pickle + base_rpm_dir: .cache/am_cache/rpms + rpm_user_type: ${RPM_USER_TYPE:-internal} + cve_checker_segmentation: + _type: cve_checker_segmentation + base_checker_dir: .cache/am_cache/checker + base_code_index_dir: .cache/am_cache/code_index + cve_package_code_agent: + _type: cve_package_code_agent + llm_name: cve_agent_executor_llm + base_checker_dir: .cache/am_cache/checker + base_code_index_dir: .cache/am_cache/code_index + rpm_user_type: ${RPM_USER_TYPE:-internal} + tool_names: + - Source Grep + - Code Keyword Search + cve_checker_report: + _type: cve_checker_report + llm_name: cve_agent_executor_llm + base_checker_dir: .cache/am_cache/checker + cve_build_agent: + _type: cve_build_agent + llm_name: cve_agent_executor_llm + base_checker_dir: .cache/am_cache/checker + max_iterations: 10 + tool_names: + - Source Grep + - Code Keyword Search + cve_fetch_patches: + _type: cve_fetch_patches + health_check: + _type: health_check + +llms: + checklist_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CHECKLIST_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + code_vdb_retriever_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CODE_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + doc_vdb_retriever_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${DOC_VDB_RETRIEVER_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + cve_agent_executor_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${CVE_AGENT_EXECUTOR_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 2000 + top_p: 0.01 + generate_cvss_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + summarize_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${SUMMARIZE_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + justify_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + intel_source_score_llm: + _type: openai + api_key: "EMPTY" + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 + +embedders: + nim_embedder: + _type: nim + base_url: ${NIM_EMBED_BASE_URL:-https://integrate.api.nvidia.com/v1} + model_name: ${EMBEDDER_MODEL_NAME:-nvidia/nv-embedqa-e5-v5} + truncate: END + max_batch_size: 128 + +workflow: + _type: cve_agent + cve_clone_and_deps_name: cve_clone_and_deps + cve_segmentation_name: cve_segmentation + cve_fetch_intel_name: cve_fetch_intel + cve_calculate_intel_score_name: cve_calculate_intel_score + cve_process_sbom_name: cve_process_sbom + cve_verify_vuln_package_name: cve_verify_vuln_package + cve_checklist_name: cve_checklist + cve_agent_executor_name: cve_agent_executor + cve_generate_cvss_name: cve_generate_cvss + cve_generate_vex_name: cve_generate_vex + cve_summarize_name: cve_summarize + cve_justify_name: cve_justify + cve_output_config_name: cve_http_output + cve_source_acquisition_name: cve_source_acquisition + cve_checker_segmentation_name: cve_checker_segmentation + cve_package_code_agent_name: cve_package_code_agent + cve_checker_report_name: cve_checker_report + cve_build_agent_name: cve_build_agent + cve_fetch_patches_name: cve_fetch_patches + +eval: + general: + output_dir: ./.tmp/eval/cve_agent + dataset: + _type: json + file_path: data/eval_datasets/eval_dataset.json + + profiler: + token_uniqueness_forecast: true + workflow_runtime_forecast: true + compute_llm_metrics: true + csv_exclude_io_text: true + prompt_caching_prefixes: + enable: true + min_frequency: 0.1 + bottleneck_analysis: + # Can also be simple_stack + enable_nested_stack: true + concurrency_spike_analysis: + enable: true + spike_threshold: 7 diff --git a/src/vuln_analysis/configs/config-tracing.yml b/src/vuln_analysis/configs/config-tracing.yml index 85be3db1f..c464451f2 100644 --- a/src/vuln_analysis/configs/config-tracing.yml +++ b/src/vuln_analysis/configs/config-tracing.yml @@ -14,6 +14,13 @@ # limitations under the License. general: + front_end: + _type: fastapi + endpoints: + - path: /health + method: GET + description: Perform a health check. + function_name: health_check use_uvloop: true telemetry: logging: @@ -23,58 +30,112 @@ general: tracing: phoenix: _type: phoenix - endpoint: http://localhost:6006/v1/traces + endpoint: ${OTEL_TRACES_ENDPOINT:-http://localhost:6006/v1/traces} project: cve_agent functions: - cve_generate_vdbs: - _type: cve_generate_vdbs - agent_name: cve_agent_executor # Used to determine which tools are enabled + cve_clone_and_deps: + _type: cve_clone_and_deps + base_git_dir: .cache/am_cache/git + base_pickle_dir: .cache/am_cache/pickle + base_rpm_dir: .cache/am_cache/rpms + cve_segmentation: + _type: cve_segmentation + agent_name: cve_agent_executor embedder_name: nim_embedder base_git_dir: .cache/am_cache/git base_vdb_dir: .cache/am_cache/vdb base_code_index_dir: .cache/am_cache/code_index + base_pickle_dir: .cache/am_cache/pickle + ignore_code_embedding: true cve_fetch_intel: _type: cve_fetch_intel + rpm_user_type: ${RPM_USER_TYPE:-internal} + intel_plugin_config: + plugin_name: vuln_analysis.data_models.plugins.intel_plugin.SimpleHttpIntelPlugin + plugin_config: + source: Product Security research + endpoint: http://localhost:8080/api/v1/vulnerabilities/{vuln_id}/comments cve_process_sbom: _type: cve_process_sbom - cve_check_vuln_deps : - _type: cve_check_vuln_deps cve_checklist: _type: cve_checklist llm_name: checklist_llm - Container Image Code QA System: + Call Chain Analyzer: + _type: transitive_code_search + enable_transitive_search: true + Function Caller Finder: + _type: calling_function_name_extractor + enable_functions_usage_search: true + Function Locator: + _type: package_and_function_locator + Function Library Version Finder: + _type: calling_function_library_version_finder + Code Semantic Search: _type: local_vdb_retriever embedder_name: nim_embedder llm_name: code_vdb_retriever_llm vdb_type: code return_source_documents: false - Container Image Developer Guide QA System: + Docs Semantic Search: _type: local_vdb_retriever embedder_name: nim_embedder llm_name: doc_vdb_retriever_llm vdb_type: doc return_source_documents: false - Lexical Search Container Image Code QA System: + Code Keyword Search: _type: lexical_code_search top_k: 5 - Internet Search: + CVE Web Search: _type: serp_wrapper max_retries: 5 + Container Analysis Data: + _type: container_image_analysis_data + Configuration Scanner: + _type: configuration_scanner + max_results: 15 + context_lines: 5 + Import Usage Analyzer: + _type: import_usage_analyzer + max_files: 20 cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm tool_names: - - Container Image Code QA System - - Container Image Developer Guide QA System - # - Lexical Search Container Image Code QA System # Uncomment to enable lexical search - - Internet Search + - Code Semantic Search + - Docs Semantic Search + - Code Keyword Search # Uncomment to enable keyword search + - CVE Web Search + - Call Chain Analyzer + - Function Caller Finder + - Function Locator + - Function Library Version Finder + - Configuration Scanner + - Import Usage Analyzer max_concurrency: null max_iterations: 10 prompt_examples: false replace_exceptions: true replace_exceptions_value: "I do not have a definitive answer for this checklist item." return_intermediate_steps: false + cve_web_search_enabled: true + uber_jar_file_threshold: 600 + verbose: false + cve_generate_cvss: + _type: cve_generate_cvss + skip: true + llm_name: generate_cvss_llm + tool_names: + - Code Semantic Search + - Docs Semantic Search + - Code Keyword Search + - Container Analysis Data + max_concurrency: null + max_iterations: 10 + prompt_examples: true + replace_exceptions: false + replace_exceptions_value: "Failed to generate CVSS for this analysis." + return_intermediate_steps: false verbose: false cve_summarize: _type: cve_summarize @@ -82,11 +143,24 @@ functions: cve_justify: _type: cve_justify llm_name: justify_llm + cve_generate_vex: + _type: cve_generate_vex + skip: false cve_file_output: _type: cve_file_output file_path: .tmp/output.json markdown_dir: .tmp/vulnerability_markdown_reports overwrite: true + cve_calculate_intel_score: + _type: cve_calculate_intel_score + llm_name: intel_source_score_llm + generate_intel_score: true + intel_low_score: 51 + insist_analysis: false + cve_fetch_patches: + _type: cve_fetch_patches + health_check: + _type: health_check llms: checklist_llm: @@ -117,6 +191,13 @@ llms: temperature: 0.0 max_tokens: 2000 top_p: 0.01 + generate_cvss_llm: + _type: nim + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 summarize_llm: _type: nim base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} @@ -131,6 +212,13 @@ llms: temperature: 0.0 max_tokens: 1024 top_p: 0.01 + intel_source_score_llm: + _type: nim + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 embedders: nim_embedder: @@ -142,15 +230,19 @@ embedders: workflow: _type: cve_agent - cve_generate_vdbs_name: cve_generate_vdbs + cve_clone_and_deps_name: cve_clone_and_deps + cve_segmentation_name: cve_segmentation cve_fetch_intel_name: cve_fetch_intel + cve_calculate_intel_score_name: cve_calculate_intel_score cve_process_sbom_name: cve_process_sbom - cve_check_vuln_deps_name: cve_check_vuln_deps cve_checklist_name: cve_checklist cve_agent_executor_name: cve_agent_executor + cve_generate_cvss_name: cve_generate_cvss + cve_generate_vex_name: cve_generate_vex cve_summarize_name: cve_summarize cve_justify_name: cve_justify cve_output_config_name: cve_file_output + cve_fetch_patches_name: cve_fetch_patches eval: general: diff --git a/src/vuln_analysis/configs/config.yml b/src/vuln_analysis/configs/config.yml index 08503cdec..f93258a84 100644 --- a/src/vuln_analysis/configs/config.yml +++ b/src/vuln_analysis/configs/config.yml @@ -15,56 +15,104 @@ general: use_uvloop: true + telemetry: + tracing: + phoenix: + _type: phoenix + endpoint: ${OTEL_TRACES_ENDPOINT:-http://localhost:6006/v1/traces} + project: cve_agent functions: - cve_generate_vdbs: - _type: cve_generate_vdbs - agent_name: cve_agent_executor # Used to determine which tools are enabled + cve_clone_and_deps: + _type: cve_clone_and_deps + base_git_dir: .cache/am_cache/git + base_pickle_dir: .cache/am_cache/pickle + base_rpm_dir: .cache/am_cache/rpms + cve_segmentation: + _type: cve_segmentation + agent_name: cve_agent_executor embedder_name: nim_embedder base_git_dir: .cache/am_cache/git base_vdb_dir: .cache/am_cache/vdb base_code_index_dir: .cache/am_cache/code_index + base_pickle_dir: .cache/am_cache/pickle cve_fetch_intel: _type: cve_fetch_intel + rpm_user_type: ${RPM_USER_TYPE:-internal} cve_process_sbom: _type: cve_process_sbom - cve_check_vuln_deps : - _type: cve_check_vuln_deps + cve_verify_vuln_package: + _type: cve_verify_vuln_package + skip: false + base_git_dir: .cache/am_cache/git + llm_name: checklist_llm cve_checklist: _type: cve_checklist llm_name: checklist_llm - Container Image Code QA System: + Code Semantic Search: _type: local_vdb_retriever embedder_name: nim_embedder llm_name: code_vdb_retriever_llm vdb_type: code return_source_documents: false - Container Image Developer Guide QA System: + Docs Semantic Search: _type: local_vdb_retriever embedder_name: nim_embedder llm_name: doc_vdb_retriever_llm vdb_type: doc return_source_documents: false - Lexical Search Container Image Code QA System: + Code Keyword Search: _type: lexical_code_search top_k: 5 - Internet Search: + CVE Web Search: _type: serp_wrapper max_retries: 5 + Container Analysis Data: + _type: container_image_analysis_data + Function Library Version Finder: + _type: calling_function_library_version_finder + Configuration Scanner: + _type: configuration_scanner + max_results: 15 + context_lines: 5 + Import Usage Analyzer: + _type: import_usage_analyzer + max_files: 20 cve_agent_executor: _type: cve_agent_executor llm_name: cve_agent_executor_llm tool_names: - - Container Image Code QA System - - Container Image Developer Guide QA System - # - Lexical Search Container Image Code QA System # Uncomment to enable lexical search - - Internet Search + - Code Semantic Search + - Docs Semantic Search + # - Code Keyword Search # Uncomment to enable keyword search + - CVE Web Search + - Function Library Version Finder + - Configuration Scanner + - Import Usage Analyzer max_concurrency: null max_iterations: 10 prompt_examples: false replace_exceptions: true replace_exceptions_value: "I do not have a definitive answer for this checklist item." return_intermediate_steps: false + cve_web_search_enabled: true + uber_jar_file_threshold: 600 + verbose: false + cve_generate_cvss: + _type: cve_generate_cvss + skip: true + llm_name: generate_cvss_llm + tool_names: + - Code Semantic Search + - Docs Semantic Search + - Code Keyword Search # Uncomment to enable keyword search + - Container Analysis Data + max_concurrency: null + max_iterations: 10 + prompt_examples: true + replace_exceptions: false + replace_exceptions_value: "Failed to generate CVSS for this analysis." + return_intermediate_steps: false verbose: false cve_summarize: _type: cve_summarize @@ -72,11 +120,23 @@ functions: cve_justify: _type: cve_justify llm_name: justify_llm + cve_generate_vex: + _type: cve_generate_vex + skip: false + vex_format: csaf cve_file_output: _type: cve_file_output file_path: .tmp/output.json markdown_dir: .tmp/vulnerability_markdown_reports overwrite: true + cve_calculate_intel_score: + _type: cve_calculate_intel_score + llm_name: intel_source_score_llm + generate_intel_score: true + intel_low_score: 51 + insist_analysis: false + cve_fetch_patches: + _type: cve_fetch_patches llms: checklist_llm: @@ -105,7 +165,14 @@ llms: base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} model_name: ${CVE_AGENT_EXECUTOR_MODEL_NAME:-meta/llama-3.1-70b-instruct} temperature: 0.0 - max_tokens: 2000 + max_tokens: 1000 + top_p: 0.01 + generate_cvss_llm: + _type: nim + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${GENERATE_CVSS_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 top_p: 0.01 summarize_llm: _type: nim @@ -121,6 +188,13 @@ llms: temperature: 0.0 max_tokens: 1024 top_p: 0.01 + intel_source_score_llm: + _type: nim + base_url: ${NVIDIA_API_BASE:-https://integrate.api.nvidia.com/v1} + model_name: ${JUSTIFY_MODEL_NAME:-meta/llama-3.1-70b-instruct} + temperature: 0.0 + max_tokens: 1024 + top_p: 0.01 embedders: nim_embedder: @@ -132,15 +206,20 @@ embedders: workflow: _type: cve_agent - cve_generate_vdbs_name: cve_generate_vdbs + cve_clone_and_deps_name: cve_clone_and_deps + cve_segmentation_name: cve_segmentation cve_fetch_intel_name: cve_fetch_intel + cve_calculate_intel_score_name: cve_calculate_intel_score cve_process_sbom_name: cve_process_sbom - cve_check_vuln_deps_name: cve_check_vuln_deps + cve_verify_vuln_package_name: cve_verify_vuln_package cve_checklist_name: cve_checklist cve_agent_executor_name: cve_agent_executor + cve_generate_cvss_name: cve_generate_cvss + cve_generate_vex_name: cve_generate_vex cve_summarize_name: cve_summarize cve_justify_name: cve_justify cve_output_config_name: cve_file_output + cve_fetch_patches_name: cve_fetch_patches eval: general: diff --git a/src/vuln_analysis/configs/openapi/openapi.json b/src/vuln_analysis/configs/openapi/openapi.json index bead3625d..e0cdb1ad2 100644 --- a/src/vuln_analysis/configs/openapi/openapi.json +++ b/src/vuln_analysis/configs/openapi/openapi.json @@ -1,2921 +1,4602 @@ { "openapi": "3.1.0", "info": { - "title": "FastAPI", - "version": "0.1.0" + "title": "FastAPI", + "version": "0.1.0" }, "paths": { - "/generate": { - "post": { - "summary": "Post Single", - "description": "Executes the default AIQ Toolkit workflow from the loaded configuration", - "operationId": "post_single_generate_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AgentMorpheusInput-Input" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AgentMorpheusOutput" - } - } - } - }, - "500": { - "description": "Internal Server Error", - "content": { - "application/json": { - "example": { - "detail": "Internal server error occurred" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } + "/generate": { + "post": { + "summary": "Post Single", + "description": "Executes the default NAT workflow from the loaded configuration", + "operationId": "post_single_generate_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentMorpheusInput-Input" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentMorpheusOutput" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } } - }, - "/generate/stream": { - "post": { - "summary": "Post Stream", - "description": "Executes the default AIQ Toolkit workflow from the loaded configuration", - "operationId": "post_stream_generate_stream_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AgentMorpheusInput-Input" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AgentMorpheusOutput" - } - } - } - }, - "500": { - "description": "Internal Server Error", - "content": { - "application/json": { - "example": { - "detail": "Internal server error occurred" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } + } + } + }, + "/generate/stream": { + "post": { + "summary": "Post Stream", + "description": "Executes the default NAT workflow from the loaded configuration", + "operationId": "post_stream_generate_stream_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentMorpheusInput-Input" } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentMorpheusOutput" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } } - }, - "/generate/full": { - "post": { - "summary": "Post Stream", - "description": "Stream raw intermediate steps without any step adaptor translations.\nUse filter_steps query parameter to filter steps by type (comma-separated list) or set to 'none' to suppress all intermediate steps.", - "operationId": "post_stream_generate_full_post", - "parameters": [ - { - "name": "filter_steps", - "in": "query", - "required": false, - "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Filter Steps" - } - } + } + } + }, + "/generate/full": { + "post": { + "summary": "Post Stream", + "description": "Stream raw intermediate steps without any step adaptor translations.\nUse filter_steps query parameter to filter steps by type (comma-separated list) or set to 'none' to suppress all intermediate steps.", + "operationId": "post_stream_generate_full_post", + "parameters": [ + { + "name": "filter_steps", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AgentMorpheusInput-Input" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AgentMorpheusOutput" - } - } - } - }, - "500": { - "description": "Internal Server Error", - "content": { - "application/json": { - "example": { - "detail": "Internal server error occurred" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } + "title": "Filter Steps" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentMorpheusInput-Input" } + } } - }, - "/chat": { - "post": { - "summary": "Post Single", - "description": "Executes the default AIQ Toolkit workflow from the loaded configuration", - "operationId": "post_single_chat_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AIQChatRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AIQChatResponse" - } - } - } - }, - "500": { - "description": "Internal Server Error", - "content": { - "application/json": { - "example": { - "detail": "Internal server error occurred" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentMorpheusOutput" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } } + } } - }, - "/chat/stream": { - "post": { - "summary": "Post Stream", - "description": "Executes the default AIQ Toolkit workflow from the loaded configuration", - "operationId": "post_stream_chat_stream_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AIQChatRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "$ref": "#/components/schemas/AIQChatResponseChunk" - }, - { - "$ref": "#/components/schemas/AIQResponseIntermediateStep" - } - ], - "title": "Response Post Stream Chat Stream Post" - } - } - } - }, - "500": { - "description": "Internal Server Error", - "content": { - "application/json": { - "example": { - "detail": "Internal server error occurred" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } + } + } + }, + "/generate/async": { + "post": { + "summary": "Start Async Generation", + "description": "Start an async generate job", + "operationId": "start_async_generation_generate_async_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AsyncGenerateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/AsyncGenerateResponse" + }, + { + "$ref": "#/components/schemas/AsyncGenerationStatusResponse" + } + ], + "title": "Response Start Async Generation Generate Async Post" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } } + } } - }, - "/evaluate/job/last": { - "get": { - "summary": "Get Last Job Status", - "description": "Get the status of the last created evaluation job", - "operationId": "get_last_job_status_evaluate_job_last_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AIQEvaluateStatusResponse" - } - } - } - }, - "404": { - "description": "No jobs found" - }, - "500": { - "description": "Internal Server Error", - "content": { - "application/json": { - "example": { - "detail": "Internal server error occurred" - } - } - } - } + } + } + }, + "/generate/async/job/{job_id}": { + "get": { + "summary": "Get Async Job Status", + "description": "Get the status of an async job", + "operationId": "get_async_job_status_generate_async_job__job_id__get", + "parameters": [ + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AsyncGenerationStatusResponse" + } + } + } + }, + "404": { + "description": "Job not found" + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } } + } } - }, - "/evaluate/job/{job_id}": { - "get": { - "summary": "Get Job Status", - "description": "Get the status of an evaluation job", - "operationId": "get_job_status_evaluate_job__job_id__get", - "parameters": [ - { - "name": "job_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "title": "Job Id" - } - } + } + } + }, + "/chat": { + "post": { + "summary": "Post Single", + "description": "Executes the default NAT workflow from the loaded configuration", + "operationId": "post_single_chat_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChatRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChatResponse" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/chat/stream": { + "post": { + "summary": "Post Stream", + "description": "Executes the default NAT workflow from the loaded configuration", + "operationId": "post_stream_chat_stream_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChatRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/ChatResponseChunk" + }, + { + "$ref": "#/components/schemas/ResponseIntermediateStep" + } + ], + "title": "Response Post Stream Chat Stream Post" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/chat/completions": { + "post": { + "summary": "Post Openai Api Compatible", + "description": "Executes the default NAT workflow from the loaded configuration (OpenAI Chat Completions API compatible)", + "operationId": "post_openai_api_compatible_v1_chat_completions_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChatRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/ChatResponse" + }, + { + "$ref": "#/components/schemas/ChatResponseChunk" + } + ], + "title": "Response Post Openai Api Compatible V1 Chat Completions Post" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/evaluate/job/last": { + "get": { + "summary": "Get Last Job Status", + "description": "Get the status of the last created evaluation job", + "operationId": "get_last_job_status_evaluate_job_last_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EvaluateStatusResponse" + } + } + } + }, + "404": { + "description": "No jobs found" + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + } + } + } + }, + "/evaluate/job/{job_id}": { + "get": { + "summary": "Get Job Status", + "description": "Get the status of an evaluation job", + "operationId": "get_job_status_evaluate_job__job_id__get", + "parameters": [ + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EvaluateStatusResponse" + } + } + } + }, + "404": { + "description": "Job not found" + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/evaluate/jobs": { + "get": { + "summary": "Get Jobs", + "description": "Get all jobs, optionally filtered by status", + "operationId": "get_jobs_evaluate_jobs_get", + "parameters": [ + { + "name": "status", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AIQEvaluateStatusResponse" - } - } - } - }, - "404": { - "description": "Job not found" - }, - "500": { - "description": "Internal Server Error", - "content": { - "application/json": { - "example": { - "detail": "Internal server error occurred" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } + "title": "Status" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EvaluateStatusResponse" + }, + "title": "Response Get Jobs Evaluate Jobs Get" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } } + } } - }, - "/evaluate/jobs": { - "get": { - "summary": "Get Jobs", - "description": "Get all jobs, optionally filtered by status", - "operationId": "get_jobs_evaluate_jobs_get", - "parameters": [ - { - "name": "status", - "in": "query", - "required": false, - "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Status" - } - } + } + } + }, + "/evaluate": { + "post": { + "summary": "Start Evaluation", + "description": "Evaluates the performance and accuracy of the workflow on a dataset", + "operationId": "start_evaluation_evaluate_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EvaluateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EvaluateResponse" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/auth/redirect": { + "get": { + "summary": "Redirect Uri", + "description": "Handles the authorization code and state returned from the Authorization Code Grant Flow.", + "operationId": "redirect_uri_auth_redirect_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + + } + } + } + } + } + } + }, + "/health": { + "get": { + "summary": "Get Single", + "description": "Perform a health check.", + "operationId": "get_single_health_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HealthStatusResponse" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + } + } + } + }, + "/health/stream": { + "get": { + "summary": "Get Stream", + "description": "Perform a health check.", + "operationId": "get_stream_health_stream_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HealthStatusResponse" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + } + } + } + }, + "/health/full": { + "get": { + "summary": "Get Stream", + "description": "Stream raw intermediate steps without any step adaptor translations.\nUse filter_steps query parameter to filter steps by type (comma-separated list) or set to 'none' to suppress all intermediate steps.", + "operationId": "get_stream_health_full_get", + "parameters": [ + { + "name": "filter_steps", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/AIQEvaluateStatusResponse" - }, - "title": "Response Get Jobs Evaluate Jobs Get" - } - } - } - }, - "500": { - "description": "Internal Server Error", - "content": { - "application/json": { - "example": { - "detail": "Internal server error occurred" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } + "title": "Filter Steps" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } } + } } - }, - "/evaluate": { - "post": { - "summary": "Start Evaluation", - "description": "Evaluates the performance and accuracy of the workflow on a dataset", - "operationId": "start_evaluation_evaluate_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AIQEvaluateRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AIQEvaluateResponse" - } - } - } - }, - "500": { - "description": "Internal Server Error", - "content": { - "application/json": { - "example": { - "detail": "Internal server error occurred" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } + } + } + }, + "/health/async/job/{job_id}": { + "get": { + "summary": "Get Async Job Status", + "description": "Get the status of an async job", + "operationId": "get_async_job_status_health_async_job__job_id__get", + "parameters": [ + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AsyncGenerationStatusResponse" + } + } + } + }, + "404": { + "description": "Job not found" + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "example": { + "detail": "Internal server error occurred" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } } + } } + } } + } }, "components": { - "schemas": { - "AIQChatRequest": { - "properties": { - "messages": { - "items": { - "$ref": "#/components/schemas/Message" - }, - "type": "array", - "title": "Messages" - }, - "model": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Model" - }, - "temperature": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "title": "Temperature" - }, - "max_tokens": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "title": "Max Tokens" - }, - "top_p": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "title": "Top P" - } + "schemas": { + "AcquiredArtifacts": { + "properties": { + "srpm_path": { + "anyOf": [ + { + "type": "string", + "format": "path" }, - "additionalProperties": true, - "type": "object", - "required": [ - "messages" - ], - "title": "AIQChatRequest", - "description": "AIQChatRequest is a data model that represents a request to the AIQ Toolkit chat API." - }, - "AIQChatResponse": { - "properties": { - "id": { - "type": "string", - "title": "Id" - }, - "object": { - "type": "string", - "title": "Object" - }, - "model": { - "type": "string", - "title": "Model", - "default": "" - }, - "created": { - "type": "string", - "format": "date-time", - "title": "Created" - }, - "choices": { - "items": { - "$ref": "#/components/schemas/AIQChoice" - }, - "type": "array", - "title": "Choices" - }, - "usage": { - "anyOf": [ - { - "$ref": "#/components/schemas/AIQUsage" - }, - { - "type": "null" - } - ] - } + { + "type": "null" + } + ], + "title": "Srpm Path" + }, + "source_dir": { + "anyOf": [ + { + "type": "string", + "format": "path" }, - "additionalProperties": true, - "type": "object", - "required": [ - "id", - "object", - "created", - "choices" - ], - "title": "AIQChatResponse", - "description": "AIQChatResponse is a data model that represents a response from the AIQ Toolkit chat API." - }, - "AIQChatResponseChunk": { - "properties": { - "id": { - "type": "string", - "title": "Id" - }, - "choices": { - "items": { - "$ref": "#/components/schemas/AIQChoice" - }, - "type": "array", - "title": "Choices" - }, - "created": { - "type": "string", - "format": "date-time", - "title": "Created" - }, - "model": { - "type": "string", - "title": "Model", - "default": "" - }, - "object": { - "type": "string", - "title": "Object", - "default": "chat.completion.chunk" - } + { + "type": "null" + } + ], + "title": "Source Dir" + }, + "build_log_path": { + "anyOf": [ + { + "type": "string", + "format": "path" }, - "additionalProperties": true, - "type": "object", - "required": [ - "id", - "choices", - "created" - ], - "title": "AIQChatResponseChunk", - "description": "AIQChatResponseChunk is a data model that represents a response chunk from the AIQ Toolkit chat streaming API." + { + "type": "null" + } + ], + "title": "Build Log Path" }, - "AIQChoice": { - "properties": { - "message": { - "$ref": "#/components/schemas/AIQChoiceMessage" - }, - "finish_reason": { - "anyOf": [ - { - "type": "string", - "enum": [ - "stop", - "length", - "tool_calls", - "content_filter", - "function_call" - ] - }, - { - "type": "null" - } - ], - "title": "Finish Reason" - }, - "index": { - "type": "integer", - "title": "Index" - } + "binary_rpm_path": { + "anyOf": [ + { + "type": "string", + "format": "path" }, - "additionalProperties": true, - "type": "object", - "required": [ - "message", - "index" - ], - "title": "AIQChoice" - }, - "AIQChoiceMessage": { - "properties": { - "content": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Content" - }, - "role": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Role" - } + { + "type": "null" + } + ], + "title": "Binary Rpm Path" + }, + "patch_source_dir": { + "anyOf": [ + { + "type": "string", + "format": "path" }, - "type": "object", - "title": "AIQChoiceMessage" + { + "type": "null" + } + ], + "title": "Patch Source Dir" }, - "AIQEvaluateRequest": { - "properties": { - "config_file": { - "type": "string", - "title": "Config File", - "description": "Path to the configuration file for evaluation" - }, - "job_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Job Id", - "description": "Unique identifier for the evaluation job" - }, - "reps": { - "type": "integer", - "title": "Reps", - "description": "Number of repetitions for the evaluation, defaults to 1", - "default": 1 - }, - "expiry_seconds": { - "type": "integer", - "title": "Expiry Seconds", - "description": "Optional time (in seconds) before the job expires. Clamped between 600 (10 min) and 86400 (24h).", - "default": 3600 - } + "patch_diff_path": { + "anyOf": [ + { + "type": "string", + "format": "path" }, - "type": "object", - "required": [ - "config_file" - ], - "title": "AIQEvaluateRequest", - "description": "Request model for the evaluate endpoint." - }, - "AIQEvaluateResponse": { - "properties": { - "job_id": { - "type": "string", - "title": "Job Id", - "description": "Unique identifier for the evaluation job" - }, - "status": { - "type": "string", - "title": "Status", - "description": "Current status of the evaluation job" - } + { + "type": "null" + } + ], + "title": "Patch Diff Path" + }, + "source_url": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "job_id", - "status" - ], - "title": "AIQEvaluateResponse", - "description": "Response model for the evaluate endpoint." - }, - "AIQEvaluateStatusResponse": { - "properties": { - "job_id": { - "type": "string", - "title": "Job Id", - "description": "Unique identifier for the evaluation job" - }, - "status": { - "type": "string", - "title": "Status", - "description": "Current status of the evaluation job" - }, - "config_file": { - "type": "string", - "title": "Config File", - "description": "Path to the configuration file used for evaluation" - }, - "error": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Error", - "description": "Error message if the job failed" - }, - "output_path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Output Path", - "description": "Path to the output file if the job completed successfully" - }, - "created_at": { - "type": "string", - "format": "date-time", - "title": "Created At", - "description": "Timestamp when the job was created" - }, - "updated_at": { - "type": "string", - "format": "date-time", - "title": "Updated At", - "description": "Timestamp when the job was last updated" - }, - "expires_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ], - "title": "Expires At", - "description": "Timestamp when the job will expire" - } + { + "type": "null" + } + ], + "title": "Source Url" + } + }, + "type": "object", + "title": "AcquiredArtifacts", + "description": "Resolved file locations populated by source_acquisition, consumed by downstream checker nodes." + }, + "AgentIntermediateStep": { + "properties": { + "tool_name": { + "type": "string", + "title": "Tool Name" + }, + "action_log": { + "type": "string", + "title": "Action Log" + }, + "tool_input": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "job_id", - "status", - "config_file", - "created_at", - "updated_at" - ], - "title": "AIQEvaluateStatusResponse", - "description": "Response model for the evaluate status endpoint." - }, - "AIQResponseIntermediateStep": { - "properties": { - "id": { - "type": "string", - "title": "Id" - }, - "parent_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Parent Id" - }, - "type": { - "type": "string", - "title": "Type", - "default": "markdown" - }, - "name": { - "type": "string", - "title": "Name" - }, - "payload": { - "type": "string", - "title": "Payload" - } + { + "type": "object" + } + ], + "title": "Tool Input" + }, + "tool_output": { + "title": "Tool Output" + } + }, + "type": "object", + "required": [ + "tool_name", + "action_log", + "tool_input", + "tool_output" + ], + "title": "AgentIntermediateStep", + "description": "Represents info for an intermediate step taken by an agent." + }, + "AgentMorpheusEngineOutput": { + "properties": { + "vuln_id": { + "type": "string", + "title": "Vuln Id" + }, + "checklist": { + "items": { + "$ref": "#/components/schemas/ChecklistItemOutput" + }, + "type": "array", + "title": "Checklist" + }, + "summary": { + "type": "string", + "title": "Summary" + }, + "justification": { + "$ref": "#/components/schemas/JustificationOutput" + }, + "intel_score": { + "type": "integer", + "title": "Intel Score" + }, + "cvss": { + "anyOf": [ + { + "$ref": "#/components/schemas/CVSSOutput" }, - "additionalProperties": true, - "type": "object", - "required": [ - "id", - "name", - "payload" - ], - "title": "AIQResponseIntermediateStep", - "description": "AIQResponseSerializedStep is a data model that represents a serialized step in the AIQ Toolkit chat streaming API." - }, - "AIQUsage": { - "properties": { - "prompt_tokens": { - "type": "integer", - "title": "Prompt Tokens" - }, - "completion_tokens": { - "type": "integer", - "title": "Completion Tokens" - }, - "total_tokens": { - "type": "integer", - "title": "Total Tokens" - } + { + "type": "null" + } + ] + } + }, + "type": "object", + "required": [ + "vuln_id", + "checklist", + "summary", + "justification", + "intel_score", + "cvss" + ], + "title": "AgentMorpheusEngineOutput", + "description": "Contains all output generated by the main Agent Morpheus LLM Engine for a given vulnerability.\n\n- vuln_id: the ID of the vulnerability being processed by the LLM engine.\n- checklist: a list of ChecklistItemOutput objects, each containing an input and a response from the LLM agent.\n- summary: a short summary of the checklist inputs and responses, generated by an LLM.\n- justification: a JustificationOutput object containing details of the model's justification decision.\n- cvss: a CVSSOutput object containing the CVSS score and vector string for the vulnerability." + }, + "AgentMorpheusInfo": { + "properties": { + "vdb": { + "anyOf": [ + { + "$ref": "#/components/schemas/VdbPaths" }, - "type": "object", - "required": [ - "prompt_tokens", - "completion_tokens", - "total_tokens" - ], - "title": "AIQUsage" + { + "type": "null" + } + ] }, - "AgentIntermediateStep": { - "properties": { - "tool_name": { - "type": "string", - "title": "Tool Name" - }, - "action_log": { - "type": "string", - "title": "Action Log" - }, - "tool_input": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object" - } - ], - "title": "Tool Input" - }, - "tool_output": { - "title": "Tool Output" - } + "intel": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/CveIntel" + }, + "type": "array" }, - "type": "object", - "required": [ - "tool_name", - "action_log", - "tool_input", - "tool_output" - ], - "title": "AgentIntermediateStep", - "description": "Represents info for an intermediate step taken by an agent." - }, - "AgentMorpheusEngineOutput": { - "properties": { - "vuln_id": { - "type": "string", - "title": "Vuln Id" - }, - "checklist": { - "items": { - "$ref": "#/components/schemas/ChecklistItemOutput" - }, - "type": "array", - "title": "Checklist" - }, - "summary": { - "type": "string", - "title": "Summary" - }, - "justification": { - "$ref": "#/components/schemas/JustificationOutput" - } + { + "type": "null" + } + ], + "title": "Intel" + }, + "sbom": { + "anyOf": [ + { + "$ref": "#/components/schemas/SBOMInfo" }, - "type": "object", - "required": [ - "vuln_id", - "checklist", - "summary", - "justification" - ], - "title": "AgentMorpheusEngineOutput", - "description": "Contains all output generated by the main Agent Morpheus LLM Engine for a given vulnerability.\n\n- vuln_id: the ID of the vulnerability being processed by the LLM engine.\n- checklist: a list of ChecklistItemOutput objects, each containing an input and a response from the LLM agent.\n- summary: a short summary of the checklist inputs and responses, generated by an LLM.\n- justification: a JustificationOutput object containing details of the model's justification decision." - }, - "AgentMorpheusInfo": { - "properties": { - "vdb": { - "anyOf": [ - { - "$ref": "#/components/schemas/VdbPaths" - }, - { - "type": "null" - } - ] - }, - "intel": { - "anyOf": [ - { - "items": { - "$ref": "#/components/schemas/CveIntel" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Intel" - }, - "sbom": { - "anyOf": [ - { - "$ref": "#/components/schemas/SBOMInfo" - }, - { - "type": "null" - } - ] - }, - "vulnerable_dependencies": { - "anyOf": [ - { - "items": { - "$ref": "#/components/schemas/VulnerableDependencies" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Vulnerable Dependencies" - } - }, - "type": "object", - "title": "AgentMorpheusInfo", - "description": "Information used for decisioning in the Agent Morpheus engine. These information can all be automatically\ngenerated or retrieved by the pipeline from the input information.\n\n- vdb: paths to source code and documentation vector databases (VDBs) used to understand whether a vulnerability\n is exploitable in the source code.\n- intel: list of CveIntel objects representing intelligence for each vulnerability pulled from various vulnerability\n databases and APIs.\n- sbom: software bill of materials listing the packages and versions in the container image, used to understand\n whether the vulnerable package exists in the image.\n- vulnerable_dependencies: a list of VulnerableDependencies objects for each vuln_id, representing the SBOM packages\n and transitive dependencies that are vulnerable for the vuln_id." + { + "type": "null" + } + ] }, - "AgentMorpheusInput-Input": { - "properties": { - "scan": { - "$ref": "#/components/schemas/ScanInfoInput" - }, - "image": { - "$ref": "#/components/schemas/ImageInfoInput-Input" - } + "vulnerable_dependencies": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/VulnerableDependencies" + }, + "type": "array" }, - "type": "object", - "required": [ - "scan", - "image" - ], - "title": "AgentMorpheusInput", - "description": "Inputs required by the Agent Morpheus pipeline." + { + "type": "null" + } + ], + "title": "Vulnerable Dependencies" }, - "AgentMorpheusInput-Output": { - "properties": { - "scan": { - "$ref": "#/components/schemas/ScanInfoInput" - }, - "image": { - "$ref": "#/components/schemas/ImageInfoInput-Output" - } + "checker_context": { + "anyOf": [ + { + "$ref": "#/components/schemas/PackageCheckerContext" }, - "type": "object", - "required": [ - "scan", - "image" - ], - "title": "AgentMorpheusInput", - "description": "Inputs required by the Agent Morpheus pipeline." + { + "type": "null" + } + ] + } + }, + "type": "object", + "title": "AgentMorpheusInfo", + "description": "Information used for decisioning in the Agent Morpheus engine. These information can all be automatically\ngenerated or retrieved by the pipeline from the input information.\n\n- vdb: paths to source code and documentation vector databases (VDBs) used to understand whether a vulnerability\n is exploitable in the source code.\n- intel: list of CveIntel objects representing intelligence for each vulnerability pulled from various vulnerability\n databases and APIs.\n- sbom: software bill of materials listing the packages and versions in the container image, used to understand\n whether the vulnerable package exists in the image.\n- vulnerable_dependencies: a list of VulnerableDependencies objects for each vuln_id, representing the SBOM packages\n and transitive dependencies that are vulnerable for the vuln_id." + }, + "AgentMorpheusInput-Input": { + "properties": { + "scan": { + "$ref": "#/components/schemas/ScanInfoInput" }, - "AgentMorpheusOutput": { - "properties": { - "input": { - "$ref": "#/components/schemas/AgentMorpheusInput-Output" - }, - "info": { - "$ref": "#/components/schemas/AgentMorpheusInfo" - }, - "output": { - "items": { - "$ref": "#/components/schemas/AgentMorpheusEngineOutput" - }, - "type": "array", - "title": "Output" - } + "image": { + "$ref": "#/components/schemas/ImageInfoInput-Input" + }, + "credential_id": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "input", - "info", - "output" - ], - "title": "AgentMorpheusOutput", - "description": "\"\nThe final output of the Agent Morpheus pipeline.\nContains all fields in the AgentMorpheusEngineInput, plus the AgentMorpheusEngineOuput for each input vulnerability." - }, - "AudioContent": { - "properties": { - "type": { - "type": "string", - "const": "input_audio", - "title": "Type", - "default": "input_audio" - }, - "input_audio": { - "$ref": "#/components/schemas/InputAudio", - "default": { - "data": "default", - "format": "default" - } - } + { + "type": "null" + } + ], + "title": "Credential Id" + }, + "code_index_success": { + "anyOf": [ + { + "type": "boolean" }, - "additionalProperties": false, - "type": "object", - "title": "AudioContent" + { + "type": "null" + } + ], + "title": "Code Index Success" }, - "BaseMetricV3": { - "properties": { - "cvssV3": { - "$ref": "#/components/schemas/CVSSV3" - }, - "exploitabilityScore": { - "type": "number", - "title": "Exploitabilityscore" - }, - "impactScore": { - "type": "number", - "title": "Impactscore" - } + "failure_reason": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "cvssV3", - "exploitabilityScore", - "impactScore" - ], - "title": "BaseMetricV3" - }, - "Bugzilla": { - "properties": { - "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" - }, - "id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Id" - }, - "url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Url" - } + { + "type": "null" + } + ], + "title": "Failure Reason", + "default": "No failure reason provided" + } + }, + "type": "object", + "required": [ + "scan", + "image" + ], + "title": "AgentMorpheusInput", + "description": "Inputs required by the Agent Morpheus pipeline." + }, + "AgentMorpheusInput-Output": { + "properties": { + "scan": { + "$ref": "#/components/schemas/ScanInfoInput" + }, + "image": { + "$ref": "#/components/schemas/ImageInfoInput-Output" + }, + "credential_id": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "title": "Bugzilla" - }, - "CVSS": { - "properties": { - "score": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "title": "Score" - }, - "vector_string": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Vector String" - } + { + "type": "null" + } + ], + "title": "Credential Id" + }, + "code_index_success": { + "anyOf": [ + { + "type": "boolean" }, - "type": "object", - "title": "CVSS" - }, - "CVSS3": { - "properties": { - "cvss3_base_score": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "title": "Cvss3 Base Score" - }, - "cvss3_scoring_vector": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Cvss3 Scoring Vector" - }, - "status": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Status" - } + { + "type": "null" + } + ], + "title": "Code Index Success" + }, + "failure_reason": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "title": "CVSS3" + { + "type": "null" + } + ], + "title": "Failure Reason", + "default": "No failure reason provided" + } + }, + "type": "object", + "required": [ + "scan", + "image" + ], + "title": "AgentMorpheusInput", + "description": "Inputs required by the Agent Morpheus pipeline." + }, + "AgentMorpheusOutput": { + "properties": { + "input": { + "$ref": "#/components/schemas/AgentMorpheusInput-Output" }, - "CVSSV3": { - "properties": { - "attackComplexity": { - "type": "string", - "title": "Attackcomplexity" - }, - "attackVector": { - "type": "string", - "title": "Attackvector" - }, - "availabilityImpact": { - "type": "string", - "title": "Availabilityimpact" - }, - "baseScore": { - "type": "number", - "title": "Basescore" - }, - "baseSeverity": { - "type": "string", - "title": "Baseseverity" - }, - "confidentialityImpact": { - "type": "string", - "title": "Confidentialityimpact" - }, - "integrityImpact": { - "type": "string", - "title": "Integrityimpact" - }, - "privilegesRequired": { - "type": "string", - "title": "Privilegesrequired" - }, - "scope": { - "type": "string", - "title": "Scope" - }, - "userInteraction": { - "type": "string", - "title": "Userinteraction" - }, - "vectorString": { - "type": "string", - "title": "Vectorstring" - }, - "version": { - "type": "string", - "title": "Version" - } + "info": { + "$ref": "#/components/schemas/AgentMorpheusInfo" + }, + "output": { + "$ref": "#/components/schemas/OutputPayload" + } + }, + "type": "object", + "required": [ + "input", + "info", + "output" + ], + "title": "AgentMorpheusOutput", + "description": "\"\nThe final output of the Agent Morpheus pipeline.\nContains all fields in the AgentMorpheusEngineInput, plus the AgentMorpheusEngineOuput for each input vulnerability." + }, + "AnalysisType": { + "type": "string", + "enum": [ + "image", + "source" + ], + "title": "AnalysisType" + }, + "AsyncGenerateRequest": { + "properties": { + "scan": { + "$ref": "#/components/schemas/ScanInfoInput" + }, + "image": { + "$ref": "#/components/schemas/ImageInfoInput-Input" + }, + "credential_id": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "attackComplexity", - "attackVector", - "availabilityImpact", - "baseScore", - "baseSeverity", - "confidentialityImpact", - "integrityImpact", - "privilegesRequired", - "scope", - "userInteraction", - "vectorString", - "version" - ], - "title": "CVSSV3" + { + "type": "null" + } + ], + "title": "Credential Id" }, - "CWE": { - "properties": { - "cwe_id": { - "type": "string", - "title": "Cwe Id" - }, - "name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Name" - } + "code_index_success": { + "anyOf": [ + { + "type": "boolean" }, - "type": "object", - "required": [ - "cwe_id" - ], - "title": "CWE" + { + "type": "null" + } + ], + "title": "Code Index Success" }, - "ChecklistItemOutput": { - "properties": { - "input": { - "type": "string", - "title": "Input" - }, - "response": { - "type": "string", - "title": "Response" - }, - "intermediate_steps": { - "anyOf": [ - { - "items": { - "$ref": "#/components/schemas/AgentIntermediateStep" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Intermediate Steps" - } + "failure_reason": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "input", - "response" - ], - "title": "ChecklistItemOutput", - "description": "Input, response, and intermediate steps for a single checklist item provided to the LLM agent." - }, - "Configuration": { - "properties": { - "package": { - "type": "string", - "title": "Package" - }, - "system": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "System" - }, - "versionStartExcluding": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Versionstartexcluding" - }, - "versionEndExcluding": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Versionendexcluding" - }, - "versionStartIncluding": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Versionstartincluding" - }, - "versionEndIncluding": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Versionendincluding" - } + { + "type": "null" + } + ], + "title": "Failure Reason", + "default": "No failure reason provided" + }, + "job_id": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "package" - ], - "title": "Configuration" + { + "type": "null" + } + ], + "title": "Job Id", + "description": "Unique identifier for the evaluation job" }, - "CveIntel": { - "properties": { - "vuln_id": { - "type": "string", - "title": "Vuln Id" - }, - "ghsa": { - "anyOf": [ - { - "$ref": "#/components/schemas/CveIntelGhsa" - }, - { - "type": "null" - } - ] - }, - "nvd": { - "anyOf": [ - { - "$ref": "#/components/schemas/CveIntelNvd" - }, - { - "type": "null" - } - ] - }, - "rhsa": { - "anyOf": [ - { - "$ref": "#/components/schemas/CveIntelRhsa" - }, - { - "type": "null" - } - ] - }, - "ubuntu": { - "anyOf": [ - { - "$ref": "#/components/schemas/CveIntelUbuntu" - }, - { - "type": "null" - } - ] - }, - "epss": { - "anyOf": [ - { - "$ref": "#/components/schemas/CveIntelEpss" - }, - { - "type": "null" - } - ] - }, - "has_sufficient_intel_for_agent": { - "type": "boolean", - "title": "Has Sufficient Intel For Agent", - "description": "Logic to determine if the CVE has sufficient intel and can be passed to the agent.\n\nReturns\n-------\nbool\n True if enough intel has been found for the CVE", - "readOnly": true - } + "sync_timeout": { + "type": "integer", + "maximum": 300, + "minimum": 0, + "title": "Sync Timeout", + "description": "Attempt to perform the job synchronously up until `sync_timeout` sectonds, if the job hasn't been completed by then a job_id will be returned with a status code of 202.", + "default": 0 + }, + "expiry_seconds": { + "type": "integer", + "maximum": 86400, + "minimum": 600, + "title": "Expiry Seconds", + "description": "Optional time (in seconds) before the job expires. Clamped between 600 (10 min) and 86400 (24h).", + "default": 3600 + } + }, + "type": "object", + "required": [ + "scan", + "image" + ], + "title": "AsyncGenerateRequest" + }, + "AsyncGenerateResponse": { + "properties": { + "job_id": { + "type": "string", + "title": "Job Id", + "description": "Unique identifier for the job" + }, + "status": { + "type": "string", + "title": "Status", + "description": "Current status of the job" + } + }, + "type": "object", + "required": [ + "job_id", + "status" + ], + "title": "AsyncGenerateResponse", + "description": "Response model for the async generation endpoint." + }, + "AsyncGenerationStatusResponse": { + "properties": { + "job_id": { + "type": "string", + "title": "Job Id", + "description": "Unique identifier for the evaluation job" + }, + "status": { + "type": "string", + "title": "Status", + "description": "Current status of the evaluation job" + }, + "error": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "vuln_id", - "has_sufficient_intel_for_agent" - ], - "title": "CveIntel", - "description": "Information about a CVE (Common Vulnerabilities and Exposures) entry." - }, - "CveIntelEpss": { - "properties": { - "epss": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "title": "Epss" - }, - "percentile": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "title": "Percentile" - }, - "date": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Date" - } + { + "type": "null" + } + ], + "title": "Error", + "description": "Error message if the job failed" + }, + "created_at": { + "type": "string", + "format": "date-time", + "title": "Created At", + "description": "Timestamp when the job was created" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "title": "Updated At", + "description": "Timestamp when the job was last updated" + }, + "expires_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" }, - "additionalProperties": true, - "type": "object", - "title": "CveIntelEpss", - "description": "Information about an EPSS (Elastic Product Security Service) entry." + { + "type": "null" + } + ], + "title": "Expires At", + "description": "Timestamp when the job will expire" }, - "CveIntelGhsa": { - "properties": { - "ghsa_id": { - "type": "string", - "title": "Ghsa Id" - }, - "cve_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Cve Id" - }, - "summary": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Summary" - }, - "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" - }, - "severity": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Severity" - }, - "vulnerabilities": { - "anyOf": [ - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Vulnerabilities" - }, - "cvss": { - "anyOf": [ - { - "$ref": "#/components/schemas/CVSS" - }, - { - "type": "null" - } - ] - }, - "cwes": { - "anyOf": [ - { - "items": { - "$ref": "#/components/schemas/CWE" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Cwes" - }, - "published_at": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Published At" - }, - "updated_at": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Updated At" - } + "output": { + "anyOf": [ + { + "type": "object" }, - "additionalProperties": true, - "type": "object", - "required": [ - "ghsa_id" - ], - "title": "CveIntelGhsa", - "description": "Information about a GHSA (GitHub Security Advisory) entry." - }, - "CveIntelNvd": { - "properties": { - "cve_id": { - "type": "string", - "title": "Cve Id" - }, - "cve_description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Cve Description" - }, - "cvss_vector": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Cvss Vector" - }, - "cvss_base_score": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "title": "Cvss Base Score" - }, - "cvss_severity": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Cvss Severity" - }, - "cwe_name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Cwe Name" - }, - "cwe_description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Cwe Description" - }, - "cwe_extended_description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Cwe Extended Description" - }, - "configurations": { - "anyOf": [ - { - "items": { - "$ref": "#/components/schemas/Configuration" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Configurations" - }, - "vendor_names": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Vendor Names" - }, - "references": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "References" - }, - "disputed": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "title": "Disputed" - }, - "published_at": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Published At" - }, - "updated_at": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Updated At" - } + { + "type": "null" + } + ], + "title": "Output", + "description": "Output of the generate request, this is only available if the job completed successfully." + } + }, + "type": "object", + "required": [ + "job_id", + "status", + "created_at", + "updated_at" + ], + "title": "AsyncGenerationStatusResponse" + }, + "AudioContent": { + "properties": { + "type": { + "type": "string", + "const": "input_audio", + "title": "Type", + "default": "input_audio" + }, + "input_audio": { + "$ref": "#/components/schemas/InputAudio", + "default": { + "data": "default", + "format": "default" + } + } + }, + "additionalProperties": false, + "type": "object", + "title": "AudioContent" + }, + "BaseMetricV3": { + "properties": { + "cvssV3": { + "$ref": "#/components/schemas/CVSSV3" + }, + "exploitabilityScore": { + "anyOf": [ + { + "type": "number" }, - "additionalProperties": true, - "type": "object", - "required": [ - "cve_id" - ], - "title": "CveIntelNvd", - "description": "Information about an NVD (National Vulnerability Database) entry." + { + "type": "null" + } + ], + "title": "Exploitabilityscore" }, - "CveIntelRhsa": { - "properties": { - "bugzilla": { - "$ref": "#/components/schemas/Bugzilla" - }, - "details": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Details" - }, - "statement": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Statement" - }, - "package_state": { - "anyOf": [ - { - "items": { - "$ref": "#/components/schemas/PackageState" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Package State" - }, - "upstream_fix": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Upstream Fix" - }, - "cvss3": { - "anyOf": [ - { - "$ref": "#/components/schemas/CVSS3" - }, - { - "type": "null" - } - ] - } + "impactScore": { + "anyOf": [ + { + "type": "number" }, - "additionalProperties": true, - "type": "object", - "title": "CveIntelRhsa", - "description": "Information about a RHSA (Red Hat Security Advisory) entry." - }, - "CveIntelUbuntu": { - "properties": { - "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" - }, - "notes": { - "anyOf": [ - { - "items": { - "$ref": "#/components/schemas/Note" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Notes" - }, - "notices": { - "anyOf": [ - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Notices" - }, - "priority": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Priority" - }, - "ubuntu_description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Ubuntu Description" - }, - "impact": { - "anyOf": [ - { - "$ref": "#/components/schemas/Impact" - }, - { - "type": "null" - } - ] - } + { + "type": "null" + } + ], + "title": "Impactscore" + } + }, + "type": "object", + "required": [ + "cvssV3" + ], + "title": "BaseMetricV3" + }, + "Bugzilla": { + "properties": { + "description": { + "anyOf": [ + { + "type": "string" }, - "additionalProperties": true, - "type": "object", - "title": "CveIntelUbuntu", - "description": "Information about a Ubuntu CVE entry." - }, - "DependencyPackage": { - "properties": { - "system": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "System" - }, - "name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Name" - }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Version" - }, - "relation": { - "anyOf": [ - { - "type": "string", - "enum": [ - "SELF", - "DIRECT", - "INDIRECT" - ] - }, - { - "type": "null" - } - ], - "title": "Relation" - } + { + "type": "null" + } + ], + "title": "Description" + }, + "id": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "title": "DependencyPackage", - "description": "Information about a dependency package as obtained from deps.dev API for a given SBOM package." - }, - "FileSBOMInfoInput": { - "properties": { - "_type": { - "type": "string", - "const": "file", - "title": "Type", - "description": "The type of the object", - "default": "file" - }, - "file_path": { - "type": "string", - "title": "File Path" - } + { + "type": "null" + } + ], + "title": "Id" + }, + "url": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "file_path" - ], - "title": "FileSBOMInfoInput", - "description": "A file path pointing to a Software Bill of Materials file." - }, - "HTTPSBOMInfoInput": { - "properties": { - "_type": { - "type": "string", - "const": "http", - "title": "Type", - "description": "The type of the object", - "default": "http" - }, - "url": { - "type": "string", - "minLength": 1, - "format": "uri", - "title": "Url" - } + { + "type": "null" + } + ], + "title": "Url" + } + }, + "type": "object", + "title": "Bugzilla" + }, + "CVSS": { + "properties": { + "score": { + "anyOf": [ + { + "type": "number" }, - "type": "object", - "required": [ - "url" - ], - "title": "HTTPSBOMInfoInput", - "description": "A URL pointing to a Software Bill of Materials file." - }, - "HTTPValidationError": { - "properties": { - "detail": { - "items": { - "$ref": "#/components/schemas/ValidationError" - }, - "type": "array", - "title": "Detail" - } + { + "type": "null" + } + ], + "title": "Score" + }, + "vector_string": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Vector String" + } + }, + "type": "object", + "title": "CVSS" + }, + "CVSS3": { + "properties": { + "cvss3_base_score": { + "anyOf": [ + { + "type": "number" }, - "type": "object", - "title": "HTTPValidationError" + { + "type": "null" + } + ], + "title": "Cvss3 Base Score" }, - "ImageContent": { - "properties": { - "type": { - "type": "string", - "const": "image_url", - "title": "Type", - "default": "image_url" - }, - "image_url": { - "$ref": "#/components/schemas/ImageUrl", - "default": { - "url": "http://default.com/" - } - } + "cvss3_scoring_vector": { + "anyOf": [ + { + "type": "string" }, - "additionalProperties": false, - "type": "object", - "title": "ImageContent" - }, - "ImageInfoInput-Input": { - "properties": { - "name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Name" - }, - "tag": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Tag" - }, - "digest": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Digest" - }, - "platform": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Platform" - }, - "feed_group": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Feed Group" - }, - "source_info": { - "items": { - "$ref": "#/components/schemas/SourceDocumentsInfo" - }, - "type": "array", - "title": "Source Info" - }, - "sbom_info": { - "oneOf": [ - { - "$ref": "#/components/schemas/ManualSBOMInfoInput" - }, - { - "$ref": "#/components/schemas/FileSBOMInfoInput" - }, - { - "$ref": "#/components/schemas/HTTPSBOMInfoInput" - } - ], - "title": "Sbom Info" - } + { + "type": "null" + } + ], + "title": "Cvss3 Scoring Vector" + }, + "status": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "source_info", - "sbom_info" - ], - "title": "ImageInfoInput", - "description": "Information about a container image, including the source information and sbom information." - }, - "ImageInfoInput-Output": { - "properties": { - "name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Name" - }, - "tag": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Tag" - }, - "digest": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Digest" - }, - "platform": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Platform" - }, - "feed_group": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Feed Group" - }, - "source_info": { - "items": { - "$ref": "#/components/schemas/SourceDocumentsInfo" - }, - "type": "array", - "title": "Source Info" - }, - "sbom_info": { - "oneOf": [ - { - "$ref": "#/components/schemas/ManualSBOMInfoInput" - }, - { - "$ref": "#/components/schemas/FileSBOMInfoInput" - }, - { - "$ref": "#/components/schemas/HTTPSBOMInfoInput" - } - ], - "title": "Sbom Info" - } + { + "type": "null" + } + ], + "title": "Status" + } + }, + "type": "object", + "title": "CVSS3" + }, + "CVSSOutput": { + "properties": { + "vector_string": { + "type": "string", + "title": "Vector String" + }, + "score": { + "type": "string", + "title": "Score" + } + }, + "type": "object", + "required": [ + "vector_string", + "score" + ], + "title": "CVSSOutput", + "description": "CVSS (Common Vulnerability Scoring System) representing the severity of a vulnerability in reference to an image.\n- vector_string: The CVSS vector string that encodes the metric values used to calculate the score.\n- score: The calculated CVSS base score representing the severity of the vulnerability in the given image." + }, + "CVSSV3": { + "properties": { + "attackComplexity": { + "type": "string", + "title": "Attackcomplexity" + }, + "attackVector": { + "type": "string", + "title": "Attackvector" + }, + "availabilityImpact": { + "type": "string", + "title": "Availabilityimpact" + }, + "baseScore": { + "type": "number", + "title": "Basescore" + }, + "baseSeverity": { + "type": "string", + "title": "Baseseverity" + }, + "confidentialityImpact": { + "type": "string", + "title": "Confidentialityimpact" + }, + "integrityImpact": { + "type": "string", + "title": "Integrityimpact" + }, + "privilegesRequired": { + "type": "string", + "title": "Privilegesrequired" + }, + "scope": { + "type": "string", + "title": "Scope" + }, + "userInteraction": { + "type": "string", + "title": "Userinteraction" + }, + "vectorString": { + "type": "string", + "title": "Vectorstring" + }, + "version": { + "type": "string", + "title": "Version" + } + }, + "type": "object", + "required": [ + "attackComplexity", + "attackVector", + "availabilityImpact", + "baseScore", + "baseSeverity", + "confidentialityImpact", + "integrityImpact", + "privilegesRequired", + "scope", + "userInteraction", + "vectorString", + "version" + ], + "title": "CVSSV3" + }, + "CWE": { + "properties": { + "cwe_id": { + "type": "string", + "title": "Cwe Id" + }, + "name": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "source_info", - "sbom_info" - ], - "title": "ImageInfoInput", - "description": "Information about a container image, including the source information and sbom information." - }, - "ImageUrl": { - "properties": { - "url": { - "type": "string", - "maxLength": 2083, - "minLength": 1, - "format": "uri", - "title": "Url", - "default": "http://default.com/" - } + { + "type": "null" + } + ], + "title": "Name" + } + }, + "type": "object", + "required": [ + "cwe_id" + ], + "title": "CWE" + }, + "ChatRequest": { + "properties": { + "messages": { + "items": { + "$ref": "#/components/schemas/Message" + }, + "type": "array", + "title": "Messages" + }, + "model": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "title": "ImageUrl" + { + "type": "null" + } + ], + "title": "Model", + "description": "name of the model to use" }, - "Impact": { - "properties": { - "baseMetricV3": { - "$ref": "#/components/schemas/BaseMetricV3" - } + "frequency_penalty": { + "anyOf": [ + { + "type": "number" }, - "type": "object", - "required": [ - "baseMetricV3" - ], - "title": "Impact" - }, - "InputAudio": { - "properties": { - "data": { - "type": "string", - "title": "Data", - "default": "default" - }, - "format": { - "type": "string", - "title": "Format", - "default": "default" - } + { + "type": "null" + } + ], + "title": "Frequency Penalty", + "description": "Penalty for new tokens based on frequency in text", + "default": 0 + }, + "logit_bias": { + "anyOf": [ + { + "additionalProperties": { + "type": "number" + }, + "type": "object" }, - "type": "object", - "title": "InputAudio" + { + "type": "null" + } + ], + "title": "Logit Bias", + "description": "Modify likelihood of specified tokens appearing" }, - "JustificationOutput": { - "properties": { - "label": { - "type": "string", - "title": "Label" - }, - "reason": { - "type": "string", - "title": "Reason" - }, - "status": { - "type": "string", - "enum": [ - "TRUE", - "FALSE", - "UNKNOWN" - ], - "title": "Status" - } + "logprobs": { + "anyOf": [ + { + "type": "boolean" }, - "type": "object", - "required": [ - "label", - "reason", - "status" - ], - "title": "JustificationOutput", - "description": "Final justification for the vulnerability.\n\n- label: a categorical justification label classifying the status of an image against a given vulnerability, e.g.\n code_not_present, code_not_reachable, false_positive.\n- reason: a human-readable explanation for why justification label was selected.\n- status: a ternary status (TRUE, FALSE, OR UNKNOWN) that indicates whether the image can be exploited for a given\n vulnerability. Determined based on a mapping from the justification label." - }, - "ManualSBOMInfoInput": { - "properties": { - "_type": { - "type": "string", - "const": "manual", - "title": "Type", - "description": "The type of the object", - "default": "manual" - }, - "packages": { - "items": { - "$ref": "#/components/schemas/SBOMPackage" - }, - "type": "array", - "title": "Packages" - } + { + "type": "null" + } + ], + "title": "Logprobs", + "description": "Whether to return log probabilities" + }, + "top_logprobs": { + "anyOf": [ + { + "type": "integer" }, - "type": "object", - "required": [ - "packages" - ], - "title": "ManualSBOMInfoInput", - "description": "Manually provided Software Bill of Materials, consisting of a list of SBOMPackage objects." - }, - "Message": { - "properties": { - "content": { - "anyOf": [ - { - "type": "string" - }, - { - "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/TextContent" - }, - { - "$ref": "#/components/schemas/ImageContent" - }, - { - "$ref": "#/components/schemas/AudioContent" - } - ], - "discriminator": { - "propertyName": "type", - "mapping": { - "image_url": "#/components/schemas/ImageContent", - "input_audio": "#/components/schemas/AudioContent", - "text": "#/components/schemas/TextContent" - } - } - }, - "type": "array" - } - ], - "title": "Content" - }, - "role": { - "type": "string", - "title": "Role" - } + { + "type": "null" + } + ], + "title": "Top Logprobs", + "description": "Number of most likely tokens to return" + }, + "max_tokens": { + "anyOf": [ + { + "type": "integer" }, - "type": "object", - "required": [ - "content", - "role" - ], - "title": "Message" - }, - "Note": { - "properties": { - "author": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Author" - }, - "note": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Note" - } + { + "type": "null" + } + ], + "title": "Max Tokens", + "description": "Maximum number of tokens to generate" + }, + "n": { + "anyOf": [ + { + "type": "integer" }, - "type": "object", - "title": "Note" - }, - "PackageState": { - "properties": { - "product_name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Product Name" - }, - "fix_state": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Fix State" + { + "type": "null" + } + ], + "title": "N", + "description": "Number of chat completion choices to generate", + "default": 1 + }, + "presence_penalty": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Presence Penalty", + "description": "Penalty for new tokens based on presence in text", + "default": 0 + }, + "response_format": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Response Format", + "description": "Response format specification" + }, + "seed": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Seed", + "description": "Random seed for deterministic sampling" + }, + "service_tier": { + "anyOf": [ + { + "type": "string", + "enum": [ + "auto", + "default" + ] + }, + { + "type": "null" + } + ], + "title": "Service Tier", + "description": "Service tier for the request" + }, + "stream": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Stream", + "description": "Whether to stream partial message deltas", + "default": false + }, + "stream_options": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Stream Options", + "description": "Options for streaming" + }, + "temperature": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Temperature", + "description": "Sampling temperature between 0 and 2", + "default": 1 + }, + "top_p": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Top P", + "description": "Nucleus sampling parameter" + }, + "tools": { + "anyOf": [ + { + "items": { + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tools", + "description": "List of tools the model may call" + }, + "tool_choice": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Choice", + "description": "Controls which tool is called" + }, + "parallel_tool_calls": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Parallel Tool Calls", + "description": "Whether to enable parallel function calling", + "default": true + }, + "user": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "User", + "description": "Unique identifier representing end-user" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "messages" + ], + "title": "ChatRequest", + "description": "ChatRequest is a data model that represents a request to the NAT chat API.\nFully compatible with OpenAI Chat Completions API specification.", + "example": { + "messages": [ + { + "content": "who are you?", + "role": "user" + } + ], + "model": "nvidia/nemotron", + "stream": false, + "temperature": 0.7 + } + }, + "ChatResponse": { + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "object": { + "type": "string", + "title": "Object", + "default": "chat.completion" + }, + "model": { + "type": "string", + "title": "Model", + "default": "" + }, + "created": { + "type": "integer", + "title": "Created" + }, + "choices": { + "items": { + "$ref": "#/components/schemas/Choice" + }, + "type": "array", + "title": "Choices" + }, + "usage": { + "anyOf": [ + { + "$ref": "#/components/schemas/Usage" + }, + { + "type": "null" + } + ] + }, + "system_fingerprint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "System Fingerprint" + }, + "service_tier": { + "anyOf": [ + { + "type": "string", + "enum": [ + "scale", + "default" + ] + }, + { + "type": "null" + } + ], + "title": "Service Tier" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "id", + "created", + "choices" + ], + "title": "ChatResponse", + "description": "ChatResponse is a data model that represents a response from the NAT chat API.\nFully compatible with OpenAI Chat Completions API specification." + }, + "ChatResponseChunk": { + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "choices": { + "items": { + "$ref": "#/components/schemas/Choice" + }, + "type": "array", + "title": "Choices" + }, + "created": { + "type": "integer", + "title": "Created" + }, + "model": { + "type": "string", + "title": "Model", + "default": "" + }, + "object": { + "type": "string", + "title": "Object", + "default": "chat.completion.chunk" + }, + "system_fingerprint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "System Fingerprint" + }, + "service_tier": { + "anyOf": [ + { + "type": "string", + "enum": [ + "scale", + "default" + ] + }, + { + "type": "null" + } + ], + "title": "Service Tier" + }, + "usage": { + "anyOf": [ + { + "$ref": "#/components/schemas/Usage" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "id", + "choices", + "created" + ], + "title": "ChatResponseChunk", + "description": "ChatResponseChunk is a data model that represents a response chunk from the NAT chat streaming API.\nFully compatible with OpenAI Chat Completions API specification." + }, + "ChecklistItemOutput": { + "properties": { + "input": { + "type": "string", + "title": "Input" + }, + "response": { + "type": "string", + "title": "Response" + }, + "intermediate_steps": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/AgentIntermediateStep" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Intermediate Steps" + } + }, + "type": "object", + "required": [ + "input", + "response" + ], + "title": "ChecklistItemOutput", + "description": "Input, response, and intermediate steps for a single checklist item provided to the LLM agent." + }, + "Choice": { + "properties": { + "message": { + "anyOf": [ + { + "$ref": "#/components/schemas/ChoiceMessage" + }, + { + "type": "null" + } + ] + }, + "delta": { + "anyOf": [ + { + "$ref": "#/components/schemas/ChoiceDelta" + }, + { + "type": "null" + } + ] + }, + "finish_reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "stop", + "length", + "tool_calls", + "content_filter", + "function_call" + ] + }, + { + "type": "null" + } + ], + "title": "Finish Reason" + }, + "index": { + "type": "integer", + "title": "Index" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "index" + ], + "title": "Choice" + }, + "ChoiceDelta": { + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Content" + }, + "role": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Role" + } + }, + "type": "object", + "title": "ChoiceDelta", + "description": "Delta object for streaming responses (OpenAI-compatible)" + }, + "ChoiceMessage": { + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Content" + }, + "role": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Role" + } + }, + "type": "object", + "title": "ChoiceMessage" + }, + "Configuration": { + "properties": { + "package": { + "type": "string", + "title": "Package" + }, + "vendor": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Vendor" + }, + "system": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "System" + }, + "versionStartExcluding": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Versionstartexcluding" + }, + "versionEndExcluding": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Versionendexcluding" + }, + "versionStartIncluding": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Versionstartincluding" + }, + "versionEndIncluding": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Versionendincluding" + } + }, + "type": "object", + "required": [ + "package" + ], + "title": "Configuration" + }, + "CveIntel": { + "properties": { + "vuln_id": { + "type": "string", + "title": "Vuln Id" + }, + "ghsa": { + "anyOf": [ + { + "$ref": "#/components/schemas/CveIntelGhsa" + }, + { + "type": "null" + } + ] + }, + "nvd": { + "anyOf": [ + { + "$ref": "#/components/schemas/CveIntelNvd" + }, + { + "type": "null" + } + ] + }, + "rhsa": { + "anyOf": [ + { + "$ref": "#/components/schemas/CveIntelRhsa" + }, + { + "type": "null" + } + ] + }, + "ubuntu": { + "anyOf": [ + { + "$ref": "#/components/schemas/CveIntelUbuntu" + }, + { + "type": "null" + } + ] + }, + "epss": { + "anyOf": [ + { + "$ref": "#/components/schemas/CveIntelEpss" + }, + { + "type": "null" + } + ] + }, + "plugin_data": { + "items": { + "$ref": "#/components/schemas/IntelPluginData" + }, + "type": "array", + "title": "Plugin Data", + "default": [] + }, + "intel_score": { + "type": "integer", + "title": "Intel Score", + "default": 0 + }, + "has_sufficient_intel_for_agent": { + "type": "boolean", + "title": "Has Sufficient Intel For Agent", + "description": "Logic to determine if the CVE has sufficient intel and can be passed to the agent.\n\nReturns\n-------\nbool\n True if enough intel has been found for the CVE", + "readOnly": true + } + }, + "type": "object", + "required": [ + "vuln_id", + "has_sufficient_intel_for_agent" + ], + "title": "CveIntel", + "description": "Information about a CVE (Common Vulnerabilities and Exposures) entry." + }, + "CveIntelEpss": { + "properties": { + "epss": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Epss" + }, + "percentile": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Percentile" + }, + "date": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Date" + } + }, + "additionalProperties": true, + "type": "object", + "title": "CveIntelEpss", + "description": "Information about an EPSS (Elastic Product Security Service) entry." + }, + "CveIntelGhsa": { + "properties": { + "ghsa_id": { + "type": "string", + "title": "Ghsa Id" + }, + "cve_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cve Id" + }, + "summary": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Summary" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "severity": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Severity" + }, + "vulnerabilities": { + "anyOf": [ + { + "items": { + + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Vulnerabilities" + }, + "cvss": { + "anyOf": [ + { + "$ref": "#/components/schemas/CVSS" + }, + { + "type": "null" + } + ] + }, + "cwes": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/CWE" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Cwes" + }, + "published_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Published At" + }, + "updated_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "ghsa_id" + ], + "title": "CveIntelGhsa", + "description": "Information about a GHSA (GitHub Security Advisory) entry." + }, + "CveIntelNvd": { + "properties": { + "cve_id": { + "type": "string", + "title": "Cve Id" + }, + "cve_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cve Description" + }, + "cvss_vector": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cvss Vector" + }, + "cvss_base_score": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cvss Base Score" + }, + "cvss_severity": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cvss Severity" + }, + "cwe_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cwe Id" + }, + "cwe_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cwe Name" + }, + "cwe_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cwe Description" + }, + "cwe_extended_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cwe Extended Description" + }, + "configurations": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/Configuration" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Configurations" + }, + "vendor_names": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Vendor Names" + }, + "references": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "References" + }, + "disputed": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Disputed" + }, + "published_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Published At" + }, + "updated_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "cve_id" + ], + "title": "CveIntelNvd", + "description": "Information about an NVD (National Vulnerability Database) entry." + }, + "CveIntelRhsa": { + "properties": { + "bugzilla": { + "$ref": "#/components/schemas/Bugzilla" + }, + "details": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Details" + }, + "statement": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Statement" + }, + "package_state": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/PackageState" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Package State" + }, + "upstream_fix": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Upstream Fix" + }, + "cvss3": { + "anyOf": [ + { + "$ref": "#/components/schemas/CVSS3" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": true, + "type": "object", + "title": "CveIntelRhsa", + "description": "Information about a RHSA (Red Hat Security Advisory) entry." + }, + "CveIntelUbuntu": { + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "notes": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/Note" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Notes" + }, + "notices": { + "anyOf": [ + { + "items": { + + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Notices" + }, + "priority": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Priority" + }, + "ubuntu_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Ubuntu Description" + }, + "impact": { + "anyOf": [ + { + "$ref": "#/components/schemas/Impact" + }, + { + "type": "null" + } + ] + }, + "patches": { + "anyOf": [ + { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Patches", + "description": "Map of package name to patch refs (e.g., 'upstream: https://github.com/.../commit/...')" + } + }, + "additionalProperties": true, + "type": "object", + "title": "CveIntelUbuntu", + "description": "Information about a Ubuntu CVE entry." + }, + "DependencyPackage": { + "properties": { + "system": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "System" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version" + }, + "relation": { + "anyOf": [ + { + "type": "string", + "enum": [ + "SELF", + "DIRECT", + "INDIRECT" + ] + }, + { + "type": "null" + } + ], + "title": "Relation" + } + }, + "type": "object", + "title": "DependencyPackage", + "description": "Information about a dependency package as obtained from deps.dev API for a given SBOM package." + }, + "Ecosystem": { + "type": "string", + "enum": [ + "go", + "python", + "javascript", + "java", + "c" + ], + "title": "Ecosystem" + }, + "EnumIdentifyResult": { + "type": "string", + "enum": [ + "yes", + "no", + "unknown" + ], + "title": "EnumIdentifyResult", + "description": "Result of the PackageIdentify phase for a single CVE." + }, + "EvaluateRequest": { + "properties": { + "config_file": { + "type": "string", + "title": "Config File", + "description": "Path to the configuration file for evaluation" + }, + "job_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Job Id", + "description": "Unique identifier for the evaluation job" + }, + "reps": { + "type": "integer", + "exclusiveMinimum": 0, + "title": "Reps", + "description": "Number of repetitions for the evaluation, defaults to 1", + "default": 1 + }, + "expiry_seconds": { + "type": "integer", + "exclusiveMinimum": 0, + "title": "Expiry Seconds", + "description": "Optional time (in seconds) before the job expires. Clamped between 600 (10 min) and 86400 (24h).", + "default": 3600 + } + }, + "type": "object", + "required": [ + "config_file" + ], + "title": "EvaluateRequest", + "description": "Request model for the evaluate endpoint." + }, + "EvaluateResponse": { + "properties": { + "job_id": { + "type": "string", + "title": "Job Id", + "description": "Unique identifier for the job" + }, + "status": { + "type": "string", + "title": "Status", + "description": "Current status of the job" + } + }, + "type": "object", + "required": [ + "job_id", + "status" + ], + "title": "EvaluateResponse", + "description": "Response model for the evaluate endpoint." + }, + "EvaluateStatusResponse": { + "properties": { + "job_id": { + "type": "string", + "title": "Job Id", + "description": "Unique identifier for the evaluation job" + }, + "status": { + "type": "string", + "title": "Status", + "description": "Current status of the evaluation job" + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error", + "description": "Error message if the job failed" + }, + "created_at": { + "type": "string", + "format": "date-time", + "title": "Created At", + "description": "Timestamp when the job was created" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "title": "Updated At", + "description": "Timestamp when the job was last updated" + }, + "expires_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ], + "title": "Expires At", + "description": "Timestamp when the job will expire" + }, + "config_file": { + "type": "string", + "title": "Config File", + "description": "Path to the configuration file used for evaluation" + }, + "output_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Output Path", + "description": "Path to the output file if the job completed successfully" + } + }, + "type": "object", + "required": [ + "job_id", + "status", + "created_at", + "updated_at", + "config_file" + ], + "title": "EvaluateStatusResponse", + "description": "Response model for the evaluate status endpoint." + }, + "FileSBOMInfoInput": { + "properties": { + "_type": { + "type": "string", + "const": "file", + "title": "Type", + "description": "The type of the object", + "default": "file" + }, + "file_path": { + "type": "string", + "title": "File Path" + } + }, + "type": "object", + "required": [ + "file_path" + ], + "title": "FileSBOMInfoInput", + "description": "A file path pointing to a Software Bill of Materials file." + }, + "HTTPSBOMInfoInput": { + "properties": { + "_type": { + "type": "string", + "const": "http", + "title": "Type", + "description": "The type of the object", + "default": "http" + }, + "url": { + "type": "string", + "minLength": 1, + "format": "uri", + "title": "Url" + } + }, + "type": "object", + "required": [ + "url" + ], + "title": "HTTPSBOMInfoInput", + "description": "A URL pointing to a Software Bill of Materials file." + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "type": "array", + "title": "Detail" + } + }, + "type": "object", + "title": "HTTPValidationError" + }, + "HealthStatusResponse": { + "properties": { + "status": { + "type": "string", + "title": "Status" + } + }, + "type": "object", + "required": [ + "status" + ], + "title": "HealthStatusResponse" + }, + "ImageContent": { + "properties": { + "type": { + "type": "string", + "const": "image_url", + "title": "Type", + "default": "image_url" + }, + "image_url": { + "$ref": "#/components/schemas/ImageUrl", + "default": { + "url": "http://default.com/" + } + } + }, + "additionalProperties": false, + "type": "object", + "title": "ImageContent" + }, + "ImageInfoInput-Input": { + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "tag": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Tag" + }, + "digest": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Digest" + }, + "platform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Platform" + }, + "feed_group": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feed Group" + }, + "ecosystem": { + "anyOf": [ + { + "$ref": "#/components/schemas/Ecosystem" + }, + { + "type": "null" + } + ] + }, + "manifest_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Manifest Path" + }, + "analysis_type": { + "$ref": "#/components/schemas/AnalysisType" + }, + "pipeline_mode": { + "$ref": "#/components/schemas/PipelineMode", + "default": "full_pipeline" + }, + "target_package": { + "anyOf": [ + { + "$ref": "#/components/schemas/TargetPackage" + }, + { + "type": "null" + } + ] + }, + "source_info": { + "items": { + "$ref": "#/components/schemas/SourceDocumentsInfo" + }, + "type": "array", + "title": "Source Info", + "default": [] + }, + "sbom_info": { + "anyOf": [ + { + "oneOf": [ + { + "$ref": "#/components/schemas/ManualSBOMInfoInput" }, - "package_name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Package Name" + { + "$ref": "#/components/schemas/FileSBOMInfoInput" }, - "cpe": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Cpe" + { + "$ref": "#/components/schemas/HTTPSBOMInfoInput" } + ] }, - "type": "object", - "title": "PackageState" - }, - "SBOMInfo": { - "properties": { - "packages": { - "items": { - "$ref": "#/components/schemas/SBOMPackage" - }, - "type": "array", - "title": "Packages" - } + { + "type": "null" + } + ], + "title": "Sbom Info" + } + }, + "type": "object", + "required": [ + "analysis_type" + ], + "title": "ImageInfoInput", + "description": "Information about a container image, including the source information and sbom information." + }, + "ImageInfoInput-Output": { + "properties": { + "name": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "packages" - ], - "title": "SBOMInfo", - "description": "List of SBOMPackage objects representing the packages found in the input image." - }, - "SBOMPackage": { - "properties": { - "name": { - "type": "string", - "title": "Name" - }, - "version": { - "type": "string", - "title": "Version" + { + "type": "null" + } + ], + "title": "Name" + }, + "tag": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Tag" + }, + "digest": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Digest" + }, + "platform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Platform" + }, + "feed_group": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feed Group" + }, + "ecosystem": { + "anyOf": [ + { + "$ref": "#/components/schemas/Ecosystem" + }, + { + "type": "null" + } + ] + }, + "manifest_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Manifest Path" + }, + "analysis_type": { + "$ref": "#/components/schemas/AnalysisType" + }, + "pipeline_mode": { + "$ref": "#/components/schemas/PipelineMode", + "default": "full_pipeline" + }, + "target_package": { + "anyOf": [ + { + "$ref": "#/components/schemas/TargetPackage" + }, + { + "type": "null" + } + ] + }, + "source_info": { + "items": { + "$ref": "#/components/schemas/SourceDocumentsInfo" + }, + "type": "array", + "title": "Source Info", + "default": [] + }, + "sbom_info": { + "anyOf": [ + { + "oneOf": [ + { + "$ref": "#/components/schemas/ManualSBOMInfoInput" }, - "path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Path" + { + "$ref": "#/components/schemas/FileSBOMInfoInput" }, - "system": { - "type": "string", - "title": "System" + { + "$ref": "#/components/schemas/HTTPSBOMInfoInput" } + ] }, - "type": "object", - "required": [ - "name", - "version", - "system" - ], - "title": "SBOMPackage", - "description": "Information about a single package in the container image's Software Bill of Materials (SBOM)." - }, - "ScanInfoInput": { - "properties": { - "id": { - "type": "string", - "title": "Id" - }, - "type": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Type" - }, - "started_at": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Started At" - }, - "completed_at": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Completed At" - }, - "vulns": { - "items": { - "$ref": "#/components/schemas/VulnInfo" - }, - "type": "array", - "minItems": 1, - "title": "Vulns" - } + { + "type": "null" + } + ], + "title": "Sbom Info" + } + }, + "type": "object", + "required": [ + "analysis_type" + ], + "title": "ImageInfoInput", + "description": "Information about a container image, including the source information and sbom information." + }, + "ImageUrl": { + "properties": { + "url": { + "type": "string", + "maxLength": 2083, + "minLength": 1, + "format": "uri", + "title": "Url", + "default": "http://default.com/" + } + }, + "type": "object", + "title": "ImageUrl" + }, + "Impact": { + "properties": { + "baseMetricV3": { + "$ref": "#/components/schemas/BaseMetricV3" + } + }, + "type": "object", + "required": [ + "baseMetricV3" + ], + "title": "Impact" + }, + "InputAudio": { + "properties": { + "data": { + "type": "string", + "title": "Data", + "default": "default" + }, + "format": { + "type": "string", + "title": "Format", + "default": "default" + } + }, + "type": "object", + "title": "InputAudio" + }, + "IntelPluginData": { + "properties": { + "label": { + "type": "string", + "title": "Label" + }, + "description": { + "type": "string", + "title": "Description" + } + }, + "type": "object", + "required": [ + "label", + "description" + ], + "title": "IntelPluginData" + }, + "JustificationOutput": { + "properties": { + "label": { + "type": "string", + "title": "Label" + }, + "reason": { + "type": "string", + "title": "Reason" + }, + "status": { + "type": "string", + "enum": [ + "TRUE", + "FALSE", + "UNKNOWN" + ], + "title": "Status" + } + }, + "type": "object", + "required": [ + "label", + "reason", + "status" + ], + "title": "JustificationOutput", + "description": "Final justification for the vulnerability.\n\n- label: a categorical justification label classifying the status of an image against a given vulnerability, e.g.\n code_not_present, code_not_reachable, false_positive.\n- reason: a human-readable explanation for why justification label was selected.\n- status: a ternary status (TRUE, FALSE, OR UNKNOWN) that indicates whether the image can be exploited for a given\n vulnerability. Determined based on a mapping from the justification label." + }, + "L1InvestigationResult": { + "properties": { + "downstream_report": { + "anyOf": [ + { + "type": "object" }, - "type": "object", - "required": [ - "vulns" - ], - "title": "ScanInfoInput", - "description": "Information about a unique scan for a container image against a list of vulnerabilies." - }, - "SourceDocumentsInfo": { - "properties": { - "type": { - "type": "string", - "enum": [ - "code", - "doc" - ], - "title": "Type" - }, - "git_repo": { - "type": "string", - "minLength": 1, - "title": "Git Repo" - }, - "ref": { - "type": "string", - "minLength": 1, - "title": "Ref" - }, - "include": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Include", - "default": [ - "*.py", - "*.ipynb" - ] - }, - "exclude": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Exclude", - "default": [] - } + { + "type": "null" + } + ], + "title": "Downstream Report", + "description": "Serialized DownstreamSearchReport from L1 investigation" + }, + "upstream_report": { + "anyOf": [ + { + "type": "object" }, - "type": "object", - "required": [ - "type", - "git_repo", - "ref" - ], - "title": "SourceDocumentsInfo", - "description": "Information about the source documents for the container image.\n\n- type: document type.\n- git_repo: git repo URL where the source documents can be cloned.\n- ref: git reference, such as tag/branch/commit_id\n- include: file extensions to include when indexing the source documents.\n- exclude: file extensions to exclude when indexing the source documents." - }, - "TextContent": { - "properties": { - "type": { - "type": "string", - "const": "text", - "title": "Type", - "default": "text" - }, - "text": { - "type": "string", - "title": "Text", - "default": "default" - } + { + "type": "null" + } + ], + "title": "Upstream Report", + "description": "Serialized UpstreamSearchReport from L1 investigation" + }, + "l1_agent_answer": { + "anyOf": [ + { + "type": "string" }, - "additionalProperties": false, - "type": "object", - "title": "TextContent" - }, - "ValidationError": { - "properties": { - "loc": { - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "integer" - } - ] - }, - "type": "array", - "title": "Location" - }, - "msg": { - "type": "string", - "title": "Message" - }, - "type": { - "type": "string", - "title": "Error Type" - } + { + "type": "null" + } + ], + "title": "L1 Agent Answer", + "description": "Final answer from the L1 ReAct agent" + }, + "vulnerability_intel": { + "anyOf": [ + { + "$ref": "#/components/schemas/VulnerabilityIntel" }, - "type": "object", - "required": [ - "loc", - "msg", - "type" - ], - "title": "ValidationError" - }, - "VdbPaths": { - "properties": { - "code_vdb_path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Code Vdb Path" - }, - "doc_vdb_path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Doc Vdb Path" - }, - "code_index_path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Code Index Path" - } + { + "type": "null" + } + ], + "description": "Structured vulnerability intelligence extracted from CVE advisories and patches" + }, + "preliminary_verdict": { + "type": "string", + "enum": [ + "vulnerable", + "protected", + "not_present", + "uncertain" + ], + "title": "Preliminary Verdict", + "description": "L1 verdict before L2 refinement", + "default": "uncertain" + }, + "confidence": { + "type": "number", + "maximum": 1, + "minimum": 0, + "title": "Confidence", + "description": "Confidence in the preliminary verdict", + "default": 0 + } + }, + "type": "object", + "title": "L1InvestigationResult", + "description": "Intermediate result from L1 investigation, input to L2 or report generation." + }, + "L2BuildResult": { + "properties": { + "compilation_status": { + "type": "string", + "enum": [ + "compiled", + "not_compiled", + "unknown" + ], + "title": "Compilation Status", + "description": "Whether vulnerable code is compiled into the binary", + "default": "unknown" + }, + "compilation_confidence": { + "type": "number", + "maximum": 1, + "minimum": 0, + "title": "Compilation Confidence", + "description": "Confidence in compilation status", + "default": 0 + }, + "compilation_evidence": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "title": "VdbPaths", - "description": "Paths to where the generated VDBs are stored." + { + "type": "null" + } + ], + "title": "Compilation Evidence", + "description": "Evidence supporting compilation status" }, - "VulnInfo": { - "properties": { - "vuln_id": { - "type": "string", - "title": "Vuln Id" - }, - "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" - }, - "score": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "title": "Score" - }, - "severity": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Severity" - }, - "published_date": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Published Date" - }, - "last_modified_date": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Last Modified Date" - }, - "url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Url" - }, - "feed_group": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Feed Group" - }, - "package": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Package" - }, - "package_version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Package Version" - }, - "package_name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Package Name" - }, - "package_type": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Package Type" - } + "hardening_relevant": { + "anyOf": [ + { + "type": "boolean" }, - "additionalProperties": true, - "type": "object", - "required": [ - "vuln_id" - ], - "title": "VulnInfo", - "description": "Information about a vulnerability." - }, - "VulnerableDependencies": { - "properties": { - "vuln_id": { - "type": "string", - "title": "Vuln Id" - }, - "vuln_package_intel_sources": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Vuln Package Intel Sources" - }, - "vulnerable_sbom_packages": { - "items": { - "$ref": "#/components/schemas/VulnerableSBOMPackage" - }, - "type": "array", - "title": "Vulnerable Sbom Packages" - } + { + "type": "null" + } + ], + "title": "Hardening Relevant", + "description": "Whether detected hardening flags are relevant to the CVE" + }, + "hardening_flags": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Hardening Flags", + "description": "Hardening flags detected in build log or binary" + }, + "hardening_rationale": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "vuln_id", - "vuln_package_intel_sources", - "vulnerable_sbom_packages" - ], - "title": "VulnerableDependencies", - "description": "Information about the vulnerable SBOM packages associated with the vuln_id.\n\n- vuln_id: vulnerability ID (e.g. CVE ID, GHSA ID) associated with the vulnerable package list.\n- vuln_package_intel_sources: list of sources (e.g. \"ghsa\", \"nvd\", \"ubuntu\", \"rhsa\") that provided\n the vulnerable package/version intel for the vuln_id.\n- vulnerable_sbom_packages: list of VulnerableSBOMPackage objects, representing the SBOM packages that are\n vulnerable for a given vuln_id." - }, - "VulnerableSBOMPackage": { - "properties": { - "name": { - "type": "string", - "title": "Name" - }, - "version": { - "type": "string", - "title": "Version" - }, - "vulnerable_dependency_package": { - "$ref": "#/components/schemas/DependencyPackage" + { + "type": "null" + } + ], + "title": "Hardening Rationale", + "description": "Rationale for hardening relevance judgment" + }, + "l2_override_verdict": { + "enum": [ + "not_vulnerable", + "vulnerable_mitigated", + null], + "title": "L2 Override Verdict", + "description": "L2 verdict override (if any)" + } + }, + "type": "object", + "title": "L2BuildResult", + "description": "Result from L2 Build Agent (BuildCompilationCheck + HardeningCheck)." + }, + "ManualSBOMInfoInput": { + "properties": { + "_type": { + "type": "string", + "const": "manual", + "title": "Type", + "description": "The type of the object", + "default": "manual" + }, + "packages": { + "items": { + "$ref": "#/components/schemas/SBOMPackage" + }, + "type": "array", + "title": "Packages" + } + }, + "type": "object", + "required": [ + "packages" + ], + "title": "ManualSBOMInfoInput", + "description": "Manually provided Software Bill of Materials, consisting of a list of SBOMPackage objects." + }, + "Message": { + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/TextContent" + }, + { + "$ref": "#/components/schemas/ImageContent" + }, + { + "$ref": "#/components/schemas/AudioContent" + } + ], + "discriminator": { + "propertyName": "type", + "mapping": { + "image_url": "#/components/schemas/ImageContent", + "input_audio": "#/components/schemas/AudioContent", + "text": "#/components/schemas/TextContent" + } } + }, + "type": "array" + } + ], + "title": "Content" + }, + "role": { + "type": "string", + "title": "Role" + } + }, + "type": "object", + "required": [ + "content", + "role" + ], + "title": "Message" + }, + "Note": { + "properties": { + "author": { + "anyOf": [ + { + "type": "string" }, - "type": "object", - "required": [ - "name", - "version", - "vulnerable_dependency_package" - ], - "title": "VulnerableSBOMPackage", - "description": "Information about a vulnerable SBOM package and its related vulnerable dependency package.\n\n- name: SBOM package name\n- version: SBOM package version\n- vulnerable_dependency_package: DependencyPackage object with info about the vulnerable dependency package.\n If an SBOM package itself is vulnerable, the vulnerable_dependency_package.relation will be \"SELF\".\n Otherwise, if it is vulnerable due to its dependency, the vulnerable_dependency_package.relation will be either\n \"DIRECT\" or \"INDIRECT\"." + { + "type": "null" + } + ], + "title": "Author" + }, + "note": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Note" + } + }, + "type": "object", + "title": "Note" + }, + "OutputPayload": { + "properties": { + "analysis": { + "items": { + "$ref": "#/components/schemas/AgentMorpheusEngineOutput" + }, + "type": "array", + "title": "Analysis" + }, + "vex": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Vex" + } + }, + "type": "object", + "required": [ + "analysis", + "vex" + ], + "title": "OutputPayload", + "description": "Wrapper for final pipeline results.\n- analysis: per-vulnerability analysis results\n- vex: the vulnerability exploitability exchange document JSON" + }, + "PackageCheckerContext": { + "properties": { + "status": { + "anyOf": [ + { + "$ref": "#/components/schemas/PackageCheckerStatus" + }, + { + "type": "null" + } + ] + }, + "source_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source Key" + }, + "artifacts": { + "$ref": "#/components/schemas/AcquiredArtifacts" + }, + "identify_result": { + "$ref": "#/components/schemas/PackageIdentifyResult" + }, + "l1_result": { + "anyOf": [ + { + "$ref": "#/components/schemas/L1InvestigationResult" + }, + { + "type": "null" + } + ], + "description": "Result from L1 Code Agent investigation" + }, + "l2_result": { + "anyOf": [ + { + "$ref": "#/components/schemas/L2BuildResult" + }, + { + "type": "null" + } + ], + "description": "Result from L2 Build Agent (optional)" + } + }, + "type": "object", + "title": "PackageCheckerContext", + "description": "Consolidates all checker-specific state on AgentMorpheusInfo." + }, + "PackageCheckerStatus": { + "type": "integer", + "enum": [0, 1, 2, 3, 4, 5], + "title": "PackageCheckerStatus", + "description": "Per-CVE status codes produced by the PackageIdentify phase." + }, + "PackageIdentifyResult": { + "properties": { + "affected_rpm_list": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Affected Rpm List", + "default": [] + }, + "fixed_rpm_list": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Fixed Rpm List", + "default": [] + }, + "is_target_package_affected": { + "$ref": "#/components/schemas/EnumIdentifyResult", + "default": "unknown" + }, + "is_target_package_fixed": { + "$ref": "#/components/schemas/EnumIdentifyResult", + "default": "unknown" + }, + "conclusion_reason": { + "type": "string", + "title": "Conclusion Reason", + "description": "Detailed explanation of why the package was determined to be vulnerable or not vulnerable", + "default": "" + } + }, + "type": "object", + "title": "PackageIdentifyResult", + "description": "Result of the PackageIdentify phase for a single CVE." + }, + "PackageState": { + "properties": { + "product_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Product Name" + }, + "fix_state": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Fix State" + }, + "package_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Package Name" + }, + "cpe": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cpe" + } + }, + "type": "object", + "title": "PackageState" + }, + "PipelineMode": { + "type": "string", + "enum": [ + "full_pipeline", + "rpm_package_checker" + ], + "title": "PipelineMode", + "description": "Controls which investigation path the pipeline takes after process_sbom.\nOrthogonal to AnalysisType (input format) -- any combination is valid." + }, + "ResponseIntermediateStep": { + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "parent_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Parent Id" + }, + "type": { + "type": "string", + "title": "Type", + "default": "markdown" + }, + "name": { + "type": "string", + "title": "Name" + }, + "payload": { + "type": "string", + "title": "Payload" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "id", + "name", + "payload" + ], + "title": "ResponseIntermediateStep", + "description": "ResponseSerializedStep is a data model that represents a serialized step in the NAT chat streaming API." + }, + "SBOMInfo": { + "properties": { + "packages": { + "items": { + "$ref": "#/components/schemas/SBOMPackage" + }, + "type": "array", + "title": "Packages" + } + }, + "type": "object", + "required": [ + "packages" + ], + "title": "SBOMInfo", + "description": "List of SBOMPackage objects representing the packages found in the input image." + }, + "SBOMPackage": { + "properties": { + "name": { + "type": "string", + "title": "Name" + }, + "version": { + "type": "string", + "title": "Version" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Path" + }, + "system": { + "type": "string", + "title": "System" + } + }, + "type": "object", + "required": [ + "name", + "version", + "system" + ], + "title": "SBOMPackage", + "description": "Information about a single package in the container image's Software Bill of Materials (SBOM)." + }, + "ScanInfoInput": { + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Type" + }, + "started_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Started At", + "description": "Scan start time as ISO-8601 with UTC offset (e.g. ...+00:00)." + }, + "completed_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Completed At", + "description": "Scan completion time as ISO-8601 with UTC offset (e.g. ...+00:00)." + }, + "vulns": { + "items": { + "$ref": "#/components/schemas/VulnInfo" + }, + "type": "array", + "minItems": 1, + "title": "Vulns" + } + }, + "type": "object", + "required": [ + "vulns" + ], + "title": "ScanInfoInput", + "description": "Information about a unique scan for a container image against a list of vulnerabilies." + }, + "SourceDocumentsInfo": { + "properties": { + "type": { + "type": "string", + "enum": [ + "code", + "doc" + ], + "title": "Type" + }, + "git_repo": { + "type": "string", + "minLength": 1, + "title": "Git Repo" + }, + "ref": { + "type": "string", + "minLength": 1, + "title": "Ref" + }, + "include": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Include", + "default": [ + "*.py", + "*.ipynb" + ] + }, + "exclude": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Exclude", + "default": [] + } + }, + "type": "object", + "required": [ + "type", + "git_repo", + "ref" + ], + "title": "SourceDocumentsInfo", + "description": "Information about the source documents for the container image.\n\n- type: document type.\n- git_repo: git repo URL where the source documents can be cloned.\n- ref: git reference, such as tag/branch/commit_id\n- include: file extensions to include when indexing the source documents.\n- exclude: file extensions to exclude when indexing the source documents." + }, + "TargetPackage": { + "properties": { + "name": { + "type": "string", + "title": "Name" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version" + }, + "release": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Release" + }, + "arch": { + "type": "string", + "title": "Arch", + "default": "x86_64" + } + }, + "type": "object", + "required": [ + "name" + ], + "title": "TargetPackage", + "description": "A package to investigate." + }, + "TextContent": { + "properties": { + "type": { + "type": "string", + "const": "text", + "title": "Type", + "default": "text" + }, + "text": { + "type": "string", + "title": "Text", + "default": "default" + } + }, + "additionalProperties": false, + "type": "object", + "title": "TextContent" + }, + "Usage": { + "properties": { + "prompt_tokens": { + "type": "integer", + "title": "Prompt Tokens" + }, + "completion_tokens": { + "type": "integer", + "title": "Completion Tokens" + }, + "total_tokens": { + "type": "integer", + "title": "Total Tokens" + } + }, + "type": "object", + "required": [ + "prompt_tokens", + "completion_tokens", + "total_tokens" + ], + "title": "Usage" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "type": "array", + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + } + }, + "type": "object", + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError" + }, + "VdbPaths": { + "properties": { + "code_vdb_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Code Vdb Path" + }, + "doc_vdb_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Doc Vdb Path" + }, + "code_index_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Code Index Path" + } + }, + "type": "object", + "title": "VdbPaths", + "description": "Paths to where the generated VDBs are stored." + }, + "VulnInfo": { + "properties": { + "vuln_id": { + "type": "string", + "title": "Vuln Id" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "score": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Score" + }, + "severity": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Severity" + }, + "published_date": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Published Date" + }, + "last_modified_date": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Last Modified Date" + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Url" + }, + "feed_group": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feed Group" + }, + "package": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Package" + }, + "package_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Package Version" + }, + "package_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Package Name" + }, + "package_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Package Type" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "vuln_id" + ], + "title": "VulnInfo", + "description": "Information about a vulnerability." + }, + "VulnerabilityIntel": { + "properties": { + "affected_files": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Affected Files", + "description": "Source file paths likely to contain vulnerable code" + }, + "vulnerable_functions": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Vulnerable Functions", + "description": "Function names that contain or handle the vulnerability" + }, + "vulnerable_variables": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Vulnerable Variables", + "description": "Variable names involved in the vulnerability" + }, + "vulnerable_patterns": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Vulnerable Patterns", + "description": "Code patterns/snippets indicating vulnerable code (from - lines)" + }, + "fix_patterns": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Fix Patterns", + "description": "Code patterns/snippets indicating fixed code (from + lines)" + }, + "root_cause": { + "type": "string", + "title": "Root Cause", + "description": "Technical explanation of why the code is vulnerable", + "default": "" + }, + "vulnerability_type": { + "type": "string", + "title": "Vulnerability Type", + "description": "Category: buffer_overflow, integer_overflow, use_after_free, null_deref, etc.", + "default": "" + }, + "search_keywords": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Search Keywords", + "description": "Recommended grep patterns ordered by specificity (most specific first)" + }, + "affected_architectures": { + "type": "string", + "enum": [ + "32-bit", + "64-bit", + "both" + ], + "title": "Affected Architectures", + "description": "Which CPU architectures are affected: 32-bit only, 64-bit only, or both (default)", + "default": "both" + }, + "is_downstream_patch_available": { + "type": "boolean", + "title": "Is Downstream Patch Available", + "description": "True if a CVE-specific patch file exists in the downstream package", + "default": false + }, + "is_patch_applied_in_build": { + "type": "boolean", + "title": "Is Patch Applied In Build", + "description": "True if the patch was confirmed applied in build logs", + "default": false + }, + "patch_file_name": { + "type": "string", + "title": "Patch File Name", + "description": "Name of the CVE-specific patch file (if available)", + "default": "" + } + }, + "type": "object", + "title": "VulnerabilityIntel", + "description": "Structured intelligence extracted from CVE advisories and patches.\n\nUsed to provide grep-ready patterns and context for L1 agent source searches." + }, + "VulnerableDependencies": { + "properties": { + "vuln_id": { + "type": "string", + "title": "Vuln Id" + }, + "vuln_package_intel_sources": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Vuln Package Intel Sources" + }, + "vulnerable_sbom_packages": { + "items": { + "$ref": "#/components/schemas/VulnerableSBOMPackage" + }, + "type": "array", + "title": "Vulnerable Sbom Packages" + } + }, + "type": "object", + "required": [ + "vuln_id", + "vuln_package_intel_sources", + "vulnerable_sbom_packages" + ], + "title": "VulnerableDependencies", + "description": "Information about the vulnerable SBOM packages associated with the vuln_id.\n\n- vuln_id: vulnerability ID (e.g. CVE ID, GHSA ID) associated with the vulnerable package list.\n- vuln_package_intel_sources: list of sources (e.g. \"ghsa\", \"nvd\", \"ubuntu\", \"rhsa\") that provided\n the vulnerable package/version intel for the vuln_id.\n- vulnerable_sbom_packages: list of VulnerableSBOMPackage objects, representing the SBOM packages that are\n vulnerable for a given vuln_id." + }, + "VulnerableSBOMPackage": { + "properties": { + "name": { + "type": "string", + "title": "Name" + }, + "version": { + "type": "string", + "title": "Version" + }, + "vulnerable_dependency_package": { + "$ref": "#/components/schemas/DependencyPackage" } + }, + "type": "object", + "required": [ + "name", + "version", + "vulnerable_dependency_package" + ], + "title": "VulnerableSBOMPackage", + "description": "Information about a vulnerable SBOM package and its related vulnerable dependency package.\n\n- name: SBOM package name\n- version: SBOM package version\n- vulnerable_dependency_package: DependencyPackage object with info about the vulnerable dependency package.\n If an SBOM package itself is vulnerable, the vulnerable_dependency_package.relation will be \"SELF\".\n Otherwise, if it is vulnerable due to its dependency, the vulnerable_dependency_package.relation will be either\n \"DIRECT\" or \"INDIRECT\"." } + } } -} \ No newline at end of file + } \ No newline at end of file diff --git a/src/vuln_analysis/configs/vex/csaf/v2.0/csaf_json_schema.json b/src/vuln_analysis/configs/vex/csaf/v2.0/csaf_json_schema.json new file mode 100644 index 000000000..93ff152a2 --- /dev/null +++ b/src/vuln_analysis/configs/vex/csaf/v2.0/csaf_json_schema.json @@ -0,0 +1,1414 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://docs.oasis-open.org/csaf/csaf/v2.0/csaf_json_schema.json", + "title": "Common Security Advisory Framework", + "description": "Representation of security advisory information as a JSON document.", + "type": "object", + "$defs": { + "acknowledgments_t": { + "title": "List of acknowledgments", + "description": "Contains a list of acknowledgment elements.", + "type": "array", + "minItems": 1, + "items": { + "title": "Acknowledgment", + "description": "Acknowledges contributions by describing those that contributed.", + "type": "object", + "minProperties": 1, + "properties": { + "names": { + "title": "List of acknowledged names", + "description": "Contains the names of contributors being recognized.", + "type": "array", + "minItems": 1, + "items": { + "title": "Name of the contributor", + "description": "Contains the name of a single contributor being recognized.", + "type": "string", + "minLength": 1, + "examples": [ + "Albert Einstein", + "Johann Sebastian Bach" + ] + } + }, + "organization": { + "title": "Contributing organization", + "description": "Contains the name of a contributing organization being recognized.", + "type": "string", + "minLength": 1, + "examples": [ + "CISA", + "Google Project Zero", + "Talos" + ] + }, + "summary": { + "title": "Summary of the acknowledgment", + "description": "SHOULD represent any contextual details the document producers wish to make known about the acknowledgment or acknowledged parties.", + "type": "string", + "minLength": 1, + "examples": [ + "First analysis of Coordinated Multi-Stream Attack (CMSA)" + ] + }, + "urls": { + "title": "List of URLs", + "description": "Specifies a list of URLs or location of the reference to be acknowledged.", + "type": "array", + "minItems": 1, + "items": { + "title": "URL of acknowledgment", + "description": "Contains the URL or location of the reference to be acknowledged.", + "type": "string", + "format": "uri" + } + } + } + } + }, + "branches_t": { + "title": "List of branches", + "description": "Contains branch elements as children of the current element.", + "type": "array", + "minItems": 1, + "items": { + "title": "Branch", + "description": "Is a part of the hierarchical structure of the product tree.", + "type": "object", + "maxProperties": 3, + "minProperties": 3, + "required": [ + "category", + "name" + ], + "properties": { + "branches": { + "$ref": "#/$defs/branches_t" + }, + "category": { + "title": "Category of the branch", + "description": "Describes the characteristics of the labeled branch.", + "type": "string", + "enum": [ + "architecture", + "host_name", + "language", + "legacy", + "patch_level", + "product_family", + "product_name", + "product_version", + "product_version_range", + "service_pack", + "specification", + "vendor" + ] + }, + "name": { + "title": "Name of the branch", + "description": "Contains the canonical descriptor or 'friendly name' of the branch.", + "type": "string", + "minLength": 1, + "examples": [ + "10", + "365", + "Microsoft", + "Office", + "PCS 7", + "SIMATIC", + "Siemens", + "Windows" + ] + }, + "product": { + "$ref": "#/$defs/full_product_name_t" + } + } + } + }, + "full_product_name_t": { + "title": "Full product name", + "description": "Specifies information about the product and assigns the product_id.", + "type": "object", + "required": [ + "name", + "product_id" + ], + "properties": { + "name": { + "title": "Textual description of the product", + "description": "The value should be the product’s full canonical name, including version number and other attributes, as it would be used in a human-friendly document.", + "type": "string", + "minLength": 1, + "examples": [ + "Cisco AnyConnect Secure Mobility Client 2.3.185", + "Microsoft Host Integration Server 2006 Service Pack 1" + ] + }, + "product_id": { + "$ref": "#/$defs/product_id_t" + }, + "product_identification_helper": { + "title": "Helper to identify the product", + "description": "Provides at least one method which aids in identifying the product in an asset database.", + "type": "object", + "minProperties": 1, + "properties": { + "cpe": { + "title": "Common Platform Enumeration representation", + "description": "The Common Platform Enumeration (CPE) attribute refers to a method for naming platforms external to this specification.", + "type": "string", + "pattern": "^(cpe:2\\.3:[aho\\*\\-](:(((\\?*|\\*?)([a-zA-Z0-9\\-\\._]|(\\\\[\\\\\\*\\?!\"#\\$%&'\\(\\)\\+,/:;<=>@\\[\\]\\^`\\{\\|\\}~]))+(\\?*|\\*?))|[\\*\\-])){5}(:(([a-zA-Z]{2,3}(-([a-zA-Z]{2}|[0-9]{3}))?)|[\\*\\-]))(:(((\\?*|\\*?)([a-zA-Z0-9\\-\\._]|(\\\\[\\\\\\*\\?!\"#\\$%&'\\(\\)\\+,/:;<=>@\\[\\]\\^`\\{\\|\\}~]))+(\\?*|\\*?))|[\\*\\-])){4})|([c][pP][eE]:/[AHOaho]?(:[A-Za-z0-9\\._\\-~%]*){0,6})$", + "minLength": 5 + }, + "hashes": { + "title": "List of hashes", + "description": "Contains a list of cryptographic hashes usable to identify files.", + "type": "array", + "minItems": 1, + "items": { + "title": "Cryptographic hashes", + "description": "Contains all information to identify a file based on its cryptographic hash values.", + "type": "object", + "required": [ + "file_hashes", + "filename" + ], + "properties": { + "file_hashes": { + "title": "List of file hashes", + "description": "Contains a list of cryptographic hashes for this file.", + "type": "array", + "minItems": 1, + "items": { + "title": "File hash", + "description": "Contains one hash value and algorithm of the file to be identified.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "title": "Algorithm of the cryptographic hash", + "description": "Contains the name of the cryptographic hash algorithm used to calculate the value.", + "type": "string", + "default": "sha256", + "minLength": 1, + "examples": [ + "blake2b512", + "sha256", + "sha3-512", + "sha384", + "sha512" + ] + }, + "value": { + "title": "Value of the cryptographic hash", + "description": "Contains the cryptographic hash value in hexadecimal representation.", + "type": "string", + "pattern": "^[0-9a-fA-F]{32,}$", + "minLength": 32, + "examples": [ + "37df33cb7464da5c7f077f4d56a32bc84987ec1d85b234537c1c1a4d4fc8d09dc29e2e762cb5203677bf849a2855a0283710f1f5fe1d6ce8d5ac85c645d0fcb3", + "4775203615d9534a8bfca96a93dc8b461a489f69124a130d786b42204f3341cc", + "9ea4c8200113d49d26505da0e02e2f49055dc078d1ad7a419b32e291c7afebbb84badfbd46dec42883bea0b2a1fa697c" + ] + } + } + } + }, + "filename": { + "title": "Filename", + "description": "Contains the name of the file which is identified by the hash values.", + "type": "string", + "minLength": 1, + "examples": [ + "WINWORD.EXE", + "msotadddin.dll", + "sudoers.so" + ] + } + } + } + }, + "model_numbers": { + "title": "List of models", + "description": "Contains a list of full or abbreviated (partial) model numbers.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "title": "Model number", + "description": "Contains a full or abbreviated (partial) model number of the component to identify.", + "type": "string", + "minLength": 1 + } + }, + "purl": { + "title": "package URL representation", + "description": "The package URL (purl) attribute refers to a method for reliably identifying and locating software packages external to this specification.", + "type": "string", + "format": "uri", + "pattern": "^pkg:[A-Za-z\\.\\-\\+][A-Za-z0-9\\.\\-\\+]*/.+", + "minLength": 7 + }, + "sbom_urls": { + "title": "List of SBOM URLs", + "description": "Contains a list of URLs where SBOMs for this product can be retrieved.", + "type": "array", + "minItems": 1, + "items": { + "title": "SBOM URL", + "description": "Contains a URL of one SBOM for this product.", + "type": "string", + "format": "uri" + } + }, + "serial_numbers": { + "title": "List of serial numbers", + "description": "Contains a list of full or abbreviated (partial) serial numbers.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "title": "Serial number", + "description": "Contains a full or abbreviated (partial) serial number of the component to identify.", + "type": "string", + "minLength": 1 + } + }, + "skus": { + "title": "List of stock keeping units", + "description": "Contains a list of full or abbreviated (partial) stock keeping units.", + "type": "array", + "minItems": 1, + "items": { + "title": "Stock keeping unit", + "description": "Contains a full or abbreviated (partial) stock keeping unit (SKU) which is used in the ordering process to identify the component.", + "type": "string", + "minLength": 1 + } + }, + "x_generic_uris": { + "title": "List of generic URIs", + "description": "Contains a list of identifiers which are either vendor-specific or derived from a standard not yet supported.", + "type": "array", + "minItems": 1, + "items": { + "title": "Generic URI", + "description": "Provides a generic extension point for any identifier which is either vendor-specific or derived from a standard not yet supported.", + "type": "object", + "required": [ + "namespace", + "uri" + ], + "properties": { + "namespace": { + "title": "Namespace of the generic URI", + "description": "Refers to a URL which provides the name and knowledge about the specification used or is the namespace in which these values are valid.", + "type": "string", + "format": "uri" + }, + "uri": { + "title": "URI", + "description": "Contains the identifier itself.", + "type": "string", + "format": "uri" + } + } + } + } + } + } + } + }, + "lang_t": { + "title": "Language type", + "description": "Identifies a language, corresponding to IETF BCP 47 / RFC 5646. See IETF language registry: https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry", + "type": "string", + "pattern": "^(([A-Za-z]{2,3}(-[A-Za-z]{3}(-[A-Za-z]{3}){0,2})?|[A-Za-z]{4,8})(-[A-Za-z]{4})?(-([A-Za-z]{2}|[0-9]{3}))?(-([A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(-[A-WY-Za-wy-z0-9](-[A-Za-z0-9]{2,8})+)*(-[Xx](-[A-Za-z0-9]{1,8})+)?|[Xx](-[A-Za-z0-9]{1,8})+|[Ii]-[Dd][Ee][Ff][Aa][Uu][Ll][Tt]|[Ii]-[Mm][Ii][Nn][Gg][Oo])$", + "examples": [ + "de", + "en", + "fr", + "frc", + "jp" + ] + }, + "notes_t": { + "title": "List of notes", + "description": "Contains notes which are specific to the current context.", + "type": "array", + "minItems": 1, + "items": { + "title": "Note", + "description": "Is a place to put all manner of text blobs related to the current context.", + "type": "object", + "required": [ + "category", + "text" + ], + "properties": { + "audience": { + "title": "Audience of note", + "description": "Indicates who is intended to read it.", + "type": "string", + "minLength": 1, + "examples": [ + "all", + "executives", + "operational management and system administrators", + "safety engineers" + ] + }, + "category": { + "title": "Note category", + "description": "Contains the information of what kind of note this is.", + "type": "string", + "enum": [ + "description", + "details", + "faq", + "general", + "legal_disclaimer", + "other", + "summary" + ] + }, + "text": { + "title": "Note content", + "description": "Holds the content of the note. Content varies depending on type.", + "type": "string", + "minLength": 1 + }, + "title": { + "title": "Title of note", + "description": "Provides a concise description of what is contained in the text of the note.", + "type": "string", + "minLength": 1, + "examples": [ + "Details", + "Executive summary", + "Technical summary", + "Impact on safety systems" + ] + } + } + } + }, + "product_group_id_t": { + "title": "Reference token for product group instance", + "description": "Token required to identify a group of products so that it can be referred to from other parts in the document. There is no predefined or required format for the product_group_id as long as it uniquely identifies a group in the context of the current document.", + "type": "string", + "minLength": 1, + "examples": [ + "CSAFGID-0001", + "CSAFGID-0002", + "CSAFGID-0020" + ] + }, + "product_groups_t": { + "title": "List of product_group_ids", + "description": "Specifies a list of product_group_ids to give context to the parent item.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/product_group_id_t" + } + }, + "product_id_t": { + "title": "Reference token for product instance", + "description": "Token required to identify a full_product_name so that it can be referred to from other parts in the document. There is no predefined or required format for the product_id as long as it uniquely identifies a product in the context of the current document.", + "type": "string", + "minLength": 1, + "examples": [ + "CSAFPID-0004", + "CSAFPID-0008" + ] + }, + "products_t": { + "title": "List of product_ids", + "description": "Specifies a list of product_ids to give context to the parent item.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/product_id_t" + } + }, + "references_t": { + "title": "List of references", + "description": "Holds a list of references.", + "type": "array", + "minItems": 1, + "items": { + "title": "Reference", + "description": "Holds any reference to conferences, papers, advisories, and other resources that are related and considered related to either a surrounding part of or the entire document and to be of value to the document consumer.", + "type": "object", + "required": [ + "summary", + "url" + ], + "properties": { + "category": { + "title": "Category of reference", + "description": "Indicates whether the reference points to the same document or vulnerability in focus (depending on scope) or to an external resource.", + "type": "string", + "default": "external", + "enum": [ + "external", + "self" + ] + }, + "summary": { + "title": "Summary of the reference", + "description": "Indicates what this reference refers to.", + "type": "string", + "minLength": 1 + }, + "url": { + "title": "URL of reference", + "description": "Provides the URL for the reference.", + "type": "string", + "format": "uri" + } + } + } + }, + "version_t": { + "title": "Version", + "description": "Specifies a version string to denote clearly the evolution of the content of the document. Format must be either integer or semantic versioning.", + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$|^((0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?)$", + "examples": [ + "1", + "4", + "0.9.0", + "1.4.3", + "2.40.0+21AF26D3" + ] + } + }, + "required": [ + "document" + ], + "properties": { + "document": { + "title": "Document level meta-data", + "description": "Captures the meta-data about this document describing a particular set of security advisories.", + "type": "object", + "required": [ + "category", + "csaf_version", + "publisher", + "title", + "tracking" + ], + "properties": { + "acknowledgments": { + "title": "Document acknowledgments", + "description": "Contains a list of acknowledgment elements associated with the whole document.", + "$ref": "#/$defs/acknowledgments_t" + }, + "aggregate_severity": { + "title": "Aggregate severity", + "description": "Is a vehicle that is provided by the document producer to convey the urgency and criticality with which the one or more vulnerabilities reported should be addressed. It is a document-level metric and applied to the document as a whole — not any specific vulnerability. The range of values in this field is defined according to the document producer's policies and procedures.", + "type": "object", + "required": [ + "text" + ], + "properties": { + "namespace": { + "title": "Namespace of aggregate severity", + "description": "Points to the namespace so referenced.", + "type": "string", + "format": "uri" + }, + "text": { + "title": "Text of aggregate severity", + "description": "Provides a severity which is independent of - and in addition to - any other standard metric for determining the impact or severity of a given vulnerability (such as CVSS).", + "type": "string", + "minLength": 1, + "examples": [ + "Critical", + "Important", + "Moderate" + ] + } + } + }, + "category": { + "title": "Document category", + "description": "Defines a short canonical name, chosen by the document producer, which will inform the end user as to the category of document.", + "type": "string", + "pattern": "^[^\\s\\-_\\.](.*[^\\s\\-_\\.])?$", + "minLength": 1, + "examples": [ + "csaf_base", + "csaf_security_advisory", + "csaf_vex", + "Example Company Security Notice" + ] + }, + "csaf_version": { + "title": "CSAF version", + "description": "Gives the version of the CSAF specification which the document was generated for.", + "type": "string", + "enum": [ + "2.0" + ] + }, + "distribution": { + "title": "Rules for sharing document", + "description": "Describe any constraints on how this document might be shared.", + "type": "object", + "minProperties": 1, + "properties": { + "text": { + "title": "Textual description", + "description": "Provides a textual description of additional constraints.", + "type": "string", + "minLength": 1, + "examples": [ + "Copyright 2021, Example Company, All Rights Reserved.", + "Distribute freely.", + "Share only on a need-to-know-basis only." + ] + }, + "tlp": { + "title": "Traffic Light Protocol (TLP)", + "description": "Provides details about the TLP classification of the document.", + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "title": "Label of TLP", + "description": "Provides the TLP label of the document.", + "type": "string", + "enum": [ + "AMBER", + "GREEN", + "RED", + "WHITE" + ] + }, + "url": { + "title": "URL of TLP version", + "description": "Provides a URL where to find the textual description of the TLP version which is used in this document. Default is the URL to the definition by FIRST.", + "type": "string", + "default": "https://www.first.org/tlp/", + "format": "uri", + "examples": [ + "https://www.us-cert.gov/tlp", + "https://www.bsi.bund.de/SharedDocs/Downloads/DE/BSI/Kritis/Merkblatt_TLP.pdf" + ] + } + } + } + } + }, + "lang": { + "title": "Document language", + "description": "Identifies the language used by this document, corresponding to IETF BCP 47 / RFC 5646.", + "$ref": "#/$defs/lang_t" + }, + "notes": { + "title": "Document notes", + "description": "Holds notes associated with the whole document.", + "$ref": "#/$defs/notes_t" + }, + "publisher": { + "title": "Publisher", + "description": "Provides information about the publisher of the document.", + "type": "object", + "required": [ + "category", + "name", + "namespace" + ], + "properties": { + "category": { + "title": "Category of publisher", + "description": "Provides information about the category of publisher releasing the document.", + "type": "string", + "enum": [ + "coordinator", + "discoverer", + "other", + "translator", + "user", + "vendor" + ] + }, + "contact_details": { + "title": "Contact details", + "description": "Information on how to contact the publisher, possibly including details such as web sites, email addresses, phone numbers, and postal mail addresses.", + "type": "string", + "minLength": 1, + "examples": [ + "Example Company can be reached at contact_us@example.com, or via our website at https://www.example.com/contact." + ] + }, + "issuing_authority": { + "title": "Issuing authority", + "description": "Provides information about the authority of the issuing party to release the document, in particular, the party's constituency and responsibilities or other obligations.", + "type": "string", + "minLength": 1 + }, + "name": { + "title": "Name of publisher", + "description": "Contains the name of the issuing party.", + "type": "string", + "minLength": 1, + "examples": [ + "BSI", + "Cisco PSIRT", + "Siemens ProductCERT" + ] + }, + "namespace": { + "title": "Namespace of publisher", + "description": "Contains a URL which is under control of the issuing party and can be used as a globally unique identifier for that issuing party.", + "type": "string", + "format": "uri", + "examples": [ + "https://csaf.io", + "https://www.example.com" + ] + } + } + }, + "references": { + "title": "Document references", + "description": "Holds a list of references associated with the whole document.", + "$ref": "#/$defs/references_t" + }, + "source_lang": { + "title": "Source language", + "description": "If this copy of the document is a translation then the value of this property describes from which language this document was translated.", + "$ref": "#/$defs/lang_t" + }, + "title": { + "title": "Title of this document", + "description": "This SHOULD be a canonical name for the document, and sufficiently unique to distinguish it from similar documents.", + "type": "string", + "minLength": 1, + "examples": [ + "Cisco IPv6 Crafted Packet Denial of Service Vulnerability", + "Example Company Cross-Site-Scripting Vulnerability in Example Generator" + ] + }, + "tracking": { + "title": "Tracking", + "description": "Is a container designated to hold all management attributes necessary to track a CSAF document as a whole.", + "type": "object", + "required": [ + "current_release_date", + "id", + "initial_release_date", + "revision_history", + "status", + "version" + ], + "properties": { + "aliases": { + "title": "Aliases", + "description": "Contains a list of alternate names for the same document.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "title": "Alternate name", + "description": "Specifies a non-empty string that represents a distinct optional alternative ID used to refer to the document.", + "type": "string", + "minLength": 1, + "examples": [ + "CVE-2019-12345" + ] + } + }, + "current_release_date": { + "title": "Current release date", + "description": "The date when the current revision of this document was released", + "type": "string", + "format": "date-time" + }, + "generator": { + "title": "Document generator", + "description": "Is a container to hold all elements related to the generation of the document. These items will reference when the document was actually created, including the date it was generated and the entity that generated it.", + "type": "object", + "required": [ + "engine" + ], + "properties": { + "date": { + "title": "Date of document generation", + "description": "This SHOULD be the current date that the document was generated. Because documents are often generated internally by a document producer and exist for a nonzero amount of time before being released, this field MAY be different from the Initial Release Date and Current Release Date.", + "type": "string", + "format": "date-time" + }, + "engine": { + "title": "Engine of document generation", + "description": "Contains information about the engine that generated the CSAF document.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "title": "Engine name", + "description": "Represents the name of the engine that generated the CSAF document.", + "type": "string", + "minLength": 1, + "examples": [ + "Red Hat rhsa-to-cvrf", + "Secvisogram", + "TVCE" + ] + }, + "version": { + "title": "Engine version", + "description": "Contains the version of the engine that generated the CSAF document.", + "type": "string", + "minLength": 1, + "examples": [ + "0.6.0", + "1.0.0-beta+exp.sha.a1c44f85", + "2" + ] + } + } + } + } + }, + "id": { + "title": "Unique identifier for the document", + "description": "The ID is a simple label that provides for a wide range of numbering values, types, and schemes. Its value SHOULD be assigned and maintained by the original document issuing authority.", + "type": "string", + "pattern": "^[\\S](.*[\\S])?$", + "minLength": 1, + "examples": [ + "Example Company - 2019-YH3234", + "RHBA-2019:0024", + "cisco-sa-20190513-secureboot" + ] + }, + "initial_release_date": { + "title": "Initial release date", + "description": "The date when this document was first published.", + "type": "string", + "format": "date-time" + }, + "revision_history": { + "title": "Revision history", + "description": "Holds one revision item for each version of the CSAF document, including the initial one.", + "type": "array", + "minItems": 1, + "items": { + "title": "Revision", + "description": "Contains all the information elements required to track the evolution of a CSAF document.", + "type": "object", + "required": [ + "date", + "number", + "summary" + ], + "properties": { + "date": { + "title": "Date of the revision", + "description": "The date of the revision entry", + "type": "string", + "format": "date-time" + }, + "legacy_version": { + "title": "Legacy version of the revision", + "description": "Contains the version string used in an existing document with the same content.", + "type": "string", + "minLength": 1 + }, + "number": { + "$ref": "#/$defs/version_t" + }, + "summary": { + "title": "Summary of the revision", + "description": "Holds a single non-empty string representing a short description of the changes.", + "type": "string", + "minLength": 1, + "examples": [ + "Initial version." + ] + } + } + } + }, + "status": { + "title": "Document status", + "description": "Defines the draft status of the document.", + "type": "string", + "enum": [ + "draft", + "final", + "interim" + ] + }, + "version": { + "$ref": "#/$defs/version_t" + } + } + } + } + }, + "product_tree": { + "title": "Product tree", + "description": "Is a container for all fully qualified product names that can be referenced elsewhere in the document.", + "type": "object", + "minProperties": 1, + "properties": { + "branches": { + "$ref": "#/$defs/branches_t" + }, + "full_product_names": { + "title": "List of full product names", + "description": "Contains a list of full product names.", + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/full_product_name_t" + } + }, + "product_groups": { + "title": "List of product groups", + "description": "Contains a list of product groups.", + "type": "array", + "minItems": 1, + "items": { + "title": "Product group", + "description": "Defines a new logical group of products that can then be referred to in other parts of the document to address a group of products with a single identifier.", + "type": "object", + "required": [ + "group_id", + "product_ids" + ], + "properties": { + "group_id": { + "$ref": "#/$defs/product_group_id_t" + }, + "product_ids": { + "title": "List of Product IDs", + "description": "Lists the product_ids of those products which known as one group in the document.", + "type": "array", + "minItems": 2, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/product_id_t" + } + }, + "summary": { + "title": "Summary of the product group", + "description": "Gives a short, optional description of the group.", + "type": "string", + "minLength": 1, + "examples": [ + "Products supporting Modbus.", + "The x64 versions of the operating system." + ] + } + } + } + }, + "relationships": { + "title": "List of relationships", + "description": "Contains a list of relationships.", + "type": "array", + "minItems": 1, + "items": { + "title": "Relationship", + "description": "Establishes a link between two existing full_product_name_t elements, allowing the document producer to define a combination of two products that form a new full_product_name entry.", + "type": "object", + "required": [ + "category", + "full_product_name", + "product_reference", + "relates_to_product_reference" + ], + "properties": { + "category": { + "title": "Relationship category", + "description": "Defines the category of relationship for the referenced component.", + "type": "string", + "enum": [ + "default_component_of", + "external_component_of", + "installed_on", + "installed_with", + "optional_component_of" + ] + }, + "full_product_name": { + "$ref": "#/$defs/full_product_name_t" + }, + "product_reference": { + "title": "Product reference", + "description": "Holds a Product ID that refers to the Full Product Name element, which is referenced as the first element of the relationship.", + "$ref": "#/$defs/product_id_t" + }, + "relates_to_product_reference": { + "title": "Relates to product reference", + "description": "Holds a Product ID that refers to the Full Product Name element, which is referenced as the second element of the relationship.", + "$ref": "#/$defs/product_id_t" + } + } + } + } + } + }, + "vulnerabilities": { + "title": "Vulnerabilities", + "description": "Represents a list of all relevant vulnerability information items.", + "type": "array", + "minItems": 1, + "items": { + "title": "Vulnerability", + "description": "Is a container for the aggregation of all fields that are related to a single vulnerability in the document.", + "type": "object", + "minProperties": 1, + "properties": { + "acknowledgments": { + "title": "Vulnerability acknowledgments", + "description": "Contains a list of acknowledgment elements associated with this vulnerability item.", + "$ref": "#/$defs/acknowledgments_t" + }, + "cve": { + "title": "CVE", + "description": "Holds the MITRE standard Common Vulnerabilities and Exposures (CVE) tracking number for the vulnerability.", + "type": "string", + "pattern": "^CVE-[0-9]{4}-[0-9]{4,}$" + }, + "cwe": { + "title": "CWE", + "description": "Holds the MITRE standard Common Weakness Enumeration (CWE) for the weakness associated.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "title": "Weakness ID", + "description": "Holds the ID for the weakness associated.", + "type": "string", + "pattern": "^CWE-[1-9]\\d{0,5}$", + "examples": [ + "CWE-22", + "CWE-352", + "CWE-79" + ] + }, + "name": { + "title": "Weakness name", + "description": "Holds the full name of the weakness as given in the CWE specification.", + "type": "string", + "minLength": 1, + "examples": [ + "Cross-Site Request Forgery (CSRF)", + "Improper Limitation of a Pathname to a Restricted Directory ('Path Traversal')", + "Improper Neutralization of Input During Web Page Generation ('Cross-site Scripting')" + ] + } + } + }, + "discovery_date": { + "title": "Discovery date", + "description": "Holds the date and time the vulnerability was originally discovered.", + "type": "string", + "format": "date-time" + }, + "flags": { + "title": "List of flags", + "description": "Contains a list of machine readable flags.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "title": "Flag", + "description": "Contains product specific information in regard to this vulnerability as a single machine readable flag.", + "type": "object", + "required": [ + "label" + ], + "properties": { + "date": { + "title": "Date of the flag", + "description": "Contains the date when assessment was done or the flag was assigned.", + "type": "string", + "format": "date-time" + }, + "group_ids": { + "$ref": "#/$defs/product_groups_t" + }, + "label": { + "title": "Label of the flag", + "description": "Specifies the machine readable label.", + "type": "string", + "enum": [ + "component_not_present", + "inline_mitigations_already_exist", + "vulnerable_code_cannot_be_controlled_by_adversary", + "vulnerable_code_not_in_execute_path", + "vulnerable_code_not_present" + ] + }, + "product_ids": { + "$ref": "#/$defs/products_t" + } + } + } + }, + "ids": { + "title": "List of IDs", + "description": "Represents a list of unique labels or tracking IDs for the vulnerability (if such information exists).", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "title": "ID", + "description": "Contains a single unique label or tracking ID for the vulnerability.", + "type": "object", + "required": [ + "system_name", + "text" + ], + "properties": { + "system_name": { + "title": "System name", + "description": "Indicates the name of the vulnerability tracking or numbering system.", + "type": "string", + "minLength": 1, + "examples": [ + "Cisco Bug ID", + "GitHub Issue" + ] + }, + "text": { + "title": "Text", + "description": "Is unique label or tracking ID for the vulnerability (if such information exists).", + "type": "string", + "minLength": 1, + "examples": [ + "CSCso66472", + "oasis-tcs/csaf#210" + ] + } + } + } + }, + "involvements": { + "title": "List of involvements", + "description": "Contains a list of involvements.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "title": "Involvement", + "description": "Is a container, that allows the document producers to comment on the level of involvement (or engagement) of themselves or third parties in the vulnerability identification, scoping, and remediation process.", + "type": "object", + "required": [ + "party", + "status" + ], + "properties": { + "date": { + "title": "Date of involvement", + "description": "Holds the date and time of the involvement entry.", + "type": "string", + "format": "date-time" + }, + "party": { + "title": "Party category", + "description": "Defines the category of the involved party.", + "type": "string", + "enum": [ + "coordinator", + "discoverer", + "other", + "user", + "vendor" + ] + }, + "status": { + "title": "Party status", + "description": "Defines contact status of the involved party.", + "type": "string", + "enum": [ + "completed", + "contact_attempted", + "disputed", + "in_progress", + "not_contacted", + "open" + ] + }, + "summary": { + "title": "Summary of the involvement", + "description": "Contains additional context regarding what is going on.", + "type": "string", + "minLength": 1 + } + } + } + }, + "notes": { + "title": "Vulnerability notes", + "description": "Holds notes associated with this vulnerability item.", + "$ref": "#/$defs/notes_t" + }, + "product_status": { + "title": "Product status", + "description": "Contains different lists of product_ids which provide details on the status of the referenced product related to the current vulnerability. ", + "type": "object", + "minProperties": 1, + "properties": { + "first_affected": { + "title": "First affected", + "description": "These are the first versions of the releases known to be affected by the vulnerability.", + "$ref": "#/$defs/products_t" + }, + "first_fixed": { + "title": "First fixed", + "description": "These versions contain the first fix for the vulnerability but may not be the recommended fixed versions.", + "$ref": "#/$defs/products_t" + }, + "fixed": { + "title": "Fixed", + "description": "These versions contain a fix for the vulnerability but may not be the recommended fixed versions.", + "$ref": "#/$defs/products_t" + }, + "known_affected": { + "title": "Known affected", + "description": "These versions are known to be affected by the vulnerability.", + "$ref": "#/$defs/products_t" + }, + "known_not_affected": { + "title": "Known not affected", + "description": "These versions are known not to be affected by the vulnerability.", + "$ref": "#/$defs/products_t" + }, + "last_affected": { + "title": "Last affected", + "description": "These are the last versions in a release train known to be affected by the vulnerability. Subsequently released versions would contain a fix for the vulnerability.", + "$ref": "#/$defs/products_t" + }, + "recommended": { + "title": "Recommended", + "description": "These versions have a fix for the vulnerability and are the vendor-recommended versions for fixing the vulnerability.", + "$ref": "#/$defs/products_t" + }, + "under_investigation": { + "title": "Under investigation", + "description": "It is not known yet whether these versions are or are not affected by the vulnerability. However, it is still under investigation - the result will be provided in a later release of the document.", + "$ref": "#/$defs/products_t" + } + } + }, + "references": { + "title": "Vulnerability references", + "description": "Holds a list of references associated with this vulnerability item.", + "$ref": "#/$defs/references_t" + }, + "release_date": { + "title": "Release date", + "description": "Holds the date and time the vulnerability was originally released into the wild.", + "type": "string", + "format": "date-time" + }, + "remediations": { + "title": "List of remediations", + "description": "Contains a list of remediations.", + "type": "array", + "minItems": 1, + "items": { + "title": "Remediation", + "description": "Specifies details on how to handle (and presumably, fix) a vulnerability.", + "type": "object", + "required": [ + "category", + "details" + ], + "properties": { + "category": { + "title": "Category of the remediation", + "description": "Specifies the category which this remediation belongs to.", + "type": "string", + "enum": [ + "mitigation", + "no_fix_planned", + "none_available", + "vendor_fix", + "workaround" + ] + }, + "date": { + "title": "Date of the remediation", + "description": "Contains the date from which the remediation is available.", + "type": "string", + "format": "date-time" + }, + "details": { + "title": "Details of the remediation", + "description": "Contains a thorough human-readable discussion of the remediation.", + "type": "string", + "minLength": 1 + }, + "entitlements": { + "title": "List of entitlements", + "description": "Contains a list of entitlements.", + "type": "array", + "minItems": 1, + "items": { + "title": "Entitlement of the remediation", + "description": "Contains any possible vendor-defined constraints for obtaining fixed software or hardware that fully resolves the vulnerability.", + "type": "string", + "minLength": 1 + } + }, + "group_ids": { + "$ref": "#/$defs/product_groups_t" + }, + "product_ids": { + "$ref": "#/$defs/products_t" + }, + "restart_required": { + "title": "Restart required by remediation", + "description": "Provides information on category of restart is required by this remediation to become effective.", + "type": "object", + "required": [ + "category" + ], + "properties": { + "category": { + "title": "Category of restart", + "description": "Specifies what category of restart is required by this remediation to become effective.", + "type": "string", + "enum": [ + "connected", + "dependencies", + "machine", + "none", + "parent", + "service", + "system", + "vulnerable_component", + "zone" + ] + }, + "details": { + "title": "Additional restart information", + "description": "Provides additional information for the restart. This can include details on procedures, scope or impact.", + "type": "string", + "minLength": 1 + } + } + }, + "url": { + "title": "URL to the remediation", + "description": "Contains the URL where to obtain the remediation.", + "type": "string", + "format": "uri" + } + } + } + }, + "scores": { + "title": "List of scores", + "description": "Contains score objects for the current vulnerability.", + "type": "array", + "minItems": 1, + "items": { + "title": "Score", + "description": "Specifies information about (at least one) score of the vulnerability and for which products the given value applies.", + "type": "object", + "minProperties": 2, + "required": [ + "products" + ], + "properties": { + "cvss_v2": { + "$ref": "https://www.first.org/cvss/cvss-v2.0.json" + }, + "cvss_v3": { + "oneOf": [ + { + "$ref": "https://www.first.org/cvss/cvss-v3.0.json" + }, + { + "$ref": "https://www.first.org/cvss/cvss-v3.1.json" + } + ] + }, + "products": { + "$ref": "#/$defs/products_t" + } + } + } + }, + "threats": { + "title": "List of threats", + "description": "Contains information about a vulnerability that can change with time.", + "type": "array", + "minItems": 1, + "items": { + "title": "Threat", + "description": "Contains the vulnerability kinetic information. This information can change as the vulnerability ages and new information becomes available.", + "type": "object", + "required": [ + "category", + "details" + ], + "properties": { + "category": { + "title": "Category of the threat", + "description": "Categorizes the threat according to the rules of the specification.", + "type": "string", + "enum": [ + "exploit_status", + "impact", + "target_set" + ] + }, + "date": { + "title": "Date of the threat", + "description": "Contains the date when the assessment was done or the threat appeared.", + "type": "string", + "format": "date-time" + }, + "details": { + "title": "Details of the threat", + "description": "Represents a thorough human-readable discussion of the threat.", + "type": "string", + "minLength": 1 + }, + "group_ids": { + "$ref": "#/$defs/product_groups_t" + }, + "product_ids": { + "$ref": "#/$defs/products_t" + } + } + } + }, + "title": { + "title": "Title", + "description": "Gives the document producer the ability to apply a canonical name or title to the vulnerability.", + "type": "string", + "minLength": 1 + } + } + } + } + } +} diff --git a/src/vuln_analysis/data/input_messages/morpheus:23.11-runtime.json b/src/vuln_analysis/data/input_messages/morpheus:23.11-runtime.json index 2805c6047..9ac547200 100644 --- a/src/vuln_analysis/data/input_messages/morpheus:23.11-runtime.json +++ b/src/vuln_analysis/data/input_messages/morpheus:23.11-runtime.json @@ -2,6 +2,7 @@ "image": { "name": "nvcr.io/nvidia/morpheus/morpheus", "tag": "23.11-runtime", + "analysis_type": "image", "source_info": [ { "type": "code", @@ -58,6 +59,9 @@ }, { "vuln_id": "GHSA-3ww4-gg4f-jr7f" + }, + { + "vuln_id": "CVE-2023-51767" } ] } diff --git a/src/vuln_analysis/data_models/job.py b/src/vuln_analysis/data_models/job.py new file mode 100644 index 000000000..4380ed708 --- /dev/null +++ b/src/vuln_analysis/data_models/job.py @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +from typing import Any + +from pydantic import BaseModel, Field, RootModel + + +class LocalDateTime(RootModel[datetime]): + root: datetime = Field(..., examples=['2022-03-10T12:15:50']) + + +class Job(BaseModel): + job_id: str | None = None + execution_start_timestamp: LocalDateTime | None = None + duration_in_seconds: str | None = None + cve: str | None = Field(None, pattern=r'CVE-\d{4}-\d{4,7}') + app_language: str | None = Field(None, pattern=r'go|python|c|javascript|java|all|\s*') + component: str | None = None + component_version: str | None = None + agent_git_commit: str | None = None + agent_git_tag: str | None = None + agent_config_b64: str | None = None + concrete_intel_sources_b64: str | None = None + executed_from_background_process: bool | None = None + batch_id: str | None = None + success_status: bool | None = None + env_vars: str = Field(..., description='comma delimited key=value pairs') + job_output: dict[str, Any] | None = None diff --git a/src/vuln_analysis/data_models/output.py b/src/vuln_analysis/data_models/output.py index 7d8481329..b182c9fa2 100644 --- a/src/vuln_analysis/data_models/output.py +++ b/src/vuln_analysis/data_models/output.py @@ -16,9 +16,20 @@ import typing from pydantic import BaseModel +from pydantic import Field from pydantic import model_validator -from .input import AgentMorpheusEngineInput +from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput + + +class FailureReport(BaseModel): + """ + Lightweight error payload sent to the failure endpoint when processing fails + (e.g. code indexing / repository cloning). + """ + scan_id: str = Field(serialization_alias="scanId") + error_type: str = Field(serialization_alias="errorType") + error_message: str = Field(serialization_alias="errorMessage") class AgentIntermediateStep(BaseModel): @@ -39,7 +50,15 @@ class ChecklistItemOutput(BaseModel): response: str intermediate_steps: list[AgentIntermediateStep] | None = None - +class CVSSOutput(BaseModel): + """ + CVSS (Common Vulnerability Scoring System) representing the severity of a vulnerability in reference to an image. + - vector_string: The CVSS vector string that encodes the metric values used to calculate the score. + - score: The calculated CVSS base score representing the severity of the vulnerability in the given image. + """ + vector_string: str + score: str + class JustificationOutput(BaseModel): """ Final justification for the vulnerability. @@ -63,11 +82,25 @@ class AgentMorpheusEngineOutput(BaseModel): - checklist: a list of ChecklistItemOutput objects, each containing an input and a response from the LLM agent. - summary: a short summary of the checklist inputs and responses, generated by an LLM. - justification: a JustificationOutput object containing details of the model's justification decision. + - cvss: a CVSSOutput object containing the CVSS score and vector string for the vulnerability. """ vuln_id: str checklist: list[ChecklistItemOutput] summary: str justification: JustificationOutput + intel_score: int + cvss: CVSSOutput | None + details: str | None = None + + +class OutputPayload(BaseModel): + """ + Wrapper for final pipeline results. + - analysis: per-vulnerability analysis results + - vex: the vulnerability exploitability exchange document JSON + """ + analysis: list[AgentMorpheusEngineOutput] + vex: dict[str, typing.Any] | None class AgentMorpheusOutput(AgentMorpheusEngineInput): @@ -75,7 +108,7 @@ class AgentMorpheusOutput(AgentMorpheusEngineInput): The final output of the Agent Morpheus pipeline. Contains all fields in the AgentMorpheusEngineInput, plus the AgentMorpheusEngineOuput for each input vulnerability. """ - output: list[AgentMorpheusEngineOutput] + output: OutputPayload @model_validator(mode="before") @classmethod diff --git a/src/vuln_analysis/data_models/plugin.py b/src/vuln_analysis/data_models/plugin.py new file mode 100644 index 000000000..108e6c980 --- /dev/null +++ b/src/vuln_analysis/data_models/plugin.py @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import typing +from abc import ABC, abstractmethod +from pydoc import locate + +import aiohttp + +from exploit_iq_commons.data_models.common import TypedBaseModel +from exploit_iq_commons.data_models.cve_intel import IntelPluginData + + +_T = typing.TypeVar('_T', bound='PluginSchema') + + +class PluginConfig(TypedBaseModel[typing.Literal["plugin"]]): + plugin_name: str + + plugin_config: dict[str, typing.Any] = {} + + +class PluginSchema(ABC): + + @classmethod + def locate(cls: type[_T], plugin_name: str, **kwargs) -> _T: + '''Locate plugin''' + pluginClass: type | None = locate(plugin_name) + + if not pluginClass: + raise ValueError(f"Plugin not found: {plugin_name}") + if not issubclass(pluginClass, cls): + raise ValueError(f"{pluginClass} object must be a subclass of {cls}") + + return pluginClass(**kwargs) + + +class InputPluginSchema(PluginSchema): + + @abstractmethod + def build_input(self): + # add the plugin specific input building logic here + pass + + +class OutputPluginSchema(PluginSchema): + + @abstractmethod + def build_output(self, config): + # add the plugin specific output building logic here + pass + + +class IntelPluginSchema(PluginSchema): + + def __init__(self, session: aiohttp.ClientSession, config: dict = {}): + self._session = session + self._config = config + + @abstractmethod + async def retrieve(self, vuln_id: str) -> IntelPluginData: + """ + Asynchronously retrieve the details of a given CVE Id + """ + pass diff --git a/src/vuln_analysis/data_models/plugins/intel_plugin.py b/src/vuln_analysis/data_models/plugins/intel_plugin.py new file mode 100644 index 000000000..ea122d808 --- /dev/null +++ b/src/vuln_analysis/data_models/plugins/intel_plugin.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests +from pydantic import BaseModel + +from exploit_iq_commons.data_models.cve_intel import IntelPluginData +from vuln_analysis.data_models.plugin import IntelPluginSchema + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) + + +class SimpleHttpIntelPluginConfig(BaseModel): + source: str + endpoint: str + api_key: str | None = None + token_path: str | None = None + verify_path: str | None = None + + +class SimpleHttpIntelPlugin(IntelPluginSchema): + """ + This is a simple Http intel retriever that sends an http GET to the configured + endpoint and returns the text content as a result. + The configuration expects an endpoint with the `{vuln_id}` template parameter + that will be replaced by the current vuln_id. e.g. https://example.com/{vuln_id}/details + Additionally if the `api_key` is provided, it will be sent as an HTTP Authorization Header + with value: Bearer + """ + + def __init__(self, session, config): + self._session = session + self._config = SimpleHttpIntelPluginConfig(**config) + + async def retrieve(self, vuln_id: str) -> IntelPluginData: + endpoint = self._config.endpoint.format(vuln_id=vuln_id) + try: + headers = {"Accept": "text/plain"} + if self._config.api_key: + headers["Authorization"] = f"Bearer {self._config.api_key}" + elif self._config.token_path: + headers["Authorization"] = f"{await self.get_token_value()}" + response = requests.get( + endpoint, headers=headers, verify=await self.get_verify_path() + ) + response.raise_for_status() + if not response.ok: + logger.error( + f"Unable to request intel data from {endpoint}. Error: {response.status_code} - {response.reason}" + ) + else: + logger.debug(f"Successfully requested intel data to: {endpoint}") + return IntelPluginData(label=self._config.source, description=response.text) + except requests.exceptions.RequestException as e: + logger.error(f"Error fetching intel data from {endpoint}: {e}") + return IntelPluginData( + label=self._config.source, + description="No data available for this intel source", + ) + + async def get_verify_path(self) -> str | None: + if self._config.verify_path: + return self._config.verify_path + else: + return None + + async def get_token_value(self): + try: + with open(self._config.token_path, "r") as file: + return f"Bearer {file.read().strip()}" + except Exception as e: + logger.warning(f"Unable to read OAuth token: {e}") + return None diff --git a/src/vuln_analysis/data_models/state.py b/src/vuln_analysis/data_models/state.py index ac61bc7d4..9d147cbb6 100644 --- a/src/vuln_analysis/data_models/state.py +++ b/src/vuln_analysis/data_models/state.py @@ -17,7 +17,8 @@ from pydantic import BaseModel -from vuln_analysis.data_models.cve_intel import CveIntel +from exploit_iq_commons.data_models.cve_intel import CveIntel +from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput class AgentMorpheusEngineState(BaseModel): @@ -25,7 +26,16 @@ class AgentMorpheusEngineState(BaseModel): doc_vdb_path: str | None = None code_index_path: str | None = None cve_intel: list[CveIntel] + transitive_code_searcher: typing.Any | None = None + original_input: AgentMorpheusEngineInput | None = None checklist_plans: dict[str, list[str]] = {} checklist_results: dict[str, list[dict[str, typing.Any]]] = {} final_summaries: dict[str, str] = {} justifications: dict[str, dict[str, str]] = {} + poor_quality_intel_vul: dict[str, int] = {} + cvss_results: dict[str, dict[str, str]] = {} + vex: dict[str, typing.Any] | None = None + current_vuln_id: str | None = None + patch_results: dict[str, typing.Any] = {} + uber_jar_file_threshold: int = 600 + diff --git a/src/vuln_analysis/functions/agent_registry.py b/src/vuln_analysis/functions/agent_registry.py new file mode 100644 index 000000000..4c12f81a0 --- /dev/null +++ b/src/vuln_analysis/functions/agent_registry.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from vuln_analysis.functions.base_graph_agent import BaseGraphAgent + +_AGENT_REGISTRY: dict[str, type[BaseGraphAgent]] = {} + + +def register_agent(agent_type: str): + """Class decorator that registers a BaseGraphAgent subclass under the given type name.""" + def wrapper(cls): + _AGENT_REGISTRY[agent_type] = cls + return cls + return wrapper + + +def get_agent_class(agent_type: str) -> type[BaseGraphAgent]: + if agent_type not in _AGENT_REGISTRY: + raise KeyError(f"Unknown agent type '{agent_type}'. Registered: {list(_AGENT_REGISTRY.keys())}") + return _AGENT_REGISTRY[agent_type] + + +def get_all_agent_types() -> list[str]: + return list(_AGENT_REGISTRY.keys()) diff --git a/src/vuln_analysis/functions/base_graph_agent.py b/src/vuln_analysis/functions/base_graph_agent.py new file mode 100644 index 000000000..05ab11387 --- /dev/null +++ b/src/vuln_analysis/functions/base_graph_agent.py @@ -0,0 +1,601 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import uuid +from abc import ABC, abstractmethod + +from langchain_core.messages import SystemMessage, HumanMessage, AIMessage, RemoveMessage +from langgraph.graph import StateGraph, END, START +from langgraph.prebuilt import ToolNode + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from nat.builder.context import Context +from vuln_analysis.tools.tool_names import ToolNames +from vuln_analysis.functions.react_internals import ( + AgentState, + BaseRulesTracker, + Thought, + Observation, + CodeFindings, + PackageSelection, + _build_tool_arguments, + invoke_comprehension, + FORCED_FINISH_PROMPT, + COMPREHENSION_PROMPT, + MEMORY_UPDATE_PROMPT, +) +from vuln_analysis.runtime_context import ctx_state +from vuln_analysis.utils.token_utils import count_tokens, estimate_tokens, truncate_tool_output + +logger = LoggingFactory.get_agent_logger(__name__) +AGENT_TRACER = Context.get() + +_TOOL_AVAILABILITY = { + ToolNames.CODE_SEMANTIC_SEARCH: lambda config, state: state.code_vdb_path is not None, + ToolNames.DOCS_SEMANTIC_SEARCH: lambda config, state: state.doc_vdb_path is not None, + ToolNames.CODE_KEYWORD_SEARCH: lambda config, state: state.code_index_path is not None, + ToolNames.IMPORT_USAGE_ANALYZER: lambda config, state: state.code_index_path is not None, + ToolNames.CVE_WEB_SEARCH: lambda config, state: config.cve_web_search_enabled, + ToolNames.CALL_CHAIN_ANALYZER: lambda config, state: config.transitive_search_tool_enabled and state.code_index_path is not None, + ToolNames.FUNCTION_CALLER_FINDER: lambda config, state: config.transitive_search_tool_enabled and state.code_index_path is not None, + ToolNames.FUNCTION_LOCATOR: lambda config, state: config.transitive_search_tool_enabled and state.code_index_path is not None, +} + + +def _is_tool_available(tool_name, config, state): + check = _TOOL_AVAILABILITY.get(tool_name) + return check(config, state) if check else True + + +class BaseGraphAgent(ABC): + """Template for LangGraph-based CVE investigation agents. + + Subclasses must implement: + - pre_process_node: initialize agent state (prompts, package selection, etc.) + - get_tools: load and select which tools this agent uses + - create_rules_tracker: return the appropriate RulesTracker instance + + Shared graph nodes (thought, observation, forced_finish, should_continue) + and the graph wiring are provided by this base class. + """ + + def __init__(self, tools: list, llm, config): + self.tools = tools + self.config = config + self.thought_llm = llm.with_structured_output(Thought) + self.comprehension_llm = llm.with_structured_output(CodeFindings) + self.observation_llm = llm.with_structured_output(Observation) + self.package_filter_llm = llm.with_structured_output(PackageSelection) + + @property + def agent_type(self) -> str: + """Short identifier for tracing spans (e.g. 'reachability', 'cu').""" + return "base" + + @staticmethod + def _load_all_tools(builder, config) -> list: + from aiq.builder.framework_enum import LLMFrameworkEnum + return builder.get_tools(tool_names=config.tool_names, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + + _GO_STDLIB_ROOTS = frozenset({ + "archive", "bufio", "builtin", "bytes", "cmp", "compress", "container", + "context", "crypto", "database", "debug", "embed", "encoding", "errors", + "expvar", "flag", "fmt", "go", "hash", "html", "image", "index", "io", + "iter", "log", "maps", "math", "mime", "net", "os", "path", "plugin", + "reflect", "regexp", "runtime", "slices", "sort", "strconv", "strings", + "structs", "sync", "syscall", "testing", "text", "time", "unicode", + "unique", "unsafe", + }) + + _GO_VULN_DB_HINT_RE = re.compile(r"Vulnerable module \(Go vuln DB hint\): (\S+)") + _OBS_CONTEXT_MAX_TOKENS = 1200 + _OBS_CONTEXT_TRUNCATION_NOTE = "... (context truncated to fit token window)" + + @classmethod + def _find_go_stdlib_candidate(cls, ecosystem: str, candidate_packages: list[dict], + critical_context: list[str] | None = None) -> str | None: + """If any candidate is a Go standard library package, return it directly. + Go stdlib packages start with a known root (e.g. crypto/x509, net/http) + unlike third-party packages (github.com/..., golang.org/x/...). + Also checks critical_context for Go vuln DB hints (OSV enrichment).""" + if ecosystem != "go": + return None + for pkg in candidate_packages: + name = pkg.get("name", "") + if name and name.split("/")[0] in cls._GO_STDLIB_ROOTS: + return name + for ctx in critical_context or []: + m = cls._GO_VULN_DB_HINT_RE.search(ctx) + if m and m.group(1).split("/")[0] in cls._GO_STDLIB_ROOTS: + return m.group(1) + return None + + async def _select_package( + self, ecosystem: str, candidate_packages: list[dict], critical_context: list[str], + workflow_state, + ) -> tuple[list[str], str | None]: + """Select target package from candidates using LLM filter. + Returns (filtered_critical_context, selected_package). + """ + from vuln_analysis.functions.react_internals import build_package_filter_prompt, _find_image_matching_candidate + from vuln_analysis.utils.intel_utils import filter_context_to_package + + selected_package = None + if len(candidate_packages) > 1: + stdlib_match = self._find_go_stdlib_candidate(ecosystem, candidate_packages, critical_context) + if stdlib_match: + selected_package = stdlib_match + logger.info("Package filter selected Go stdlib '%s' directly (no LLM call needed, %d candidates)", + selected_package, len(candidate_packages)) + critical_context = filter_context_to_package(critical_context, selected_package, candidate_packages) + return critical_context, selected_package + + image_input = workflow_state.original_input.input.image + image_name = image_input.name + image_repo = image_input.source_info[0].git_repo if image_input.source_info else None + + matched = _find_image_matching_candidate(candidate_packages, image_name, image_repo) + if matched: + selected_package = matched + logger.info("Package filter matched '%s' from image/repo (no LLM call needed, %d candidates)", + selected_package, len(candidate_packages)) + else: + filter_prompt = build_package_filter_prompt( + ecosystem, candidate_packages, + image_name=image_name, image_repo=image_repo, + critical_context=critical_context, + ) + selection = await self.package_filter_llm.ainvoke([HumanMessage(content=filter_prompt)]) + selected_package = selection.selected_package + logger.info("Package filter selected '%s' from %d candidates (reason: %s)", + selected_package, len(candidate_packages), selection.reason) + critical_context = filter_context_to_package(critical_context, selected_package, candidate_packages) + elif len(candidate_packages) == 1: + selected_package = candidate_packages[0].get("name") + logger.info("Single candidate package: '%s'", selected_package) + critical_context = filter_context_to_package(critical_context, selected_package, candidate_packages) + return critical_context, selected_package + + # --- Template methods (subclasses MUST override) --- + + @abstractmethod + async def pre_process_node(self, state: AgentState) -> AgentState: + """Initialize agent state: select package, build prompts, set rules.""" + ... + + @staticmethod + @abstractmethod + def get_tools(builder, config, state) -> list: + """Load tools from the builder and select those this agent needs.""" + ... + + @staticmethod + @abstractmethod + def create_rules_tracker() -> BaseRulesTracker: + """Return a fresh RulesTracker instance for a single question.""" + ... + + # --- Optional hooks (subclasses MAY override) --- + + def build_comprehension_context(self, state: AgentState) -> str: + """Return the vulnerability context string for the comprehension prompt. + Override in subclasses to reduce context and prevent hallucination.""" + ctx_lines = state.get("critical_context", []) + return "\n".join(ctx_lines) if ctx_lines else "N/A" + + def sanitize_findings(self, findings: list[str], state: AgentState) -> list[str]: + """Replace hallucinated CVE IDs in comprehension findings. + Only CVE IDs present in the current investigation are kept.""" + workflow_state = ctx_state.get() + allowed_ids = {intel.vuln_id for intel in workflow_state.cve_intel} + sanitized = [] + for finding in findings: + def _replace(m): + return m.group(0) if m.group(0) in allowed_ids else "the investigated vulnerability" + sanitized.append(re.sub(r"CVE-\d{4}-\d+", _replace, finding)) + return sanitized + + def post_observation(self, state: AgentState, tool_used: str, + tool_output: str, tool_input_detail: str) -> dict: + """Per-agent post-processing after observation_node core logic. + Returns dict to merge into observation_node output. Default: empty.""" + return {} + + def should_truncate_tool_output(self, state: AgentState, tool_used: str) -> bool: + """Whether to apply Java-specific truncation to tool output. Default: False.""" + return False + + def check_finish_allowed(self, state: AgentState) -> tuple[bool, str]: + """Hook to block premature finish. Returns (allowed, error_message). + Override in subclasses to enforce rules before allowing the agent to finish.""" + return True, "" + + # --- Shared helpers --- + + def _prune_messages_to_fit(self, messages: list, keep_tail: int, step_num: int, caller: str) -> None: + """Remove oldest conversation messages to fit within the token limit. + + Preserves messages[0] (system prompt) and the last ``keep_tail`` messages. + Mutates the list in place. + """ + max_tokens = self.config.context_window_token_limit + total = sum( + count_tokens(m.content) for m in messages + if hasattr(m, "content") and isinstance(m.content, str) + ) + min_count = 1 + keep_tail + if total <= max_tokens or len(messages) <= min_count: + return + orig_len = len(messages) + prunable = messages[1:-keep_tail] + for msg in list(prunable): + if total <= max_tokens: + break + total -= count_tokens(msg.content) if hasattr(msg, "content") and isinstance(msg.content, str) else 0 + messages.remove(msg) + removed = orig_len - len(messages) + if removed: + logger.info( + "%s pruning: removed %d messages, estimated tokens now ~%d (limit %d) at step %d", + caller, removed, total, max_tokens, step_num, + ) + + # --- Shared concrete graph nodes --- + + async def thought_node(self, state: AgentState) -> AgentState: + step_num = state.get("step", 0) + with AGENT_TRACER.push_active_function(f"{self.agent_type}_thought", input_data=f"step:{step_num}") as span: + try: + active_prompt = state.get("runtime_prompt") + messages = [SystemMessage(content=active_prompt)] + state["messages"] + obs = state.get("observation", None) + crit_context = state.get("critical_context") + context_block = self._build_observation_context(obs, crit_context) + if context_block is None and obs is not None: + context_block = "KNOWLEDGE:\n- No prior knowledge.\nLATEST FINDINGS:\n- No recent findings." + if context_block: + messages.append(SystemMessage(content=context_block)) + + # Keep system prompt (messages[0]) and KNOWLEDGE block (messages[-1]). + # ToolMessages are prunable because observation_node already distilled + # their content into the KNOWLEDGE block via comprehension. + self._prune_messages_to_fit(messages, keep_tail=1, step_num=step_num, caller="thought_node") + + response: Thought = await self.thought_llm.ainvoke(messages) + + final_answer = "waiting for the agent to respond" + if response.mode == "finish": + finish_ok, finish_msg = self.check_finish_allowed(state) + if not finish_ok: + logger.info("%s finish blocked by rules at step %d: %s", self.agent_type, step_num, finish_msg) + span.set_output({"mode": "finish_blocked", "step": step_num + 1}) + blocked_ai = AIMessage(content=response.final_answer or response.thought or "I want to finish.") + return { + "messages": [blocked_ai, HumanMessage(content=finish_msg)], + "thought": None, + "step": step_num + 1, + "max_steps": self.config.max_iterations, + "output": "waiting for the agent to respond", + } + ai_message = AIMessage(content=response.final_answer) + final_answer = response.final_answer + elif response.actions is None: + logger.warning("%s LLM returned mode='act' but actions is None, forcing finish", self.agent_type) + ai_message = AIMessage(content=response.thought or "No actions provided, finishing.") + response = Thought( + thought=response.thought or "No actions provided", + mode="finish", + actions=None, + final_answer=response.thought or "Insufficient evidence to provide a definitive answer." + ) + final_answer = response.final_answer + else: + tool_name = response.actions.tool + try: + arguments = _build_tool_arguments(response.actions) + except ValueError as e: + logger.warning( + "%s bad tool arguments at step %d: %s", self.agent_type, step_num, e, + ) + span.set_output({"error": str(e), "step": step_num + 1}) + error_ai = AIMessage(content=response.thought or "I want to call a tool.") + return { + "messages": [error_ai, HumanMessage( + content=f"ERROR: {e}. Provide the required arguments and try again." + )], + "thought": None, + "step": step_num + 1, + "max_steps": self.config.max_iterations, + "output": "waiting for the agent to respond", + } + tool_call_id = str(uuid.uuid4()) + ai_message = AIMessage( + content=response.thought, + tool_calls=[{ + "name": tool_name, + "args": arguments, + "id": tool_call_id + }] + ) + + span.set_output({"mode": response.mode, "step": step_num + 1}) + return { + "messages": [ai_message], + "thought": response, + "step": step_num + 1, + "max_steps": self.config.max_iterations, + "output": final_answer + } + except Exception as e: + logger.exception("%s thought_node failed at step %d", self.agent_type, step_num) + span.set_output({"error": str(e), "exception_type": type(e).__name__, "step": step_num}) + raise + + async def should_continue(self, state: AgentState) -> str: + if state.get("step", 0) >= state.get("max_steps", self.config.max_iterations): + return "forced_finish_node" + thought = state.get("thought", None) + if thought is None: + return "thought_node" + if thought.mode == "finish": + return END + return "tool_node" + + @staticmethod + def _merge_unique_context_lines(dynamic_memory: list[str], static_context: list[str]) -> list[str]: + merged: list[str] = [] + seen: set[str] = set() + for line in [*dynamic_memory, *static_context]: + if not isinstance(line, str): + continue + normalized_line = line.strip() + if not normalized_line or normalized_line in seen: + continue + seen.add(normalized_line) + merged.append(normalized_line) + return merged + + def _build_observation_context(self, obs, crit_context) -> str | None: + """Format merged observation and critical context within token budget.""" + dynamic_memory = list(obs.memory) if obs is not None and obs.memory else [] + static_context = list(crit_context) if crit_context else [] + recent_findings = list(obs.results) if obs is not None and obs.results else [] + + merged_knowledge = self._merge_unique_context_lines(dynamic_memory, static_context) + if not merged_knowledge and not recent_findings: + return None + + findings_block = None + if recent_findings: + findings_block = "LATEST FINDINGS:\n" + "\n".join(f"- {finding}" for finding in recent_findings) + + # Reserve room for recent findings while capping merged knowledge growth. + knowledge_budget = self._OBS_CONTEXT_MAX_TOKENS + if findings_block: + knowledge_budget = max(150, self._OBS_CONTEXT_MAX_TOKENS - count_tokens(findings_block)) + + knowledge_header = "KNOWLEDGE:" + kept_knowledge: list[str] = [] + # Use running counter to avoid O(N²) re-tokenization on each iteration + running_tokens = count_tokens(knowledge_header + "\n") + for entry in merged_knowledge: + entry_line = f"- {entry}\n" + entry_tokens = count_tokens(entry_line) + if running_tokens + entry_tokens > knowledge_budget: + break + kept_knowledge.append(f"- {entry}") + running_tokens += entry_tokens + + if kept_knowledge and len(kept_knowledge) < len(merged_knowledge): + truncation_line = f"- {self._OBS_CONTEXT_TRUNCATION_NOTE}" + candidate_block = knowledge_header + "\n" + "\n".join([*kept_knowledge, truncation_line]) + if count_tokens(candidate_block) <= knowledge_budget: + kept_knowledge.append(truncation_line) + + parts = [] + if kept_knowledge: + parts.append(knowledge_header + "\n" + "\n".join(kept_knowledge)) + elif merged_knowledge: + # No entries fit in budget - show "no knowledge" rather than misleading truncation + parts.append(f"{knowledge_header}\n- No prior knowledge (budget exceeded).") + elif recent_findings: + parts.append("KNOWLEDGE:\n- No prior knowledge.") + if findings_block: + parts.append(findings_block) + return "\n".join(parts) + + async def forced_finish_node(self, state: AgentState) -> AgentState: + step_num = state.get("step", 0) + with AGENT_TRACER.push_active_function(f"{self.agent_type}_forced_finish", input_data=f"step:{step_num}") as span: + try: + active_prompt = state.get("runtime_prompt") + messages = [SystemMessage(content=active_prompt)] + state["messages"] + crit_context = state.get("critical_context",None) + context_block = self._build_observation_context(state.get("observation", None), crit_context) + if context_block: + messages.append(SystemMessage(content=context_block)) + question = state.get("input", "") + finish_prompt = f"QUESTION: {question}\n\n{FORCED_FINISH_PROMPT}" if question else FORCED_FINISH_PROMPT + messages.append(HumanMessage(content=finish_prompt)) + keep_tail = 2 if context_block else 1 + self._prune_messages_to_fit(messages, keep_tail=keep_tail, step_num=step_num, caller="forced_finish_node") + response: Thought = await self.thought_llm.ainvoke(messages) + if response.mode == "finish" and response.final_answer: + ai_message = AIMessage(content=response.final_answer) + final_answer = response.final_answer + else: + final_answer = "Failed to generate a final answer within the maximum allowed steps." + ai_message = AIMessage(content=final_answer) + response = Thought( + thought=response.thought or "Max steps exceeded", + mode="finish", + actions=None, + final_answer=final_answer + ) + span.set_output({"final_answer_length": len(final_answer), "step": step_num}) + return { + "messages": [ai_message], + "thought": response, + "step": step_num, + "max_steps": state.get("max_steps", self.config.max_iterations), + "observation": state.get("observation", None), + "output": final_answer + } + except Exception as e: + logger.exception("%s forced_finish_node failed at step %d", self.agent_type, step_num) + span.set_output({"error": str(e), "exception_type": type(e).__name__, "step": step_num}) + raise + + async def _invoke_comprehension(self, prompt: str, tool_used: str, tool_input: str, tool_output: str) -> CodeFindings: + """Invoke comprehension LLM with fallback on token limit overflow.""" + return await invoke_comprehension( + self.comprehension_llm, prompt, tool_used, tool_input, tool_output, agent_label=self.agent_type, + ) + + async def observation_node(self, state: AgentState) -> AgentState: + tool_message = state["messages"][-1] + last_thought_text = state["thought"].thought if state.get("thought") else "No previous thought." + tool_used = state["thought"].actions.tool if state.get("thought") and state["thought"].actions else "Unknown" + tool_input_detail = "" + if state.get("thought") and state["thought"].actions: + actions = state["thought"].actions + if actions.package_name and actions.function_name: + tool_input_detail = f"{actions.package_name},{actions.function_name}" + elif actions.query: + tool_input_detail = actions.query + elif actions.tool_input: + tool_input_detail = actions.tool_input + previous_memory = state.get("observation").memory if state.get("observation") else ["No data gathered yet."] + rules_tracker = state.get("rules_tracker") + with AGENT_TRACER.push_active_function(f"{self.agent_type}_observation", input_data=f"tool:{tool_used}") as span: + try: + tool_output_for_llm = tool_message.content + result, error_message = rules_tracker.check_thought_behavior(tool_used, tool_input_detail, tool_output_for_llm) + if result: + span.set_output({"rule_error": error_message}) + return {"messages": [HumanMessage(content=error_message)]} + + if self.should_truncate_tool_output(state, tool_used): + truncated_output = truncate_tool_output(tool_output_for_llm, tool_used) + else: + truncated_output = tool_output_for_llm + + critical_context_text = self.build_comprehension_context(state) + comp_prompt = COMPREHENSION_PROMPT.format( + goal=state.get('input'), + selected_package=state.get('app_package') or "N/A", + critical_context=critical_context_text, + tool_used=tool_used, + tool_input_detail=tool_input_detail, + last_thought_text=last_thought_text, + tool_output=truncated_output, + ) + max_input_tokens = self.config.context_window_token_limit - 2000 + prompt_tokens = count_tokens(comp_prompt) + if prompt_tokens > max_input_tokens: + overflow = prompt_tokens - max_input_tokens + output_tokens = count_tokens(truncated_output) + safe_budget = max(200, output_tokens - overflow) + truncated_output = truncate_tool_output(truncated_output, tool_used, max_tokens=safe_budget) + comp_prompt = COMPREHENSION_PROMPT.format( + goal=state.get('input'), + selected_package=state.get('app_package') or "N/A", + critical_context=critical_context_text, + tool_used=tool_used, + tool_input_detail=tool_input_detail, + last_thought_text=last_thought_text, + tool_output=truncated_output, + ) + logger.info( + "Comprehension prompt truncated: %d -> %d tokens (limit %d)", + prompt_tokens, count_tokens(comp_prompt), max_input_tokens, + ) + code_findings = await self._invoke_comprehension(comp_prompt, tool_used, tool_input_detail, truncated_output) + + sanitized = self.sanitize_findings(code_findings.findings, state) + findings_text = "\n".join(f"- {f}" for f in sanitized) + + mem_prompt = MEMORY_UPDATE_PROMPT.format( + goal=state.get('input'), + selected_package=state.get('app_package') or "N/A", + previous_memory=previous_memory, + findings=findings_text, + tool_outcome=code_findings.tool_outcome, + ) + new_observation: Observation = await self.observation_llm.ainvoke([SystemMessage(content=mem_prompt)]) + + messages = state["messages"] + active_prompt = state.get("runtime_prompt") + estimated = estimate_tokens(active_prompt, messages, new_observation) + prune_messages = [] + orig_estimated = estimated + + span_trace_dict = {"comprehension_findings": sanitized, "tool_outcome": code_findings.tool_outcome} + + if estimated > self.config.context_window_token_limit and len(messages) > 3: + prunable = messages[1:-2] + for msg in prunable: + prune_messages.append(RemoveMessage(id=msg.id)) + estimated -= count_tokens(msg.content) if hasattr(msg, "content") and isinstance(msg.content, str) else 0 + if estimated <= self.config.context_window_token_limit: + break + logger.info( + "Context pruning: removed %d messages, estimated tokens now ~%d (limit %d)", + len(prune_messages), estimated, self.config.context_window_token_limit, + ) + span_trace_dict["orig_estimated"] = orig_estimated + span_trace_dict["estimated"] = estimated + + # Agent-specific post-processing hook + extra = self.post_observation(state, tool_used, tool_output_for_llm, tool_input_detail) + + span.set_output(span_trace_dict) + base_result = { + "messages": prune_messages, + "observation": new_observation, + "step": state.get("step", 0), + } + base_result.update(extra) + return base_result + except Exception as e: + logger.exception("%s observation_node failed", self.agent_type) + span.set_output({"error": str(e), "exception_type": type(e).__name__}) + raise + + # --- Graph wiring --- + # If overridden, should_continue must also be updated to match the new node names. + + async def build_graph(self): + tool_node = ToolNode(self.tools, handle_tool_errors=True) + + flow = StateGraph(AgentState) + flow.add_node("thought_node", self.thought_node) + flow.add_node("tool_node", tool_node) + flow.add_node("forced_finish_node", self.forced_finish_node) + flow.add_node("pre_process_node", self.pre_process_node) + flow.add_node("observation_node", self.observation_node) + flow.add_edge(START, "pre_process_node") + flow.add_edge("pre_process_node", "thought_node") + flow.add_conditional_edges( + "thought_node", + self.should_continue, + {END: END, "tool_node": "tool_node", "forced_finish_node": "forced_finish_node", "thought_node": "thought_node"} + ) + flow.add_edge("tool_node", "observation_node") + flow.add_edge("observation_node", "thought_node") + flow.add_edge("forced_finish_node", END) + + return flow.compile() diff --git a/src/vuln_analysis/functions/build_agent_graph_defs.py b/src/vuln_analysis/functions/build_agent_graph_defs.py new file mode 100644 index 000000000..5f98c5dbb --- /dev/null +++ b/src/vuln_analysis/functions/build_agent_graph_defs.py @@ -0,0 +1,803 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Graph definitions for the L2 Build Agent (BuildCompilationCheck). + +Houses the LangGraph state schema and structured-output schemas for BuildHarvestReport. +Prompt templates are in vuln_analysis.utils.rpm_checker_prompts. +""" + +from __future__ import annotations + +import logging +import re +from pathlib import Path +from typing import Literal, NotRequired + +from langgraph.graph import MessagesState +from pydantic import BaseModel, Field + +from exploit_iq_commons.utils.hardening_kb import HardeningEntry +from vuln_analysis.functions.react_internals import CheckerThought, Observation +from vuln_analysis.utils.rpm_checker_prompts import ( + L2_CONFIG_SYS_PROMPT, + L2_CONFIG_PROMPT_TEMPLATE, + L2_CONFIG_THOUGHT_INSTRUCTIONS, + L2_HARDENING_SYS_PROMPT, + L2_HARDENING_PROMPT_TEMPLATE, + L2_HARDENING_THOUGHT_INSTRUCTIONS, + L2_COMPILATION_VERDICT_PROMPT, + L2_HARDENING_VERDICT_PROMPT, + L2_COMPREHENSION_PROMPT, + L2_MEMORY_UPDATE_PROMPT, + L2_HARDENING_COMPREHENSION_PROMPT, + L2_HARDENING_MEMORY_UPDATE_PROMPT, +) +logger = logging.getLogger(__name__) + +BASE_KERNEL_PACKAGE = "kernel" +KERNEL_RHEL_CONFIG_SUFFIX = "rhel.config" +KERNEL_DEBUG_FLAVOR = "debug" +KERNEL_RT_FLAVOR = "rt" +KERNEL_RT_DEBUG_FLAVOR = "rt-debug" + + +# --------------------------------------------------------------------------- +# Data Models +# --------------------------------------------------------------------------- + + +class BuildHarvestReport(BaseModel): + """Deterministic data harvested from build artifacts. + + Extracted during the data_harvest_node before the ReAct loop begins. + + Key vulnerability-relevant data: + - Feature disable flags that prevent vulnerable code from being compiled: + - OpenSSL style: no-sm2, no-ssl3, no-md5, no-asm + - Autoconf style: --disable-feature, --without-feature + - CMake style: -DENABLE_FEATURE=OFF + - Feature enable flags that explicitly enable optional features: + - Autoconf style: --enable-feature, --with-feature + - CMake style: -DENABLE_FEATURE=ON + - Architecture flags to understand target platform + - Hardening flags relevant to the CVE's CWE class + + Note: Compiled files are NOT pre-extracted. The LLM searches the build log + for affected files from l1_result.vulnerability_intel.affected_files during the ReAct loop. + """ + + disabled_features: list[str] = Field( + default_factory=list, + description="Feature-disabling flags from build log (e.g., '-DOPENSSL_NO_SM2', '-DNO_GZIP')", + ) + spec_disabled_features: list[str] = Field( + default_factory=list, + description="Feature-disabling flags from spec %build section (e.g., 'no-sm2', '--disable-ssl3', '--without-openssl')", + ) + enabled_features: list[str] = Field( + default_factory=list, + description="Feature-enabling flags from build log (e.g., '-DENABLE_LDAP', '-DLDAP_ENABLED')", + ) + spec_enabled_features: list[str] = Field( + default_factory=list, + description="Feature-enabling flags from spec %build section (e.g., '--enable-ldap', '--with-openssl')", + ) + expected_hardening: list[HardeningEntry] = Field( + default_factory=list, + description="Hardening flags relevant to the CVE's CWE, with descriptions for LLM context", + ) + build_architecture: Literal["32-bit", "64-bit", "unknown"] = Field( + default="unknown", + description="Target architecture from -m64/-m32 flags or build target (x86_64/i686)", + ) + linked_libraries: list[str] = Field( + default_factory=list, + description="Libraries linked at build time (extracted from -l flags in build log)", + ) + built_subpackages: list[str] = Field( + default_factory=list, + description="Subpackages defined in spec %package sections", + ) + excluded_subpackages: list[str] = Field( + default_factory=list, + description="Subpackages excluded via %bcond_without, ExcludeArch, or conditional guards", + ) + kernel_config: dict[str, str] = Field( + default_factory=dict, + description="Kernel CONFIG_* options (y/n/m) - only populated for kernel packages", + ) + is_kernel: bool = Field( + default=False, + description="True if this is a kernel package (detected by Kconfig presence)", + ) + kernel_config_path: str | None = Field( + default=None, + description="Path to kernel config file (e.g., kernel-x86_64-rhel.config) for LLM grep access", + ) + kernel_source_root: str | None = Field( + default=None, + description="Path to kernel source tree root (directory containing Kconfig)", + ) + + +class L2CompileVerdictExtraction(BaseModel): + """LLM-extracted verdict from L2 agent final answer.""" + + compilation_status: str = Field( + description="Whether vulnerable code is compiled into the binary: one of 'compiled', 'not_compiled', or 'unknown'" + ) + confidence: float = Field(description="Confidence in the verdict (0.0 to 1.0)") + reasoning: str = Field(description="Brief explanation of the verdict") + +class L2HardeningVerdictExtraction(BaseModel): + """LLM-extracted verdict from L2 Hardening investigation.""" + hardening_status: str = Field( + description="Whether hardening flags mitigate the vulnerability: one of 'mitigated', 'not_mitigated', 'not_applicable', or 'unknown'" + ) + hardening_flags: list[str] = Field( + default_factory=list, + description="List of specific hardening flags found (e.g., -fstack-protector-strong, -D_FORTIFY_SOURCE=2)", + ) + confidence: float = Field(description="Confidence in the verdict (0.0 to 1.0)") + reasoning: str = Field(description="Brief explanation of the verdict") + + +# --------------------------------------------------------------------------- +# Graph State +# --------------------------------------------------------------------------- + + +class BuildAgentState(MessagesState): + """LangGraph state for the L2 Build Agent.""" + + harvest_report: NotRequired[BuildHarvestReport | None] + vulnerability_intel_str: NotRequired[str | None] + l1_preliminary_verdict: NotRequired[str | None] + runtime_prompt: NotRequired[str | None] + thought: NotRequired[CheckerThought | None] + observation: NotRequired[Observation | None] + step: NotRequired[int] + max_steps: NotRequired[int] + L2CompileVerdict: NotRequired[L2CompileVerdictExtraction | None] + L2HardeningVerdict: NotRequired[L2HardeningVerdictExtraction | None] + evidence_sources: NotRequired[list[str] | None] + +# --------------------------------------------------------------------------- +# Spec File Parsing Helpers +# --------------------------------------------------------------------------- + + +def _extract_spec_build_section(spec_path: Path) -> str: + """Extract the %build section from an RPM spec file. + + The %build section contains configure/Configure commands with feature flags + that determine what code is compiled. + + Args: + spec_path: Path to the RPM spec file + + Returns: + The raw %build section content, or empty string if not found + """ + try: + content = spec_path.read_text(encoding="utf-8", errors="replace") + except OSError as e: + logger.warning("Failed to read spec file %s: %s", spec_path, e) + return "" + + # Find %build section (ends at next RPM section marker or EOF) + # Only match actual section markers, not macros like %configure, %ifarch, %{name} + # Common RPM sections: %prep, %build, %install, %check, %files, %post, %pre, + # %preun, %postun, %changelog, %package, %description + match = re.search( + r"^%build\s*\n(.*?)(?=^%(prep|install|check|files|post|pre|preun|postun|changelog|package|description)\b|\Z)", + content, + re.MULTILINE | re.DOTALL, + ) + return match.group(1).strip() if match else "" + + +def _extract_spec_disabled_features(build_section: str) -> list[str]: + """Extract feature-disable flags from spec %build section. + + Recognizes patterns from common build systems: + - OpenSSL style: no-sm2, no-ssl3, no-asm + - Autoconf style: --disable-feature, --without-feature + - CMake style: -DENABLE_FEATURE=OFF + + Args: + build_section: The raw %build section content + + Returns: + Sorted list of disabled feature names (without prefix) + """ + disabled: set[str] = set() + + # OpenSSL style: no-feature (e.g., no-sm2, no-ssl3, no-asm) + disabled.update(re.findall(r"\bno-(\w+)", build_section)) + + # Autoconf style: --disable-feature (e.g., --disable-static) + disabled.update(re.findall(r"--disable-(\w+)", build_section)) + + # Autoconf style: --without-feature (e.g., --without-openssl) + disabled.update(re.findall(r"--without-(\w+)", build_section)) + + # CMake style: -DENABLE_FEATURE=OFF or =0 or =FALSE + disabled.update( + re.findall(r"-DENABLE_(\w+)=(?:OFF|0|FALSE)", build_section, re.IGNORECASE) + ) + + return sorted(disabled) + + +def _extract_spec_enabled_features(build_section: str) -> list[str]: + """Extract feature-enable flags from spec %build section. + + Recognizes patterns from common build systems: + - Autoconf style: --enable-feature, --with-feature + - CMake style: -DENABLE_FEATURE=ON + + Args: + build_section: The raw %build section content + + Returns: + Sorted list of enabled feature names (without prefix) + """ + enabled: set[str] = set() + + # Autoconf style: --enable-feature (e.g., --enable-ldap) + enabled.update(re.findall(r"--enable-(\w+)", build_section)) + + # Autoconf style: --with-feature (e.g., --with-openssl) + enabled.update(re.findall(r"--with-(\w+)", build_section)) + + # CMake style: -DENABLE_FEATURE=ON or =1 or =TRUE + enabled.update( + re.findall(r"-DENABLE_(\w+)=(?:ON|1|TRUE)", build_section, re.IGNORECASE) + ) + + return sorted(enabled) + + +def _extract_linked_libraries(build_log_content: str) -> list[str]: + """Extract libraries linked at build time from build log. + + Parses -l flags from gcc/g++/ld commands in the build log. + + Args: + build_log_content: Full content of the build log file + + Returns: + Sorted list of unique library names (without 'lib' prefix or path) + """ + libraries: set[str] = set() + + # Match -l flags preceded by whitespace (actual linker flags) + # Library names must start with a letter (filters out false positives like "ib500" from "lib500") + libraries.update(re.findall(r"(?:^|\s)-l([a-zA-Z][a-zA-Z0-9_]*)", build_log_content, re.MULTILINE)) + + return sorted(libraries) + + +def _extract_spec_subpackages(spec_content: str) -> tuple[list[str], list[str]]: + """Extract built and excluded subpackages from spec file. + + Parses %package directives and conditional guards to determine + which subpackages are built vs excluded. + + Args: + spec_content: Full content of the spec file + + Returns: + Tuple of (built_subpackages, excluded_subpackages) + """ + built: set[str] = set() + excluded: set[str] = set() + + # Extract all %package definitions + # Handles: %package -n libfoo, %package devel, %package -n foo-libs + for match in re.finditer(r"^%package\s+(?:-n\s+)?(\S+)", spec_content, re.MULTILINE): + pkg_name = match.group(1) + # Skip macro expansions that we can't resolve + if not pkg_name.startswith("%"): + built.add(pkg_name) + + # Extract %bcond_without directives (features disabled by default) + for match in re.finditer(r"%bcond_without\s+(\w+)", spec_content): + excluded.add(match.group(1)) + + # Extract %bcond_with directives (features that must be explicitly enabled) + # These are also "excluded by default" + for match in re.finditer(r"%bcond_with\s+(\w+)", spec_content): + excluded.add(match.group(1)) + + # Check for ExcludeArch which excludes entire package on certain architectures + if re.search(r"^ExcludeArch:", spec_content, re.MULTILINE): + # Note: We can't determine current arch, but flag that exclusions exist + pass + + return sorted(built), sorted(excluded) + + +def _extract_kernel_config(source_path: Path) -> dict[str, str]: + """Extract kernel CONFIG_* options from .config file. + + Only applicable to kernel packages. Looks for .config file + in the source tree and parses CONFIG_*=y/n/m lines. + + Args: + source_path: Path to the kernel source directory + + Returns: + Dict mapping CONFIG_* names to their values (y/n/m) + """ + config: dict[str, str] = {} + + # Look for .config file in common locations + config_paths = [ + source_path / ".config", + source_path / "configs" / ".config", + ] + + # Also check for arch-specific configs + for arch_config in source_path.glob("configs/kernel-*.config"): + config_paths.append(arch_config) + + for config_path in config_paths: + if config_path.exists() and config_path.is_file(): + try: + content = config_path.read_text(encoding="utf-8", errors="replace") + # Parse CONFIG_*=y/n/m lines + for match in re.finditer(r"^(CONFIG_\w+)=(y|n|m)", content, re.MULTILINE): + config[match.group(1)] = match.group(2) + # Also capture "# CONFIG_X is not set" as CONFIG_X=n + for match in re.finditer(r"^# (CONFIG_\w+) is not set", content, re.MULTILINE): + config[match.group(1)] = "n" + # Found and parsed a config file, no need to continue + if config: + logger.info( + "_extract_kernel_config: parsed %d options from %s", + len(config), config_path + ) + break + except OSError as e: + logger.warning("_extract_kernel_config: failed to read %s: %s", config_path, e) + + return config + + +def _is_kernel_package(source_path: Path) -> bool: + """Detect kernel package by presence of Kconfig in source root or linux-* subdir. + + Kernel source trees contain a top-level Kconfig file that defines the + kernel configuration system. This is a reliable marker for kernel packages. + + Args: + source_path: Path to the source directory + + Returns: + True if this appears to be a kernel package + """ + if not source_path or not source_path.exists(): + return False + + # Check source root for Kconfig + if (source_path / "Kconfig").exists(): + return True + + # Check linux-* subdirectory pattern (RHEL kernel structure) + for subdir in source_path.glob("linux-*"): + if subdir.is_dir(): + # Check nested linux-* (e.g., linux-5.14.0/linux-5.14.0.el9/) + for nested in subdir.glob("linux-*"): + if nested.is_dir() and (nested / "Kconfig").exists(): + return True + # Check direct subdir + if (subdir / "Kconfig").exists(): + return True + + return False + + +def _kernel_package_flavor(package_name: str | None) -> str | None: + """Return the kernel sub-flavor from an RPM name (e.g. kernel-rt -> rt).""" + if not package_name or package_name == BASE_KERNEL_PACKAGE: + return None + prefix = f"{BASE_KERNEL_PACKAGE}-" + if not package_name.startswith(prefix): + return None + return package_name.removeprefix(prefix) + + +def _kernel_config_candidate_names(arch: str, package_name: str | None) -> list[str]: + """Build ordered RHEL kernel config basenames for a package and architecture.""" + flavor = _kernel_package_flavor(package_name) + candidates: list[str] = [] + + if flavor == KERNEL_RT_DEBUG_FLAVOR: + # kernel-rt-debug -> support both layouts seen in RHEL sources: + # kernel-rt-x86_64-debug-rhel.config + # kernel-x86_64-rt-debug-rhel.config + candidates.append( + f"kernel-{KERNEL_RT_FLAVOR}-{arch}-{KERNEL_DEBUG_FLAVOR}-{KERNEL_RHEL_CONFIG_SUFFIX}" + ) + candidates.append( + f"kernel-{arch}-{KERNEL_RT_FLAVOR}-{KERNEL_DEBUG_FLAVOR}-{KERNEL_RHEL_CONFIG_SUFFIX}" + ) + elif flavor and KERNEL_DEBUG_FLAVOR in flavor: + # kernel-debug -> kernel-x86_64-debug-rhel.config + candidates.append( + f"kernel-{arch}-{KERNEL_DEBUG_FLAVOR}-{KERNEL_RHEL_CONFIG_SUFFIX}" + ) + elif flavor: + # kernel-rt -> kernel-rt-x86_64-rhel.config; also try flavor-after-arch variant + candidates.append(f"kernel-{flavor}-{arch}-{KERNEL_RHEL_CONFIG_SUFFIX}") + candidates.append(f"kernel-{arch}-{flavor}-{KERNEL_RHEL_CONFIG_SUFFIX}") + else: + candidates.append(f"kernel-{arch}-{KERNEL_RHEL_CONFIG_SUFFIX}") + + return list(dict.fromkeys(candidates)) + + +def _score_kernel_config_basename(name: str, arch: str, flavor: str | None) -> int: + """Rank glob matches: exact segment match wins over substring.""" + if arch not in name: + return -1 + + score = 0 + if flavor: + if f"-{flavor}-" in name or name.startswith(f"kernel-{flavor}-"): + score += 50 + elif flavor in name: + score += 25 + + wants_debug = flavor is not None and KERNEL_DEBUG_FLAVOR in flavor + is_debug = f"-{KERNEL_DEBUG_FLAVOR}-" in name + if is_debug: + score += 20 if wants_debug else -40 + + return score + + +def _find_kernel_config_file( + source_path: Path, + arch: str, + package_name: str | None = None, +) -> Path | None: + """Find the kernel config file for a specific architecture and package flavor. + + RHEL SRPMs use several naming layouts in the source root, for example: + - kernel-x86_64-rhel.config (base kernel) + - kernel-rt-x86_64-rhel.config (kernel-rt: flavor before arch) + - kernel-x86_64-debug-rhel.config (kernel-debug: flavor after arch) + + Args: + source_path: Path to the source directory + arch: Target architecture (e.g., 'x86_64', 'aarch64') + package_name: RPM package name (e.g., 'kernel', 'kernel-rt') + + Returns: + Path to the config file, or None if not found + """ + if not source_path or not source_path.exists() or not arch: + return None + + flavor = _kernel_package_flavor(package_name) + for candidate_name in _kernel_config_candidate_names(arch, package_name): + config_path = source_path / candidate_name + if config_path.is_file(): + logger.info( + "_find_kernel_config_file: matched candidate %s for package=%s arch=%s", + candidate_name, + package_name, + arch, + ) + return config_path + + glob_pattern = f"kernel*{arch}*{KERNEL_RHEL_CONFIG_SUFFIX}" + glob_matches = [path for path in source_path.glob(glob_pattern) if path.is_file()] + if not glob_matches: + logger.warning( + "_find_kernel_config_file: no config found for package=%s arch=%s in %s", + package_name, + arch, + source_path, + ) + return None + + best_match = max( + glob_matches, + key=lambda path: _score_kernel_config_basename(path.name, arch, flavor), + ) + best_score = _score_kernel_config_basename(best_match.name, arch, flavor) + if best_score < 0: + logger.warning( + "_find_kernel_config_file: glob results did not contain arch %s in %s", + arch, + source_path, + ) + return None + if flavor and best_score <= 0: + logger.warning( + "_find_kernel_config_file: fallback %s does not match flavor=%s for package=%s arch=%s", + best_match, + flavor, + package_name, + arch, + ) + return None + + logger.info( + "_find_kernel_config_file: selected fallback %s for package=%s arch=%s", + best_match, + package_name, + arch, + ) + return best_match + + +def _find_kernel_source_root(source_path: Path) -> Path | None: + """Find the actual kernel source tree root containing Kconfig. + + RHEL kernel packages have a nested structure: + source/ + ├── kernel-x86_64-rhel.config (config files at root) + └── linux-5.14.0/ + └── linux-5.14.0.el9/ (actual kernel source here) + └── Kconfig + + Args: + source_path: Path to the source directory + + Returns: + Path to the kernel source root (directory containing Kconfig), or None + """ + if not source_path or not source_path.exists(): + return None + + # Check if Kconfig is directly in source_path + if (source_path / "Kconfig").exists(): + return source_path + + # Check linux-* subdirectory pattern + for subdir in source_path.glob("linux-*"): + if subdir.is_dir(): + # Check nested linux-* first (RHEL pattern) + for nested in subdir.glob("linux-*"): + if nested.is_dir() and (nested / "Kconfig").exists(): + logger.info("_find_kernel_source_root: found at %s", nested) + return nested + # Check direct subdir + if (subdir / "Kconfig").exists(): + logger.info("_find_kernel_source_root: found at %s", subdir) + return subdir + + logger.warning("_find_kernel_source_root: no kernel source root found in %s", source_path) + return None + + +# --------------------------------------------------------------------------- +# Data Harvesting Functions +# --------------------------------------------------------------------------- + + +async def harvest_build_data( + build_log_path: Path | None, + spec_path: Path | None, + cwe_id: str | None = None, + source_path: Path | None = None, + package_name: str | None = None, + arch: str | None = None, +) -> BuildHarvestReport: + """Extract structured data from build log and spec file. + + Parses: + - Feature-disabling -D defines (e.g., -DOPENSSL_NO_SM2, -DNO_GZIP) + - Feature-enabling -D defines (e.g., -DENABLE_LDAP, -DLDAP_ENABLED) + - Linked libraries from -l flags + - Subpackages from spec %package directives + - Kernel CONFIG_* options (for kernel packages only) + + For kernel packages, uses CONFIG_* based compilation checking instead of + build log analysis (kernel builds use make -s which hides individual compilations). + + Args: + build_log_path: Path to the build log file + spec_path: Path to the RPM spec file + cwe_id: CWE identifier to look up expected hardening flags (e.g., 'CWE-121') + source_path: Path to the source directory (for kernel config extraction) + package_name: Name of the package (to detect kernel packages) + arch: Target architecture (e.g., 'x86_64') for kernel config file selection + + Returns: + BuildHarvestReport with harvested data and expected hardening flags + """ + from exploit_iq_commons.utils.hardening_kb import HardeningKB + from vuln_analysis.tools.source_inspector import SourceInspector + + # Detect kernel package and find kernel-specific paths + is_kernel = _is_kernel_package(source_path) if source_path else False + kernel_config_path: str | None = None + kernel_source_root: str | None = None + + if is_kernel: + logger.info("harvest_build_data: detected kernel package") + # Find kernel config file for the target architecture + if arch: + config_file = _find_kernel_config_file(source_path, arch, package_name) + if config_file: + # Basename only: L2 Source Grep uses it as a file_glob under source root. + kernel_config_path = config_file.name + # Find kernel source root (contains Kconfig, Makefiles) + source_root = _find_kernel_source_root(source_path) + if source_root: + kernel_source_root = str(source_root) + + # Handle case where build_log_path is a directory instead of a file + if build_log_path and build_log_path.is_dir(): + log_files = list(build_log_path.glob("*-build.log")) or list(build_log_path.glob("*.log")) + if log_files: + build_log_path = log_files[0] + logger.info("harvest_build_data: resolved build log directory to file: %s", build_log_path) + else: + logger.warning("harvest_build_data: build_log_path is a directory but no .log files found") + build_log_path = None + + # Lookup expected hardening flags from KB based on CWE + expected_hardening = [] + if cwe_id: + kb = HardeningKB.get_instance() + expected_hardening = kb.lookup_by_cwe(cwe_id) + logger.info( + "harvest_build_data: CWE %s maps to %d hardening flags", + cwe_id, + len(expected_hardening), + ) + + # Extract feature-disabling defines from build log + disabled_features: list[str] = [] + if build_log_path: + inspector = SourceInspector(build_log_path.parent) + + # Grep for lines containing -D defines + matches = inspector.grep_content(r"-D\w+", file_path=build_log_path) + + # Extract unique defines from matched lines + all_defines: set[str] = set() + for match in matches: + defines = re.findall(r"-D(\w+)", match.line_content) + all_defines.update(defines) + + # Filter for feature-disabling patterns: + # - NO_* prefix (e.g., NO_GZIP) + # - DISABLE_* prefix (e.g., DISABLE_SSL) + # - WITHOUT_* prefix (e.g., WITHOUT_FEATURE) + # - *_NO_* infix (e.g., OPENSSL_NO_SM2) + # - *_DISABLE_* infix + # - *_DISABLED suffix + disable_pattern = re.compile( + r"^(NO_|DISABLE_|WITHOUT_)|(_NO_|_DISABLE_)|(_DISABLED$)" + ) + disabled_features = sorted( + d for d in all_defines if disable_pattern.search(d) + ) + + if disabled_features: + logger.info( + "harvest_build_data: found %d disabled features in build log", + len(disabled_features), + ) + + # Filter for feature-enabling patterns: + # - ENABLE_* prefix (e.g., ENABLE_LDAP) + # - *_ENABLED suffix (e.g., LDAP_ENABLED) + enable_pattern = re.compile(r"^ENABLE_|_ENABLED$") + enabled_features = sorted( + d for d in all_defines if enable_pattern.search(d) + ) + + if enabled_features: + logger.info( + "harvest_build_data: found %d enabled features in build log", + len(enabled_features), + ) + else: + enabled_features = [] + + # Extract linked libraries from build log (skip for kernel - not meaningful) + # Note: Architecture detection removed - use target_package.arch instead (checked in L1) + build_architecture: Literal["32-bit", "64-bit", "unknown"] = "unknown" + linked_libraries: list[str] = [] + if build_log_path and not is_kernel: + try: + build_log_content = build_log_path.read_text(encoding="utf-8", errors="replace") + linked_libraries = _extract_linked_libraries(build_log_content) + if linked_libraries: + logger.info( + "harvest_build_data: found %d linked libraries", + len(linked_libraries), + ) + except OSError as e: + logger.warning("harvest_build_data: failed to read build log: %s", e) + + # Extract %build section and features from spec file + spec_build_section = "" + spec_disabled_features: list[str] = [] + spec_enabled_features: list[str] = [] + built_subpackages: list[str] = [] + excluded_subpackages: list[str] = [] + if spec_path and spec_path.exists(): + spec_build_section = _extract_spec_build_section(spec_path) + spec_disabled_features = _extract_spec_disabled_features(spec_build_section) + spec_enabled_features = _extract_spec_enabled_features(spec_build_section) + + if spec_disabled_features: + logger.info( + "harvest_build_data: found %d disabled features in spec", + len(spec_disabled_features), + ) + if spec_enabled_features: + logger.info( + "harvest_build_data: found %d enabled features in spec", + len(spec_enabled_features), + ) + + # Extract subpackages from full spec content + try: + spec_content = spec_path.read_text(encoding="utf-8", errors="replace") + built_subpackages, excluded_subpackages = _extract_spec_subpackages(spec_content) + if built_subpackages: + logger.info( + "harvest_build_data: found %d built subpackages", + len(built_subpackages), + ) + if excluded_subpackages: + logger.info( + "harvest_build_data: found %d excluded subpackages", + len(excluded_subpackages), + ) + except OSError as e: + logger.warning("harvest_build_data: failed to read spec for subpackages: %s", e) + + # Extract kernel config if this is a kernel package + # Note: kernel_config dict is kept for backward compatibility but the primary + # mechanism for kernel is now CONFIG lookup via kernel_config_path + kernel_config: dict[str, str] = {} + if is_kernel and source_path and source_path.exists(): + kernel_config = _extract_kernel_config(source_path) + if kernel_config: + logger.info( + "harvest_build_data: extracted %d kernel config options", + len(kernel_config), + ) + + return BuildHarvestReport( + disabled_features=disabled_features, + spec_disabled_features=spec_disabled_features, + enabled_features=enabled_features, + spec_enabled_features=spec_enabled_features, + expected_hardening=expected_hardening, + build_architecture=build_architecture, + linked_libraries=linked_libraries, + built_subpackages=built_subpackages, + excluded_subpackages=excluded_subpackages, + kernel_config=kernel_config, + is_kernel=is_kernel, + kernel_config_path=kernel_config_path, + kernel_source_root=kernel_source_root, + ) + diff --git a/src/vuln_analysis/functions/code_agent_graph_defs.py b/src/vuln_analysis/functions/code_agent_graph_defs.py new file mode 100644 index 000000000..c1278f49a --- /dev/null +++ b/src/vuln_analysis/functions/code_agent_graph_defs.py @@ -0,0 +1,1970 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Graph definitions for the L1 Package Code Agent. + +Houses the LangGraph state schema, structured-output schemas for +DownstreamSearchReport/UpstreamSearchReport pipelines, and CodeAgentReport. +Prompt templates are in vuln_analysis.utils.rpm_checker_prompts. +""" + +from __future__ import annotations + +import logging +import re +import shutil +import subprocess +import warnings +from dataclasses import dataclass +from pathlib import Path +from typing import Literal, NotRequired, TYPE_CHECKING + +if TYPE_CHECKING: + from langchain_core.language_models import BaseChatModel + from vuln_analysis.tools.brew_downloader import BrewDownloader + +import aiohttp +import openai +from langchain_core.messages import HumanMessage, SystemMessage +from langgraph.graph import MessagesState +from pydantic import BaseModel, Field +from unidiff import PatchSet + +logger = logging.getLogger(__name__) + +from exploit_iq_commons.data_models.checker_status import L2BuildResult, VulnerabilityIntel +from exploit_iq_commons.data_models.common import TargetPackage +from vuln_analysis.functions.react_internals import CheckerThought, Observation, L1VerdictExtraction +from vuln_analysis.utils.rpm_checker_prompts import ( + L1_VERDICT_EXTRACTION_PROMPT, + VULNERABILITY_INTEL_EXTRACTION_PROMPT, + CODE_AGENT_REPORT_PROMPT, + L1_AGENT_SYS_PROMPT_PATCH_AVAILABLE, + L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH, + L1_AGENT_SYS_PROMPT_REBASE_FIX, + L1_AGENT_SYS_PROMPT_REBASE_NO_PATCH, + L1_AGENT_PROMPT_TEMPLATE, + L1_AGENT_PROMPT_TEMPLATE_NO_PATCH, + L1_AGENT_THOUGHT_INSTRUCTIONS, + L1_AGENT_THOUGHT_UPSTREAM_INSTRUCTIONS, + L1_AGENT_THOUGHT_REBASE_INSTRUCTIONS, + L1_AGENT_THOUGHT_CVE_DESC_INSTRUCTIONS, + L1_COMPREHENSION_PROMPT, + L1_MEMORY_UPDATE_PROMPT, + L1_COMPREHENSION_PROMPT_CVE_DESC, + L1_MEMORY_UPDATE_PROMPT_CVE_DESC, +) + +# --------------------------------------------------------------------------- +# Graph state +# --------------------------------------------------------------------------- + + +class CodeAgentState(MessagesState): + """LangGraph state for the L1 Code Agent (DownstreamSearch -> UpstreamSearch).""" + downstream_report: NotRequired[DownstreamSearchReport | None] + upstream_report: NotRequired[UpstreamSearchReport | None] + runtime_prompt: NotRequired[str | None] + last_thought: NotRequired[CheckerThought | None] + step: NotRequired[int] + max_steps: NotRequired[int] + output: NotRequired[str] + thought: NotRequired[CheckerThought | None] + observation: NotRequired[Observation | None] + vulnerability_intel: NotRequired["VulnerabilityIntel | None"] + arch_mismatch_reason: NotRequired[str | None] + # Reference Mining state (populated when no patch found via existing flows) + reference_hints: NotRequired["ReferenceHints | None"] + advisory_urls_remaining: NotRequired[list[str]] + advisory_urls_processed: NotRequired[list[str]] + # Repository Resolution state (Phase 2: package → upstream git repo) + repo_resolution: NotRequired["RepoResolution | None"] + # Git Search state (Phase 3: commit discovery in upstream repo) + git_search_report: NotRequired["GitSearchReport | None"] + + +# --------------------------------------------------------------------------- +# Patch schemas (must be defined before reports that use them) +# --------------------------------------------------------------------------- + + +class PatchHunk(BaseModel): + """A single hunk from a downstream patch file.""" + source_start: int + source_length: int + target_start: int + target_length: int + section_header: str = Field(default="", description="Function context from @@ hunk header") + context_lines: list[str] = Field(default_factory=list, description="Unchanged lines") + removed_lines: list[str] = Field(default_factory=list, description="Deleted lines (- stripped)") + added_lines: list[str] = Field(default_factory=list, description="Added lines (+ stripped)") + + +class PatchFile(BaseModel): + """Changes to a single file in a downstream patch.""" + source_path: str + target_path: str + hunks: list[PatchHunk] + is_new_file: bool = False + is_deleted_file: bool = False + + @property + def clean_target_path(self) -> str: + """Target path with git diff prefix (b/ or a/) stripped.""" + if self.target_path.startswith("b/"): + return self.target_path[2:] + if self.target_path.startswith("a/"): + return self.target_path[2:] + return self.target_path + + +class ParsedPatch(BaseModel): + """Structured representation of a downstream patch file.""" + patch_filename: str + files: list[PatchFile] + + +class OSVPatchResult(BaseModel): + """Result of fetching a patch from OSV/GitHub or intel references.""" + cve_id: str + fixed_commit: str + repo_url: str + patch_url: str + patch_content: str | None = Field(default=None, description="Raw .patch text") + parsed_patch: "ParsedPatch | None" = Field(default=None, description="Structured patch data") + commit_message: str | None = None + commit_author: str | None = None + commit_date: str | None = None + source: str | None = Field(default=None, description="Source provider (ghsa/nvd/rhsa/ubuntu_patches/osv)") + url_type: str | None = Field(default=None, description="URL type (commit/pull)") + platform: str | None = Field(default=None, description="Hosting platform (github/kernel.org)") + + +@dataclass +class SpecPatchMatch: + """Result of finding a CVE patch in a spec file.""" + patch_index: int + patch_filename: str + raw_directive: str + comment_block: str | None + match_source: Literal["filename", "comment"] + + +# --------------------------------------------------------------------------- +# Reflection schemas +# --------------------------------------------------------------------------- +class DownstreamSearchReport(BaseModel): + """Result of a downstream search.""" + is_patch_file_available: bool = Field(default=False, description="True if a patch file is available") + patch_file_name: str = Field(default="", description="The name of the patch file") + is_patch_in_spec_file: bool = Field(default=False, description="True if a patch file is in the spec file") + spec_file_log_change: str = Field( + default="", + description="All lines in the .spec file that match a grep for the CVE id (not changelog-only)", + ) + is_patch_applied_in_build: bool = Field(default=False, description="True if a patch file is applied in the build") + build_log_patch_applied: str = Field(default="", description="The patch applied in the build log") + spec_patch_directives_for_cve: list[str] = Field( + default_factory=list, + description="Raw PatchN: lines from the spec whose patch filename token matches this CVE", + ) + spec_changelog_cve_lines: str = Field( + default="", + description="Lines from the %changelog section of the .spec that mention the CVE", + ) + spec_source0_line: str = Field( + default="", + description="The Source0: line from the spec file (upstream tarball reference)", + ) + spec_version_line: str = Field( + default="", + description="The Version: line from the spec file", + ) + parsed_patch: ParsedPatch | None = Field(default=None, description="The parsed patch file") + + +class UpstreamSearchReport(BaseModel): + """Result of an upstream search.""" + + is_fixed_srpm_is_needed: bool = Field(default=False, description="True if a fixed SRPM is needed downstream style patch files") + fixed_srpm_file_name: str = Field(default="", description="The name of the fixed SRPM file") + fixed_parsed_patch: ParsedPatch | None = Field(default=None, description="The parsed fixed SRPM patch file") + reference_package_nvr: str = Field( + default="", + description="NVR (name-version-release) of the reference fixed package from intel", + ) + reason_cve_code: str = Field( + default="", + description="Does the CVE description match the code which is vulnerable", + ) + is_code_fixed_by_rebase: Literal["yes", "no", "unknown"] = Field( + default="unknown", + description="yes if the code is fixed by rebase", + ) + spec_file_log_change: str = Field( + default="", + description="The log change of patch in the spec file", + ) + spec_fixed_srpm_change: str = Field( + default="", + description="The change of the fixed SRPM in the spec file", + ) + reason_code_fixed_by_rebase: str = Field( + default="", + description="The reason why the code is fixed by rebase", + ) + osv_result: OSVPatchResult | None = Field(default=None, description="The result of the OSV patch retrieval") + + + + + +class ReflectionBase(BaseModel): + """Base schema for phase reports. + + Subclasses add phase-specific fields on top. + """ + instructions: str = Field( + description="Guidance to the generator for the next iteration.") + is_sufficient: bool = Field( + description="True if results are good enough to proceed.") + + +# --------------------------------------------------------------------------- +# Reference Mining schemas +# --------------------------------------------------------------------------- + +HINT_STRENGTH_STRONG = "strong" +HINT_STRENGTH_MEDIUM = "medium" +HINT_STRENGTH_WEAK = "weak" +HINT_STRENGTH_NONE = "none" + + +class ReferenceHints(BaseModel): + """Extracted hints from advisory references for commit discovery. + + These hints are extracted by LLM from advisory pages (mailing lists, + vendor advisories, security bulletins) when no direct patch/commit + URL is available. Used to guide git log searches and grep patterns. + """ + + revision_hint: str | None = Field( + default=None, + description="Single SVN revision (r1935008) or git commit hash for THIS CVE's fix", + ) + function_hints: list[str] = Field( + default_factory=list, + description="Vulnerable or patched function names", + ) + file_hints: list[str] = Field( + default_factory=list, + description="Source file paths (.c, .h, .py)", + ) + branch_hints: list[str] = Field( + default_factory=list, + description="Branch names where fix was applied", + ) + version_hints: list[str] = Field( + default_factory=list, + description="Version numbers containing the fix", + ) + sources_processed: list[str] = Field( + default_factory=list, + description="Advisory URLs that were processed", + ) + hint_strength: str = Field( + default=HINT_STRENGTH_NONE, + description="Overall strength of extracted hints: strong, medium, weak, or none", + ) + confidence: float = Field( + default=0.0, + ge=0.0, + le=1.0, + description="LLM confidence in extraction quality (0.0-1.0)", + ) + + +class AdvisoryContent(BaseModel): + """Fetched advisory page content for LLM extraction. + + Represents the result of fetching a single advisory URL (mailing list + post, vendor security bulletin, etc.) for hint extraction. + """ + + url: str = Field(description="The advisory URL that was fetched") + source_type: str = Field( + description="Classification of the source: openwall, mailing_list, vendor_advisory, etc.", + ) + raw_text: str = Field( + default="", + description="Extracted text content from the advisory page", + ) + fetch_success: bool = Field( + default=True, + description="Whether the HTTP fetch succeeded", + ) + error_message: str | None = Field( + default=None, + description="Error message if fetch_success is False", + ) + + +# --------------------------------------------------------------------------- +# Repository Resolution schema (Phase 2: package → upstream git repo) +# --------------------------------------------------------------------------- + +RESOLUTION_METHOD_OSIDB_PURL = "osidb_purl" +RESOLUTION_METHOD_CURATED_MAPPING = "curated_mapping" +RESOLUTION_METHOD_PURL_PARSE = "purl_parse" +RESOLUTION_METHOD_NOT_FOUND = "not_found" + + +# --------------------------------------------------------------------------- +# Git Search schema (Phase 3: commit discovery in upstream repo) +# --------------------------------------------------------------------------- + +SEARCH_METHOD_REVISION = "revision" +SEARCH_METHOD_FUNCTION_MESSAGE = "function_message" +SEARCH_METHOD_FUNCTION_PICKAXE = "function_pickaxe" + +CONFIDENCE_REVISION_DIRECT = 0.99 +CONFIDENCE_REVISION_BRANCH = 0.95 +CONFIDENCE_REVISION_FALLBACK = 0.85 +CONFIDENCE_FUNCTION_MESSAGE = 0.75 +CONFIDENCE_FUNCTION_PICKAXE = 0.70 +CONFIDENCE_THRESHOLD_MIN = 0.50 + + +class CommitSearchResult(BaseModel): + """Result from a single search strategy. + + Represents a candidate commit found during git search, with metadata + about how it was discovered and confidence in the match. + """ + + commit_hash: str = Field( + description="Full 40-character git commit hash", + ) + commit_hash_short: str = Field( + description="Short 7-character commit hash for display", + ) + commit_message: str = Field( + description="Full commit message", + ) + author: str = Field( + description="Commit author name and email", + ) + date: str = Field( + description="Commit date in ISO format", + ) + files_changed: list[str] = Field( + default_factory=list, + description="List of files modified in commit", + ) + + search_method: Literal[ + "revision", + "function_message", + "function_pickaxe", + ] = Field(description="Strategy that found this commit") + confidence: float = Field( + ge=0.0, + le=1.0, + description="Confidence score for this result", + ) + match_details: str = Field( + default="", + description="What matched (e.g., 'grep matched r1935008')", + ) + + +class GitSearchReport(BaseModel): + """Complete report from git search node. + + Aggregates results from all search strategies with execution metadata. + The best_result is converted to ParsedPatch format for L1 agent consumption. + """ + + repo_url: str = Field(description="Repository URL that was searched") + repo_path: str | None = Field( + default=None, + description="Local path to cloned repository", + ) + branches_searched: list[str] = Field( + default_factory=list, + description="Branches that were fetched and searched", + ) + + best_result: CommitSearchResult | None = Field( + default=None, + description="Highest confidence result", + ) + all_results: list[CommitSearchResult] = Field( + default_factory=list, + description="All candidate commits found (ranked by confidence)", + ) + + commit_diff: str | None = Field( + default=None, + description="Full diff of the best result commit", + ) + parsed_patch: ParsedPatch | None = Field( + default=None, + description="Converted to ParsedPatch format for L1 agent", + ) + + search_success: bool = Field( + default=False, + description="True if at least one commit was found", + ) + strategies_tried: list[str] = Field( + default_factory=list, + description="Search strategies that were attempted", + ) + strategies_succeeded: list[str] = Field( + default_factory=list, + description="Search strategies that found results", + ) + error_messages: list[str] = Field( + default_factory=list, + description="Errors encountered during search", + ) + + clone_time_ms: int = Field(default=0, description="Time to clone/update repo") + search_time_ms: int = Field(default=0, description="Time to search") + total_time_ms: int = Field(default=0, description="Total node execution time") + + +class RepoResolution(BaseModel): + """Result of repository resolution for a package. + + Maps package name → upstream git repository URL using multiple strategies: + 1. OSIDB upstream_purls (highest confidence) + 2. Curated package → repo mapping (PRIMARY for C/C++) + 3. PURL parsing from any intel source + + Only public repositories are supported. If resolution fails, GIT_SEARCH_NODE + is skipped and the flow proceeds directly to L1_AGENT_NODE. + """ + + # Resolution result + repo_url: str | None = Field( + default=None, + description="Upstream git repository URL (clone URL). Only public repos supported.", + ) + repo_platform: str | None = Field( + default=None, + description="Hosting platform: github, gitlab, kernel.org, bitbucket, etc.", + ) + + # Resolution metadata + resolution_method: Literal[ + "osidb_purl", + "curated_mapping", + "purl_parse", + "not_found", + ] = Field(default=RESOLUTION_METHOD_NOT_FOUND) + confidence: float = Field( + default=0.0, + ge=0.0, + le=1.0, + description="Confidence in resolution: 0.95 (OSIDB), 0.90 (mapping), 0.85 (alias), 0.80 (PURL)", + ) + is_alias: bool = Field( + default=False, + description="True if resolved via alias in curated mapping (confidence 0.85 vs 0.90)", + ) + + # Alternative repos (if multiple found) + alternative_repos: list[str] = Field( + default_factory=list, + description="Other candidate repo URLs found during resolution", + ) + + # Tracking + strategies_tried: list[str] = Field( + default_factory=list, + description="List of resolution strategies attempted", + ) + error_messages: list[str] = Field( + default_factory=list, + description="Errors encountered during resolution", + ) + + +# --------------------------------------------------------------------------- +# Code Agent Report schema +# --------------------------------------------------------------------------- + + +class CodeSnippet(BaseModel): + """A code snippet from the investigation.""" + file_path: str = Field(description="Path to the source file") + line_number: int | None = Field(default=None, description="Starting line number") + code: str = Field(description="The code content") + snippet_type: str = Field( + description="Type of snippet: one of 'vulnerable', 'fix', or 'context'") + source: str = Field( + description="Where this snippet came from: one of 'downstream_patch', 'upstream_patch', or 'source_search'") + + +class CodeAgentReportLLM(BaseModel): + """LLM-facing model for report generation. Excludes code_snippets which are populated programmatically.""" + justification_label: str = Field( + description=( + "Justification category aligned with VEX: one of " + "code_not_present, protected_by_mitigating_control, vulnerable, uncertain" + )) + executive_summary: str = Field( + description=( + "3-4 sentence synthesis. Must include: 1) Final verdict, " + "2) Technical nature of flaw, 3) Why L2 context overrides L1 (if applicable)." + )) + evidence_chain: list[str] = Field( + description="Ordered list of evidence items tracing the vulnerability through phases") + affected_files: list[str] = Field( + description="Source files where vulnerable code was identified") + patch_analysis: str | None = Field( + default=None, + description="Analysis of downstream patches if any were found") + caveats: list[str] = Field( + default_factory=list, + description="Investigation gaps or uncertainties that may need manual review") + + +class CodeAgentReport(CodeAgentReportLLM): + """Final L1 Code Agent investigation report synthesizing all phases.""" + code_snippets: list[CodeSnippet] = Field( + default_factory=list, + description="Populated programmatically from patches after generation.") + + def to_markdown( + self, + vuln_id: str = "", + target_package: str = "", + version: str = "", + release: str = "", + downstream_report: DownstreamSearchReport | None = None, + ) -> str: + """Render the report as a formatted markdown string.""" + lines: list[str] = [] + + # Header with title + lines.append("# L1 Code Agent Investigation Report") + lines.append("") + + # Verdict banner based on justification label + verdict_map = { + "protected_by_mitigating_control": ("NOT VULNERABLE", "Protected by downstream patch"), + "protected_by_compiler": ("NOT VULNERABLE", "Protected by compiler hardening"), + "code_not_present": ("NOT VULNERABLE", "Vulnerable code not present"), + "code_not_reachable": ("NOT VULNERABLE", "Vulnerable code not reachable"), + "requires_environment": ("NOT VULNERABLE", "Requires specific environment"), + "vulnerable": ("VULNERABLE", "Package requires patching"), + "uncertain": ("UNCERTAIN", "Requires manual review"), + } + verdict_status, verdict_desc = verdict_map.get( + self.justification_label, + ("UNKNOWN", "Unknown status") + ) + + lines.append(f"> **Verdict: {verdict_status}** - {verdict_desc}") + lines.append("") + + # Package information table + lines.append("## Package Information") + lines.append("") + lines.append("| Field | Value |") + lines.append("|-------|-------|") + if vuln_id: + lines.append(f"| **CVE ID** | `{vuln_id}` |") + if target_package: + lines.append(f"| **Package** | `{target_package}` |") + if version: + version_str = f"{version}-{release}" if release else version + lines.append(f"| **Version** | `{version_str}` |") + lines.append(f"| **Justification** | `{self.justification_label}` |") + lines.append("") + + # Executive Summary + lines.append("---") + lines.append("") + lines.append("## Executive Summary") + lines.append("") + lines.append(self.executive_summary) + lines.append("") + + # Evidence Chain + lines.append("---") + lines.append("") + lines.append("## Evidence Chain") + lines.append("") + lines.extend(_format_interleaved_evidence( + self.evidence_chain, + downstream_report, + )) + + # Affected Files + if self.affected_files: + lines.append("---") + lines.append("") + lines.append("## Affected Files") + lines.append("") + # Separate source files from test files + source_files = [f for f in self.affected_files if "/test/" not in f and "test_" not in f] + test_files = [f for f in self.affected_files if "/test/" in f or "test_" in f] + + if source_files: + lines.append("**Source files:**") + for f in source_files: + lines.append(f"- `{f}`") + lines.append("") + + if test_files: + lines.append("**Test files:**") + for f in test_files: + lines.append(f"- `{f}`") + lines.append("") + + # Patch Analysis + if self.patch_analysis: + lines.append("---") + lines.append("") + lines.append("## Patch Analysis") + lines.append("") + lines.append(self.patch_analysis) + lines.append("") + + # Code Snippets - separate vulnerable from fix, prioritize main source files + if self.code_snippets: + lines.append("---") + lines.append("") + lines.append("## Code Comparison") + lines.append("") + + # Filter and organize snippets + vuln_snippets = [s for s in self.code_snippets if s.snippet_type == "vulnerable"] + fix_snippets = [s for s in self.code_snippets if s.snippet_type == "fix"] + + # Prioritize main source files (not test/build files) + def is_main_source(path: str) -> bool: + return "/test/" not in path and "test_" not in path and "Makefile" not in path and "CMakeLists" not in path + + main_vuln = [s for s in vuln_snippets if is_main_source(s.file_path)] + main_fix = [s for s in fix_snippets if is_main_source(s.file_path)] + + # Show main vulnerability code + if main_vuln: + lines.append("### Vulnerable Code") + lines.append("") + for snippet in main_vuln[:2]: + file_name = snippet.file_path.split("/")[-1] + lines.append(f"**File:** `{file_name}` (Line {snippet.line_number or 'N/A'})") + lines.append("") + lines.append("```c") + lines.append(snippet.code.strip()) + lines.append("```") + lines.append("") + + # Show fix code + if main_fix: + lines.append("### Fix Code") + lines.append("") + for snippet in main_fix[:2]: + file_name = snippet.file_path.split("/")[-1] + lines.append(f"**File:** `{file_name}` (Line {snippet.line_number or 'N/A'})") + lines.append("") + lines.append("```c") + lines.append(snippet.code.strip()) + lines.append("```") + lines.append("") + + # Show other snippets (test/build files) in collapsible section if any + other_vuln = [s for s in vuln_snippets if not is_main_source(s.file_path)] + other_fix = [s for s in fix_snippets if not is_main_source(s.file_path)] + + if other_vuln or other_fix: + lines.append("
") + lines.append("Additional Changes (Test/Build Files)") + lines.append("") + for snippet in other_vuln + other_fix: + file_name = snippet.file_path.split("/")[-1] + lines.append(f"**{snippet.snippet_type.title()}** - `{file_name}`") + lines.append("") + lines.append("```") + lines.append(snippet.code.strip()) + lines.append("```") + lines.append("") + lines.append("
") + lines.append("") + + # Caveats + if self.caveats: + lines.append("---") + lines.append("") + lines.append("## Caveats") + lines.append("") + for caveat in self.caveats: + lines.append(f"- {caveat}") + lines.append("") + + # Footer + lines.append("---") + lines.append("") + lines.append("*Report generated by L1 Code Agent*") + + return "\n".join(lines) + + +def format_patch_data_for_intel( + parsed_patch: ParsedPatch | None +) -> str: + """Format patch and CVE data for intelligence extraction. + + Parameters + ---------- + parsed_patch: + Parsed patch file structure (may be None if no patch available). + cve_description: + CVE description text from advisories. + + Returns + ------- + str + Formatted string suitable for the VULNERABILITY_INTEL_EXTRACTION_PROMPT. + """ + if not parsed_patch: + return "" + + lines = [f"Patch: {parsed_patch.patch_filename}", ""] + for pf in parsed_patch.files: + lines.append(f"File: {pf.target_path}") + for hunk in pf.hunks: + if hunk.removed_lines: + lines.append(" Removed (vulnerable):") + for line in hunk.removed_lines[:10]: + lines.append(f" - {line}") + if len(hunk.removed_lines) > 10: + lines.append(f" ... (+{len(hunk.removed_lines) - 10} more lines)") + if hunk.added_lines: + lines.append(" Added (fix):") + for line in hunk.added_lines[:10]: + lines.append(f" + {line}") + if len(hunk.added_lines) > 10: + lines.append(f" ... (+{len(hunk.added_lines) - 10} more lines)") + lines.append("") + + return "\n".join(lines) + + +def get_relevant_hunks(parsed_patch: ParsedPatch | None, grep_query: str) -> str: + """Extract unified diff hunks for files matching the grep target. + + Parameters + ---------- + parsed_patch: + Parsed patch file structure (may be None if no patch available). + grep_query: + The grep query string, which may include a file filter (e.g., "pattern,filename.c"). + + Returns + ------- + str + Unified diff format string with relevant hunks, or empty string if no patch/match. + """ + if not parsed_patch: + return "" + + file_pattern = None + if "," in grep_query: + file_pattern = grep_query.split(",")[-1].strip() + + hunks = [] + for pf in parsed_patch.files: + if file_pattern and file_pattern not in pf.target_path: + continue + hunks.append(f"--- a/{pf.target_path}") + hunks.append(f"+++ b/{pf.target_path}") + for hunk in pf.hunks: + for line in hunk.removed_lines: + hunks.append(f"-\t{line}") + for line in hunk.added_lines: + hunks.append(f"+\t{line}") + + return "\n".join(hunks) if hunks else "" + + +# --------------------------------------------------------------------------- +# Report formatting helpers +# --------------------------------------------------------------------------- + +MAX_SNIPPET_CHARS = 500 +L1_EXTRACTED_FACTS_EXCERPT_CHARS = 2000 + + +def _cap_text_excerpt(text: str, max_chars: int) -> tuple[str, bool]: + """Return (possibly truncated) text and whether truncation occurred.""" + t = text.strip() + if len(t) <= max_chars: + return t, False + return t[: max_chars] + "\n[… truncated …]", True + + +def _format_interleaved_evidence( + evidence_chain: list[str], + downstream_report: DownstreamSearchReport | None, + *, + max_excerpt: int = L1_EXTRACTED_FACTS_EXCERPT_CHARS, +) -> list[str]: + """Build audit-ready markdown for the Evidence Chain section. + + Structure follows the 3-pillar model for TARGET package verification: + - Status Summary table for at-a-glance verification + - Target Patch Metadata (the "What") + - Integration Evidence (the "Plan" - spec file directives) + - Execution Evidence (the "Action" - build logs) + - Source Validation (the "Result" - L1 agent findings) + """ + lines: list[str] = [] + + if downstream_report is None: + for ev in evidence_chain: + lines.append(f"- {ev}") + return lines + + d = downstream_report + + # Categorize evidence items by keywords + patch_evidence: list[str] = [] + build_evidence: list[str] = [] + code_evidence: list[str] = [] + other_evidence: list[str] = [] + + patch_keywords = ("patch", "spec", "patchn", "directive", "target", "reference") + build_keywords = ("build", "applied", "log") + code_keywords = ("code", "function", "vulnerable", "fix", "found", "source", "l1", "agent") + + for ev in evidence_chain: + ev_lower = ev.lower() + if any(kw in ev_lower for kw in patch_keywords): + patch_evidence.append(ev) + elif any(kw in ev_lower for kw in build_keywords): + build_evidence.append(ev) + elif any(kw in ev_lower for kw in code_keywords): + code_evidence.append(ev) + else: + other_evidence.append(ev) + + # Status Summary - at-a-glance verification of TARGET package (using bullets for UI compatibility) + lines.append("### Status Summary (Target Package)") + lines.append("") + patch_check = "PASS" if d.is_patch_file_available else "FAIL" + spec_check = "PASS" if d.is_patch_in_spec_file else "FAIL" + build_check = "PASS" if d.is_patch_applied_in_build else "FAIL" + lines.append(f"- **Target patch file exists:** {patch_check}") + lines.append(f"- **Referenced in target spec:** {spec_check}") + lines.append(f"- **Applied in target build:** {build_check}") + lines.append("") + + # Section 1: Target Patch Metadata + if d.patch_file_name or patch_evidence: + lines.append("### 1. Patch Metadata") + lines.append("") + if d.patch_file_name: + lines.append(f"- **Target patch file:** `{d.patch_file_name}`") + for ev in patch_evidence: + lines.append(f"- {ev}") + lines.append("") + + # Section 2: Integration Evidence (Spec File) - the "Plan" + has_integration = d.spec_patch_directives_for_cve or d.spec_changelog_cve_lines.strip() + if has_integration: + lines.append("### 2. Integration Evidence (Spec File)") + lines.append("") + + if d.spec_patch_directives_for_cve: + # Split directives into declaration and application + declarations = [line for line in d.spec_patch_directives_for_cve + if line.strip().startswith("Patch")] + applications = [line for line in d.spec_patch_directives_for_cve + if line.strip().startswith("%patch")] + + if declarations: + lines.append("**Patch declaration:**") + lines.append("") + lines.append("```ini") + lines.append("\n".join(declarations)) + lines.append("```") + lines.append("") + + if applications: + lines.append("**Patch application directive:**") + lines.append("") + lines.append("```ini") + lines.append("\n".join(applications)) + lines.append("```") + lines.append("") + + if d.spec_changelog_cve_lines.strip(): + ex, trunc = _cap_text_excerpt(d.spec_changelog_cve_lines, max_excerpt) + hdr = "**Changelog entry:**" + if trunc: + hdr += " *(truncated)*" + lines.append(hdr) + lines.append("") + lines.append("```ini") + lines.append(ex) + lines.append("```") + lines.append("") + + # Section 3: Execution Evidence (Build Log) - the "Action" + if d.build_log_patch_applied.strip() or build_evidence: + lines.append("### 3. Execution Evidence (Build Log)") + lines.append("") + + for ev in build_evidence: + lines.append(f"- {ev}") + if build_evidence: + lines.append("") + + if d.build_log_patch_applied.strip(): + ex, trunc = _cap_text_excerpt(d.build_log_patch_applied, max_excerpt) + if trunc: + lines.append("**Build output:** *(truncated)*") + else: + lines.append("**Build output:**") + lines.append("") + lines.append("```bash") + lines.append(ex) + lines.append("```") + lines.append("") + + # Section 4: Source Validation - the "Result" + if code_evidence: + lines.append("### 4. Source Validation") + lines.append("") + for ev in code_evidence: + lines.append(f"- {ev}") + lines.append("") + + # Section 5: Tarball Reference + if d.spec_version_line or d.spec_source0_line: + lines.append("### 5. Tarball Reference") + lines.append("") + if d.spec_version_line: + lines.append(f"- `{d.spec_version_line}`") + if d.spec_source0_line: + lines.append(f"- `{d.spec_source0_line}`") + lines.append("") + + # Additional evidence (uncategorized) + if other_evidence: + lines.append("### Additional Evidence") + lines.append("") + for ev in other_evidence: + lines.append(f"- {ev}") + lines.append("") + + return lines + + +def _format_extracted_facts_section( + d: DownstreamSearchReport, + *, + max_excerpt: int = L1_EXTRACTED_FACTS_EXCERPT_CHARS, +) -> list[str]: + """Build markdown lines for the deterministic *Extracted facts* block. + + .. deprecated:: + This function is deprecated. Use `_format_interleaved_evidence()` instead, + which merges Evidence Chain and Extracted facts into a single interleaved + section for better readability. + """ + warnings.warn( + "_format_extracted_facts_section is deprecated. " + "Use _format_interleaved_evidence() instead.", + DeprecationWarning, + stacklevel=2, + ) + lines: list[str] = [ + "## Extracted facts", + "", + "*Verbatim excerpts from spec/build grep and parsers. Narrative sections below are model-generated.*", + "", + ] + lines.append(f"- **Downstream patch file found:** {d.is_patch_file_available}") + if d.patch_file_name: + lines.append(f"- **Patch file name:** `{d.patch_file_name}`") + lines.append(f"- **Patch referenced in spec (CVE grep):** {d.is_patch_in_spec_file}") + lines.append(f"- **Build log shows CVE / patch application:** {d.is_patch_applied_in_build}") + lines.append("") + + if d.spec_patch_directives_for_cve: + lines.append("**Spec `PatchN:` line(s) whose patch filename contains this CVE:**") + block = "\n".join(d.spec_patch_directives_for_cve) + lines.extend(["", "```", block, "```", ""]) + else: + lines.extend(["**Spec `PatchN:` line(s) whose patch filename contains this CVE:** *None found*", ""]) + + if d.spec_changelog_cve_lines.strip(): + ex, trunc = _cap_text_excerpt(d.spec_changelog_cve_lines, max_excerpt) + sub = f" (truncated to ~{max_excerpt} chars)" if trunc else "" + lines.append(f"**%changelog line(s) mentioning this CVE:**{sub}") + lines.extend(["", "```", ex, "```", ""]) + else: + lines.extend(["**%changelog line(s) mentioning this CVE:** *No matching lines* ", ""]) + + if d.spec_file_log_change.strip(): + ex, trunc = _cap_text_excerpt(d.spec_file_log_change, max_excerpt) + hdr = "**All spec lines matching CVE grep (may include Patch, changelog, comments):**" + if trunc: + hdr += f" *({max_excerpt} char excerpt)*" + lines.append(hdr) + lines.extend(["", "```", ex, "```", ""]) + else: + lines.extend(["**All spec lines matching CVE grep:** *None*", ""]) + + if d.build_log_patch_applied.strip(): + ex, trunc = _cap_text_excerpt(d.build_log_patch_applied, max_excerpt) + hdr = "**Build log line(s) matching CVE grep:**" + if trunc: + hdr += f" *({max_excerpt} char excerpt)*" + lines.append(hdr) + lines.extend(["", "```", ex, "```", ""]) + else: + lines.extend(["**Build log line(s) matching CVE grep:** *None or build log not available* ", ""]) + + # Spec tarball reference (Source0/Version) for delivery-model context + if d.spec_version_line or d.spec_source0_line: + lines.append("**Spec tarball reference:**") + if d.spec_version_line: + lines.append(f"- `{d.spec_version_line}`") + if d.spec_source0_line: + lines.append(f"- `{d.spec_source0_line}`") + lines.append("") + + return lines + + +def _format_downstream_for_report(report: DownstreamSearchReport | None) -> str: + """Format target package analysis results for prompt injection. + + This section reports whether the TARGET package (the one being scanned) + contains a CVE-specific patch file. + """ + if report is None: + return "Target package analysis did not produce results." + + lines = [] + lines.append(f"**Target Package Patch Available:** {report.is_patch_file_available}") + + if report.is_patch_file_available: + lines.append(f"**Target Patch File:** `{report.patch_file_name}`") + lines.append(f"**Referenced in Spec:** {report.is_patch_in_spec_file}") + if report.spec_file_log_change: + lines.append(f"**Target Spec Changelog:**\n```\n{report.spec_file_log_change[:500]}\n```") + lines.append(f"**Applied in Build:** {report.is_patch_applied_in_build}") + if report.build_log_patch_applied: + lines.append(f"**Build Log Evidence:**\n```\n{report.build_log_patch_applied[:500]}\n```") + + if report.parsed_patch: + lines.append(f"\n**Parsed Patch ({len(report.parsed_patch.files)} files):**") + for pf in report.parsed_patch.files[:5]: + added = sum(len(h.added_lines) for h in pf.hunks) + removed = sum(len(h.removed_lines) for h in pf.hunks) + lines.append(f"- `{pf.target_path}` (+{added}/-{removed} lines)") + if len(report.parsed_patch.files) > 5: + lines.append(f" (+{len(report.parsed_patch.files) - 5} more files)") + else: + lines.append("No CVE-specific patch file found in target package.") + + return "\n".join(lines) + + +def _format_upstream_for_report(report: UpstreamSearchReport | None) -> str: + """Format reference intel gathering results for prompt injection. + + This section reports TWO distinct pieces of information: + 1. Rebase indicator: Checked TARGET's spec file for CVE mention + 2. Reference package: Downloaded a known-fixed package from intel to extract patch patterns + """ + if report is None: + return "Reference intel gathering did not produce results." + + lines = [] + + # Part 1: Rebase indicator (checked in TARGET's spec file) + rebase_status = report.is_code_fixed_by_rebase + if rebase_status == "unknown": + lines.append("**Target Rebase Indicator:** not found (no CVE mention in target's spec file)") + elif rebase_status == "yes": + lines.append("**Target Rebase Indicator:** found (CVE mentioned in target's spec changelog)") + else: + lines.append(f"**Target Rebase Indicator:** {rebase_status}") + + if report.spec_file_log_change: + lines.append(f"**Target Spec Changelog Match:**\n```\n{report.spec_file_log_change[:500]}\n```") + + # Part 2: Reference package (downloaded from intel for comparison) + if report.is_fixed_srpm_is_needed: + if report.reference_package_nvr: + lines.append(f"**Reference Fixed Package:** `{report.reference_package_nvr}` (from intel)") + else: + lines.append(f"**Reference Fixed Package:** Available (from intel)") + lines.append(f"**Reference Patch File:** `{report.fixed_srpm_file_name}`") + if report.fixed_parsed_patch: + lines.append(f"\n**Reference Patch ({len(report.fixed_parsed_patch.files)} files):**") + for pf in report.fixed_parsed_patch.files[:5]: + added = sum(len(h.added_lines) for h in pf.hunks) + removed = sum(len(h.removed_lines) for h in pf.hunks) + lines.append(f"- `{pf.target_path}` (+{added}/-{removed} lines)") + + + if report.reason_code_fixed_by_rebase: + lines.append(f"\n**Rebase Reasoning:** {report.reason_code_fixed_by_rebase}") + + return "\n".join(lines) + + +def _build_agent_status_line(l2_result: L2BuildResult | None) -> str: + """Explicit build-agent run state for the report LLM (avoids inferring from empty XML).""" + if l2_result is None: + return "Build agent status: not_run" + if l2_result.l2_override_verdict is not None: + return f"Build agent status: ran_with_override ({l2_result.l2_override_verdict})" + return "Build agent status: ran_no_override (code agent verdict stands)" + + +def _format_l2_for_report(l2_result: L2BuildResult | None) -> str: + """Format build agent results for prompt injection. + + Emits context whenever L2 ran, including when there is no override verdict. + """ + if l2_result is None: + return "" + + override = l2_result.l2_override_verdict or "none" + lines = [ + "", + f"**Override verdict:** {override}", + f"**Compilation status:** {l2_result.compilation_status}", + ] + + if l2_result.compilation_evidence: + lines.append(f"**Compilation evidence:** {l2_result.compilation_evidence}") + + if l2_result.hardening_flags: + flags_str = ", ".join(l2_result.hardening_flags[:10]) + if len(l2_result.hardening_flags) > 10: + flags_str += f" (+{len(l2_result.hardening_flags) - 10} more)" + lines.append(f"**Hardening flags:** {flags_str}") + + if l2_result.hardening_rationale: + lines.append(f"**Hardening rationale:** {l2_result.hardening_rationale}") + + if l2_result.hardening_relevant is not None: + lines.append(f"**Hardening relevant to CVE:** {l2_result.hardening_relevant}") + + if l2_result.evidence_sources: + lines.append(f"**Evidence sources:** {', '.join(l2_result.evidence_sources)}") + + lines.append("") + + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Report generation pipeline +# --------------------------------------------------------------------------- + +MAX_REPORT_CODE_SNIPPETS_VULNERABLE = 3 +MAX_REPORT_CODE_SNIPPETS_FIX = 3 + + +def _normalize_snippet_path(path: str) -> str: + """Stable comparison key for patch vs affected file paths.""" + p = path.strip().replace("\\", "/") + while p.startswith("./"): + p = p[2:] + if p.startswith("ab/"): + p = p[3:] + return p.lower() + + +def _snippet_matches_any_affected_path(snippet_path: str, affected_files: list[str]) -> bool: + if not affected_files: + return False + norm_snip = _normalize_snippet_path(snippet_path) + snip_base = Path(snippet_path).name.lower() + for af in affected_files: + norm_af = _normalize_snippet_path(af) + if norm_snip == norm_af: + return True + if snip_base and snip_base == Path(af).name.lower(): + return True + if norm_snip.endswith(norm_af) or norm_af.endswith(norm_snip): + return True + return False + + +def _rank_patch_snippets_for_relevance( + snippets: list[CodeSnippet], + affected_files: list[str], +) -> list[CodeSnippet]: + """Paths matching affected_files first; preserve original order within each bucket.""" + if not affected_files: + return list(snippets) + indexed = list(enumerate(snippets)) + indexed.sort( + key=lambda pair: ( + 0 if _snippet_matches_any_affected_path(pair[1].file_path, affected_files) else 1, + pair[0], + ), + ) + return [s for _, s in indexed] + + +def _cap_snippets_by_type( + snippets: list[CodeSnippet], + *, + max_vulnerable: int = MAX_REPORT_CODE_SNIPPETS_VULNERABLE, + max_fix: int = MAX_REPORT_CODE_SNIPPETS_FIX, +) -> list[CodeSnippet]: + """Keep insertion order; at most max_vulnerable vulnerable and max_fix fix snippets.""" + n_vuln = n_fix = 0 + out: list[CodeSnippet] = [] + for s in snippets: + if s.snippet_type == "vulnerable": + if n_vuln >= max_vulnerable: + continue + n_vuln += 1 + out.append(s) + elif s.snippet_type == "fix": + if n_fix >= max_fix: + continue + n_fix += 1 + out.append(s) + else: + out.append(s) + return out + + +def _extract_snippets_from_patch( + parsed_patch: ParsedPatch | None, + source: str, +) -> list[CodeSnippet]: + """Extract vulnerable/fix snippets from a parsed patch. + + For purely additive patches (no removed lines), shows context lines + as "vulnerable" since they represent the code lacking the fix. + + Args: + parsed_patch: The parsed patch to extract snippets from. + source: The source identifier for the snippets (e.g., "downstream_patch", + "upstream_patch", "git_search"). + + Returns: + List of CodeSnippet objects extracted from the patch. + """ + if not parsed_patch: + return [] + snippets: list[CodeSnippet] = [] + for pf in parsed_patch.files: + for hunk in pf.hunks: + if hunk.removed_lines: + snippets.append(CodeSnippet( + file_path=pf.clean_target_path, + line_number=hunk.source_start, + code="\n".join(hunk.removed_lines[:10]), + snippet_type="vulnerable", + source=source, + )) + elif hunk.context_lines and hunk.added_lines: + snippets.append(CodeSnippet( + file_path=pf.clean_target_path, + line_number=hunk.source_start, + code="\n".join(hunk.context_lines[:10]), + snippet_type="vulnerable", + source=source, + )) + if hunk.added_lines: + snippets.append(CodeSnippet( + file_path=pf.clean_target_path, + line_number=hunk.target_start, + code="\n".join(hunk.added_lines[:10]), + snippet_type="fix", + source=source, + )) + return snippets + + +def _extract_downstream_patch_code_snippets( + downstream_report: DownstreamSearchReport | None, +) -> list[CodeSnippet]: + """Extract vulnerable/fix snippets from the downstream parsed patch.""" + patch = downstream_report.parsed_patch if downstream_report else None + return _extract_snippets_from_patch(patch, "downstream_patch") + + +def _extract_git_search_code_snippets( + git_search_report: GitSearchReport | None, +) -> list[CodeSnippet]: + """Extract vulnerable/fix snippets from the git search parsed patch.""" + patch = git_search_report.parsed_patch if git_search_report else None + return _extract_snippets_from_patch(patch, "git_search") + + +def _extract_code_snippets( + downstream_report: DownstreamSearchReport | None, + upstream_report: UpstreamSearchReport | None, +) -> list[CodeSnippet]: + """Extract code snippets from downstream and upstream parsed patches.""" + snippets = _extract_downstream_patch_code_snippets(downstream_report) + upstream_patch = upstream_report.fixed_parsed_patch if upstream_report else None + snippets.extend(_extract_snippets_from_patch(upstream_patch, "upstream_patch")) + return snippets + + +async def extract_l1_verdict( + llm, + vuln_id: str, + target_package: str, + final_answer: str, + tracer, +) -> L1VerdictExtraction: + """Use LLM to extract structured verdict from L1 agent's final answer. + + Parameters + ---------- + llm: + LangChain LLM for verdict extraction. + vuln_id: + CVE identifier (e.g. "CVE-2026-5121"). + target_package: + Name of the package being investigated. + final_answer: + The L1 agent's final answer text. + tracer: + Request-scoped tracing context. + + Returns + ------- + L1VerdictExtraction + Structured verdict with confidence and reasoning. + """ + verdict_llm = llm.with_structured_output(L1VerdictExtraction) + prompt = L1_VERDICT_EXTRACTION_PROMPT.format( + vuln_id=vuln_id, + target_package=target_package, + final_answer=final_answer, + ) + with tracer.push_active_function("extract_l1_verdict", input_data={"vuln_id": vuln_id}) as span: + result = await verdict_llm.ainvoke([SystemMessage(content=prompt)]) + span.set_output({ + "preliminary_verdict": result.preliminary_verdict, + "confidence": result.confidence, + }) + logger.info( + "extract_l1_verdict: verdict=%s confidence=%.2f", + result.preliminary_verdict, result.confidence, + ) + return result + + +async def generate_code_agent_report( + *, + llm, + vuln_id: str, + target_package: str, + descriptions: list[tuple[str, str]], + downstream_report: DownstreamSearchReport | None, + upstream_report: UpstreamSearchReport | None, + git_search_report: GitSearchReport | None = None, + l1_agent_answer: str | None, + tracer, + policy_context: str = "", + l2_result: L2BuildResult | None = None, +) -> CodeAgentReport: + """Generate the final L1 Code Agent investigation report. + + Synthesizes results from downstream search, upstream search, L1 agent analysis, + and optionally L2 build analysis into a comprehensive, auditable report with + a clear verdict. + + Parameters + ---------- + llm: + LangChain LLM for report generation. + vuln_id: + CVE identifier (e.g. "CVE-2026-5121"). + target_package: + Name of the package being investigated. + descriptions: + ``(source_name, text)`` pairs from CVE intel. + downstream_report: + Output of downstream search (may be None). + upstream_report: + Output of upstream search (may be None). + git_search_report: + Output of git search via reference mining (may be None). Used as + third-priority source for code snippets after downstream and upstream. + l1_agent_answer: + Final answer from the L1 ReAct agent (may be None). + tracer: + Request-scoped tracing context. + policy_context: + Pre-formatted NVR posture and RHSA excerpt context for the LLM prompt. + l2_result: + Output of L2 build analysis (may be None). When present, L2 verdicts + override L1 findings as L2 analyzes actual compiled binaries. + + Returns + ------- + CodeAgentReport + Structured report with verdict, evidence, and recommendations. + """ + from langchain_core.messages import HumanMessage, SystemMessage + + cve_description = "\n".join(f"[{src}] {txt}" for src, txt in descriptions) + + downstream_section = _format_downstream_for_report(downstream_report) + upstream_section = _format_upstream_for_report(upstream_report) + l1_agent_section = l1_agent_answer or "Code agent did not produce a final answer." + l2_context_section = _format_l2_for_report(l2_result) + build_agent_status_section = ( + f"\n{_build_agent_status_line(l2_result)}\n\n" + ) + + # Generate override notice when L2 has an override verdict + if l2_result is not None and l2_result.l2_override_verdict is not None: + override_notice_section = ( + "\n" + f"BUILD AGENT OVERRIDE IN EFFECT: The build agent's verdict ({l2_result.l2_override_verdict}) " + "SUPERSEDES the code agent's source analysis.\n" + "Do NOT use 'vulnerable' as the justification_label. " + "Follow the LABEL SELECTION DECISION TREE in the instructions.\n" + "\n" + ) + else: + override_notice_section = "" + + if policy_context: + policy_context_section = ( + "\n" + + policy_context + + "\n\n" + ) + else: + policy_context_section = "" + + prompt_text = CODE_AGENT_REPORT_PROMPT.format( + vuln_id=vuln_id, + target_package=target_package, + cve_description=cve_description, + policy_context_section=policy_context_section, + downstream_section=downstream_section, + upstream_section=upstream_section, + l1_agent_section=l1_agent_section, + l2_context_section=l2_context_section, + build_agent_status_section=build_agent_status_section, + override_notice_section=override_notice_section, + ) + + report_llm = llm.with_structured_output(CodeAgentReportLLM) + + has_l2_override = l2_result is not None and l2_result.l2_override_verdict is not None + with tracer.push_active_function( + "generate_report", + input_data={ + "vuln_id": vuln_id, + "target_package": target_package, + "has_downstream_patch": downstream_report.is_patch_file_available if downstream_report else False, + "has_upstream_patch": upstream_report.is_fixed_srpm_is_needed if upstream_report else False, + "has_l1_answer": l1_agent_answer is not None, + "has_l2_override": has_l2_override, + }, + ) as span: + messages = [ + SystemMessage(content=prompt_text), + HumanMessage(content="Generate the report."), + ] + try: + llm_report: CodeAgentReportLLM = await report_llm.ainvoke(messages) + except openai.LengthFinishReasonError as e: + logger.error("generate_code_agent_report: generation cut off due to token limits") + raw_partial_content = e.completion.choices[0].message.content + logger.debug("generate_code_agent_report: raw truncated content: %s", raw_partial_content) + logger.debug("generate_code_agent_report: usage details: %s", e.completion.usage) + raise + except openai.OpenAIError as e: + logger.error("generate_code_agent_report: OpenAI error: %s", e) + raise + except Exception as e: + logger.error("generate_code_agent_report: error generating report: %s", e) + raise + + report = CodeAgentReport(**llm_report.model_dump()) + + snippet_source = "unchanged" + downstream_patch_snippet_count_pre_cap = 0 + if downstream_report and downstream_report.parsed_patch: + raw = _extract_downstream_patch_code_snippets(downstream_report) + downstream_patch_snippet_count_pre_cap = len(raw) + ranked = _rank_patch_snippets_for_relevance(raw, report.affected_files) + report.code_snippets = _cap_snippets_by_type(ranked) + snippet_source = "downstream_patch" + elif upstream_report and upstream_report.fixed_parsed_patch: + raw = _extract_code_snippets(downstream_report, upstream_report) + ranked = _rank_patch_snippets_for_relevance(raw, report.affected_files) + report.code_snippets = _cap_snippets_by_type(ranked) + snippet_source = "upstream_patch" + elif git_search_report and git_search_report.parsed_patch: + raw = _extract_git_search_code_snippets(git_search_report) + ranked = _rank_patch_snippets_for_relevance(raw, report.affected_files) + report.code_snippets = _cap_snippets_by_type(ranked) + snippet_source = "git_search" + elif not report.code_snippets: + report.code_snippets = _extract_code_snippets(downstream_report, upstream_report) + + span.set_output({ + "justification_label": report.justification_label, + "affected_files_count": len(report.affected_files), + "caveats_count": len(report.caveats), + "code_snippets_count": len(report.code_snippets), + "snippet_source": snippet_source, + "downstream_patch_snippet_count_pre_cap": downstream_patch_snippet_count_pre_cap, + }) + + logger.info( + "generate_code_agent_report: justification=%s", + report.justification_label, + ) + + return report + + +# --------------------------------------------------------------------------- +# Diff and patch helpers +# --------------------------------------------------------------------------- + + +def download_patch_and_gen_diff(fix_info: dict, brew_downloader: BrewDownloader, source_dir: Path, patch_dir: Path) -> Path | None: + """Download the patched SRPM and generate the diff file between the source and the patched SRPM.""" + from exploit_iq_commons.utils.source_rpm_downloader import SourceRPMDownloader + + srpm_path = brew_downloader.download_patched_srpm_by_nevra(fix_info["nevra"]) + if srpm_path is None: + srpm_path = brew_downloader.download_patched_srpm(fix_info["name"], fix_info["version"], fix_info["release"],) + if srpm_path is not None: + patch_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(srpm_path, patch_dir) + SourceRPMDownloader.extract_src_rpm(srpm_path, patch_dir) + + #diff_text = _generate_tree_diff(source_dir, patch_dir) + #diff_output_path = patch_dir.parent / "locate.diff" + #diff_output_path.write_text(diff_text, encoding="utf-8") + #return diff_output_path + return None + + +# --------------------------------------------------------------------------- +# Spec/build log parsing helpers +# --------------------------------------------------------------------------- + +_SPEC_PATCH_RE = re.compile(r"^Patch(\d+)\s*:\s*(.+)$", re.IGNORECASE) + + +def _parse_spec_patch_directives( + inspector, spec_path: Path, +) -> list[tuple[int, str, str]]: + """Return ``[(index, filename, raw_line), ...]`` from ``PatchN:`` lines.""" + matches = inspector.grep_content(_SPEC_PATCH_RE.pattern, spec_path) + results: list[tuple[int, str, str]] = [] + for m in matches: + hit = _SPEC_PATCH_RE.match(m.line_content.strip()) + if hit: + results.append((int(hit.group(1)), hit.group(2).strip(), m.line_content.strip())) + return results + + +def _extract_spec_changelog(inspector, spec_path: Path) -> str | None: + """Return text after the ``%changelog`` directive, or ``None``.""" + content = inspector.read_file(spec_path) + idx = content.find("%changelog") + if idx == -1: + return None + return content[idx + len("%changelog"):] + + +def find_cve_patch_in_spec( + inspector, + cve_id: str, +) -> SpecPatchMatch | None: + """Find a patch file for a CVE by searching filename or spec comments. + + Search order: + 1. Patch files with CVE in filename (fast path) + 2. PatchN: directives with CVE in the contiguous comment block above + + Returns None if no match or patch file doesn't exist. + """ + cve_pattern = re.compile(re.escape(cve_id), re.IGNORECASE) + + spec_files = inspector.find_files("*.spec", recursive=False) + if not spec_files: + return None + spec_path = spec_files[0] + + patch_files = inspector.find_files("*.patch", recursive=False) + patch_filenames = {p.name for p in patch_files} + + for pf in patch_files: + if cve_pattern.search(pf.name): + for idx, fname, raw_line in _parse_spec_patch_directives(inspector, spec_path): + if fname == pf.name: + return SpecPatchMatch( + patch_index=idx, + patch_filename=pf.name, + raw_directive=raw_line, + comment_block=None, + match_source="filename", + ) + return SpecPatchMatch( + patch_index=0, + patch_filename=pf.name, + raw_directive=f"Patch: {pf.name}", + comment_block=None, + match_source="filename", + ) + + content = inspector.read_file(spec_path) + lines = content.splitlines() + + comment_block: list[str] = [] + for line in lines: + stripped = line.strip() + if stripped.startswith("#"): + comment_block.append(stripped) + elif _SPEC_PATCH_RE.match(stripped): + hit = _SPEC_PATCH_RE.match(stripped) + if hit and comment_block: + combined_comments = "\n".join(comment_block) + if cve_pattern.search(combined_comments): + patch_filename = hit.group(2).strip() + if patch_filename in patch_filenames: + return SpecPatchMatch( + patch_index=int(hit.group(1)), + patch_filename=patch_filename, + raw_directive=stripped, + comment_block=combined_comments, + match_source="comment", + ) + comment_block = [] + else: + comment_block = [] + + return None + + +_BINARY_FILE_EXTENSIONS = frozenset({ + '.uu','.uue','.iso', '.bin', '.gz', '.bz2', '.xz', '.zip', '.tar', '.tgz', '.tbz2', + '.png', '.jpg', '.jpeg', '.gif', '.bmp', '.ico', '.webp', + '.pdf', '.doc', '.docx', '.xls', '.xlsx', + '.exe', '.dll', '.so', '.dylib', '.a', '.o', '.obj', + '.pyc', '.pyo', '.class', '.jar', '.war', + '.woff', '.woff2', '.ttf', '.otf', '.eot', + '.mp3', '.mp4', '.wav', '.avi', '.mov', '.mkv', + '.db', '.sqlite', '.sqlite3', +}) + + +def _is_binary_file_path(path: str) -> bool: + """Check if file path has a binary file extension.""" + path_lower = path.lower() + return any(path_lower.endswith(ext) for ext in _BINARY_FILE_EXTENSIONS) + + +def parse_patch_file(patch_path: Path) -> ParsedPatch | None: + """Parse a downstream .patch file into structured data. + + Returns None if the file cannot be parsed. + """ + try: + diff_text = patch_path.read_text(encoding="utf-8", errors="replace") + patch_set = PatchSet.from_string(diff_text) + except Exception: + logger.warning("parse_patch_file: failed to parse %s", patch_path) + return None + + files: list[PatchFile] = [] + for patched_file in patch_set: + if patched_file.is_binary_file: + continue + if _is_binary_file_path(patched_file.target_file): + continue + + hunks: list[PatchHunk] = [] + for hunk in patched_file: + context, removed, added = [], [], [] + for line in hunk: + if line.is_context: + context.append(str(line.value).rstrip("\n")) + elif line.is_removed: + removed.append(str(line.value).rstrip("\n")) + elif line.is_added: + added.append(str(line.value).rstrip("\n")) + + hunks.append(PatchHunk( + source_start=hunk.source_start, + source_length=hunk.source_length, + target_start=hunk.target_start, + target_length=hunk.target_length, + section_header=hunk.section_header or "", + context_lines=context, + removed_lines=removed, + added_lines=added, + )) + + files.append(PatchFile( + source_path=patched_file.source_file, + target_path=patched_file.target_file, + hunks=hunks, + is_new_file=patched_file.is_added_file, + is_deleted_file=patched_file.is_removed_file, + )) + + return ParsedPatch(patch_filename=patch_path.name, files=files) + + +# --------------------------------------------------------------------------- +# Downstream search pipeline +# --------------------------------------------------------------------------- +def _populate_downstream_spec_and_build( + report: DownstreamSearchReport, + inspector, + cve_pattern: str, + build_log_path: Path | None, +) -> None: + """Inspect target spec and build log for CVE fix evidence (runs with or without a patch file).""" + from vuln_analysis.tools.source_inspector import SourceInspector + + spec_files = inspector.find_files("*.spec", recursive=False) + spec_path = spec_files[0] if spec_files else None + + if not spec_path: + report.is_patch_in_spec_file = False + else: + cve_c = re.compile(cve_pattern, re.IGNORECASE) + for _idx, fname, raw_line in _parse_spec_patch_directives(inspector, spec_path): + if cve_c.search(fname): + report.spec_patch_directives_for_cve.append(raw_line) + chlog = _extract_spec_changelog(inspector, spec_path) + if chlog: + cve_in_cl = [ln for ln in chlog.splitlines() if cve_c.search(ln)] + report.spec_changelog_cve_lines = "\n".join(cve_in_cl) + grep_spec_matches = inspector.grep_content(cve_pattern, spec_path) + if grep_spec_matches: + report.is_patch_in_spec_file = True + report.spec_file_log_change = "\n".join(m.line_content for m in grep_spec_matches) + else: + report.is_patch_in_spec_file = False + + source0_matches = inspector.grep_content(r"^Source0:", spec_path) + if source0_matches: + report.spec_source0_line = source0_matches[0].line_content.strip() + version_matches = inspector.grep_content(r"^Version:", spec_path) + if version_matches: + report.spec_version_line = version_matches[0].line_content.strip() + + if build_log_path and build_log_path.exists(): + build_inspector = SourceInspector(build_log_path.parent) + build_log_matches = build_inspector.grep_content(cve_pattern, build_log_path) + if build_log_matches: + report.is_patch_applied_in_build = True + report.build_log_patch_applied = "\n".join(m.line_content for m in build_log_matches) + else: + report.is_patch_applied_in_build = False + else: + report.is_patch_applied_in_build = False + + +async def downstream_search_preprocss( + *, + llm, + vuln_id: str, + descriptions: list[tuple[str, str]], + source_path: Path, + build_log_path: Path | None, + tracer, +) -> DownstreamSearchReport: + """Build the downstream search pipeline.""" + from vuln_analysis.tools.source_inspector import SourceInspector + inspector = SourceInspector(source_path) + + cve_pattern = re.escape(vuln_id) + report = DownstreamSearchReport() + patch_file = None + with tracer.push_active_function("Is_patch_file_available", input_data={"vuln_id": vuln_id}) as span: + spec_match = find_cve_patch_in_spec(inspector, vuln_id) + if spec_match: + report.is_patch_file_available = True + patch_file = inspector.root / spec_match.patch_filename + report.patch_file_name = spec_match.patch_filename + if spec_match.match_source == "comment": + report.spec_patch_directives_for_cve.append(spec_match.raw_directive) + span.set_output({ + "patch_found": True, + "match_source": spec_match.match_source, + "patch_filename": spec_match.patch_filename, + }) + else: + report.is_patch_file_available = False + span.set_output({"patch_found": False}) + + with tracer.push_active_function( + "Is_patch_in_spec_file", + input_data={"patch_file_name": report.patch_file_name or "none"}, + ) as span: + _populate_downstream_spec_and_build(report, inspector, cve_pattern, build_log_path) + + if patch_file: + with tracer.push_active_function( + "Extract_patch_details", input_data={"patch_file_name": patch_file.name} + ) as span: + details = parse_patch_file(patch_file) + report.parsed_patch = details if details else None + + return report + +async def upstream_search_preprocess( + *, + vuln_id: str, + source_path: Path, + fix_info: dict, + brew_downloader: BrewDownloader, + patch_dir: Path, + target_package: TargetPackage, + tracer, + intel: list | None = None, + commit_url_candidates: dict[str, list[str]] | None = None, + cve_description: str | None = None, + llm: "BaseChatModel | None" = None, +) -> UpstreamSearchReport: + """Build the upstream search pipeline. + + Args: + intel: Optional list of CveIntel objects for Ubuntu patch lookup. + commit_url_candidates: Optional dict of URLs from intel references for patch fetching. + cve_description: Optional CVE description for Chromium CL disambiguation. + llm: Optional LangChain LLM for Chromium CL selection when multiple MERGED CLs exist. + """ + from vuln_analysis.tools.source_inspector import SourceInspector + inspector = SourceInspector(source_path) + report = UpstreamSearchReport() + cve_pattern = re.escape(vuln_id) + need_to_find_code = True + # Store reference package NVR from fix_info if available + if fix_info and fix_info.get("nevra"): + report.reference_package_nvr = fix_info["nevra"] + + with tracer.push_active_function("Is_upstream_fixed_by_rebase", input_data={"vuln_id": vuln_id}) as span: + spec_files = inspector.find_files("*.spec", recursive=False) + spec_path = spec_files[0] if spec_files else None + + if not spec_path: + report.is_code_fixed_by_rebase = "unknown" + else: + grep_spec_matches = inspector.grep_content(cve_pattern, spec_path) + if grep_spec_matches: + report.is_code_fixed_by_rebase = "yes" + report.spec_file_log_change = "\n".join(m.line_content for m in grep_spec_matches) + else: + report.is_code_fixed_by_rebase = "unknown" + span.set_output({ + "is_code_fixed_by_rebase": report.is_code_fixed_by_rebase, + "spec_file_log_change": report.spec_file_log_change, + }) + + if patch_dir.exists(): + shutil.rmtree(patch_dir, ignore_errors=True) + + if fix_info and brew_downloader is not None: + with tracer.push_active_function( + "download_rpm_patch", input_data={"fix_info": fix_info} + ) as span: + try: + download_patch_and_gen_diff(fix_info, brew_downloader, source_path, patch_dir) + span.set_output({"patch_dir_exists": patch_dir.exists()}) + except Exception as e: + logger.warning("locate: failed to download/extract patched SRPM: %s", e) + span.set_output({"error": str(e), "patch_dir_exists": False}) + + if patch_dir.exists(): + patch_inspector = SourceInspector(patch_dir) + with tracer.push_active_function("is_patch_downsteam_patch_file", input_data={"patch_dir": patch_dir}) as span: + spec_match = find_cve_patch_in_spec(patch_inspector, vuln_id) + if spec_match: + report.is_fixed_srpm_is_needed = True + report.fixed_srpm_file_name = spec_match.patch_filename + report.fixed_parsed_patch = parse_patch_file(patch_inspector.root / spec_match.patch_filename) + span.set_output({ + "is_fixed_srpm_is_needed": True, + "match_source": spec_match.match_source, + "patch_filename": spec_match.patch_filename, + }) + return report + else: + report.is_fixed_srpm_is_needed = False + span.set_output({ + "is_fixed_srpm_is_needed": report.is_fixed_srpm_is_needed}) + + # Try intel references + OSV fallback + if (not patch_dir.exists() or need_to_find_code) and not report.fixed_parsed_patch: + from vuln_analysis.utils.web_patch_fetcher import fetch_patch_for_cve + with tracer.push_active_function( + "fetch_patch_for_cve", + input_data={ + "vuln_id": vuln_id, + "candidates_count": {src: len(urls) for src, urls in commit_url_candidates.items()} if commit_url_candidates else {}, + } + ) as span: + async with aiohttp.ClientSession() as session: + result = await fetch_patch_for_cve( + session=session, + candidates=commit_url_candidates or {}, + vuln_id=vuln_id, + cve_description=cve_description, + llm=llm, + upstream_version=target_package.version, + package_name=target_package.name, + ) + if result and result.parsed_patch: + report.fixed_parsed_patch = result.parsed_patch + report.fixed_srpm_file_name = result.patch_url + report.is_fixed_srpm_is_needed = True + report.osv_result = result + span.set_output({ + "source_found": result.source, + "url_type": result.url_type, + "platform": result.platform, + "patch_url": result.patch_url, + "commit_message": result.commit_message, + "patch_found": True, + }) + else: + span.set_output({"patch_found": False}) + return report + diff --git a/src/vuln_analysis/functions/code_understanding_agent.py b/src/vuln_analysis/functions/code_understanding_agent.py new file mode 100644 index 000000000..2cd7e5075 --- /dev/null +++ b/src/vuln_analysis/functions/code_understanding_agent.py @@ -0,0 +1,189 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from nat.builder.context import Context +from vuln_analysis.functions.agent_registry import register_agent +from vuln_analysis.functions.base_graph_agent import BaseGraphAgent, _is_tool_available + +from vuln_analysis.functions.react_internals import ( + AgentState, + Observation, +) +from vuln_analysis.functions.code_understanding_internals import ( + CU_AGENT_SYS_PROMPT, + CU_AGENT_THOUGHT_INSTRUCTIONS, + CodeUnderstandingRulesTracker, +) +from vuln_analysis.utils.code_understanding_prompt_factory import ( + CU_TOOL_SELECTION_STRATEGY, + CU_TOOL_GENERAL_DESCRIPTIONS, +) +from vuln_analysis.utils.intel_utils import build_critical_context +from vuln_analysis.runtime_context import ctx_state, cu_source_scope +from vuln_analysis.tools.tool_names import ToolNames + +logger = LoggingFactory.get_agent_logger(__name__) +AGENT_TRACER = Context.get() + + +def _build_cu_system_prompt(descriptions: str, tool_guidance: str) -> str: + return f"""{CU_AGENT_SYS_PROMPT} + + +{descriptions} + + + +{tool_guidance} + + +{CU_AGENT_THOUGHT_INSTRUCTIONS} + +RESPONSE: +{{""" + + +def _build_cu_tool_guidance(ecosystem: str, available_tools: list) -> tuple[str, str]: + tool_names = [t.name for t in available_tools] + tool_desc_lines = [f"{t.name}: {t.description}" for t in available_tools] + + lang = ecosystem.lower() if ecosystem else "" + if lang in CU_TOOL_SELECTION_STRATEGY: + logger.debug("Using %s-specific tool strategy for Code Understanding agent", lang) + guidance = CU_TOOL_SELECTION_STRATEGY[lang] + else: + logger.debug("No ecosystem-specific strategy for '%s', using generic Code Understanding tool guidance", lang) + guidance_list = [] + for name in tool_names: + if name in CU_TOOL_GENERAL_DESCRIPTIONS: + guidance_list.append(f"{name}: {CU_TOOL_GENERAL_DESCRIPTIONS[name]}") + guidance = "\n".join(guidance_list) if guidance_list else "Use the available tools to investigate the question." + + descriptions = "\n".join(tool_desc_lines) + return guidance, descriptions + + +@register_agent("code_understanding") +class CodeUnderstandingAgent(BaseGraphAgent): + + @property + def agent_type(self) -> str: + return "cu" + + _CU_TOOLS = frozenset({ + ToolNames.DOCS_SEMANTIC_SEARCH, + ToolNames.CODE_KEYWORD_SEARCH, + ToolNames.CONFIGURATION_SCANNER, + ToolNames.IMPORT_USAGE_ANALYZER, + }) + + @staticmethod + def get_tools(builder, config, state) -> list: + all_tools = BaseGraphAgent._load_all_tools(builder, config) + return [ + t for t in all_tools + if t.name in CodeUnderstandingAgent._CU_TOOLS + and _is_tool_available(t.name, config, state) + ] + + @staticmethod + def create_rules_tracker() -> CodeUnderstandingRulesTracker: + return CodeUnderstandingRulesTracker() + + def build_comprehension_context(self, state: AgentState) -> str: + """Return minimal grounding context to prevent CVE ID hallucination. + + The CU agent's comprehension LLM only needs the CVE ID and package name + to ground its findings. Feeding the full GHSA/NVD context activates + parametric knowledge and causes the model to inject CVE IDs from training. + """ + workflow_state = ctx_state.get() + vuln_id = workflow_state.cve_intel[0].vuln_id if workflow_state.cve_intel else "unknown" + package = state.get("app_package") or "unknown" + return ( + f"Investigating {vuln_id} in package {package}.\n" + "Only extract facts explicitly stated in the tool output. " + "Do not add CVE IDs, vulnerability names, or advisory details " + "from your own knowledge." + ) + + async def pre_process_node(self, state: AgentState) -> AgentState: + workflow_state = ctx_state.get() + ecosystem = ( + workflow_state.original_input.input.image.ecosystem.value + if workflow_state.original_input.input.image.ecosystem + else "" + ) + with AGENT_TRACER.push_active_function("cu_pre_process", input_data=f"ecosystem:{ecosystem}") as span: + try: + precomputed = state.get("precomputed_intel") + if precomputed is not None: + critical_context = list(precomputed[0]) + candidate_packages = [dict(p) for p in precomputed[1]] + else: + critical_context, candidate_packages, _ = build_critical_context( + workflow_state.cve_intel + ) + + critical_context, selected_package = await self._select_package( + ecosystem, candidate_packages, critical_context, workflow_state, + ) + + critical_context.append( + "TASK: Investigate usage, configuration, and presence of the vulnerable " + "component in the container. Focus on how the component is used, " + "not on call-chain reachability." + ) + + scope_parts = [] + image_input = workflow_state.original_input.input.image + if image_input.source_info: + for si in image_input.source_info: + if si.git_repo: + repo_name = si.git_repo.rstrip("/").rsplit("/", 1)[-1] + if repo_name.endswith(".git"): + repo_name = repo_name[:-4] + scope_parts.append(repo_name) + if selected_package: + scope_parts.append(selected_package) + cu_source_scope.set(scope_parts if scope_parts else None) + logger.debug("Code Understanding source scope set to: %s", scope_parts) + + tool_guidance, descriptions = _build_cu_tool_guidance(ecosystem, self.tools) + runtime_prompt = _build_cu_system_prompt(descriptions, tool_guidance) + active_tool_names = [t.name for t in self.tools] + + rules_tracker = state.get("rules_tracker") + rules_tracker.set_target_package(selected_package) + rules_tracker.set_allowed_tools(active_tool_names) + + span.set_output({ + "selected_package": selected_package, + "agent_type": "code_understanding", + }) + + return { + "ecosystem": ecosystem, + "runtime_prompt": runtime_prompt, + "is_reachability": "no", + "observation": Observation(memory=[], results=[]), + "critical_context": critical_context, + "app_package": selected_package, + } + except Exception as e: + logger.exception("cu_pre_process_node failed") + span.set_output({"error": str(e), "exception_type": type(e).__name__}) + raise diff --git a/src/vuln_analysis/functions/code_understanding_internals.py b/src/vuln_analysis/functions/code_understanding_internals.py new file mode 100644 index 000000000..e6fc2bd2a --- /dev/null +++ b/src/vuln_analysis/functions/code_understanding_internals.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.functions.react_internals import BaseRulesTracker + +logger = LoggingFactory.get_agent_logger(__name__) + + +class CodeUnderstandingRulesTracker(BaseRulesTracker): + """Behavioral rules for the Code Understanding sub-agent.""" + + def check_thought_behavior(self, action: str, action_input: str, output) -> tuple[bool, str]: + """Check all Code Understanding rules in priority order. + + Returns (True, error_message) if a rule is violated, (False, '') otherwise. + Rule check order: duplicate call -> Rule 7 (dotted keywords) -> allowed tools. + """ + if self._rule_duplicate_call(action, action_input): + logger.debug("CU duplicate call rule triggered: '%s' with same input", action) + return True, ( + f"You already called {action} with this exact input. " + "You MUST use a DIFFERENT tool or a DIFFERENT input query. " + "Check KNOWLEDGE for what was already tried." + ) + if self._rule_number_7(action, action_input, output): + logger.debug("Code Understanding Rule 7 triggered: dotted query with empty results for tool '%s'", action) + return True, ( + "You are NOT following Rule 7. Your query contains dots and returned " + "no results. You MUST retry with just the final component. Follow the rules." + ) + if self._rule_use_allowed_tools(action): + logger.debug("Code Understanding allowed-tools rule triggered: '%s' not in %s", action, self.allowed_tools) + return True, ( + f"You are NOT following AVAILABLE_TOOLS. You MUST use the allowed tools " + f"{self.allowed_tools}. Follow the rules." + ) + self.add_action(action, action_input, output) + return False, "" + + +CU_AGENT_SYS_PROMPT = ( + "You are a security analyst investigating a code understanding question about a CVE vulnerability.\n" + "Your goal is to collect evidence about how the codebase uses, configures, or depends on " + "the affected component. This is NOT a reachability question -- do NOT trace call chains.\n" + "GENERAL RULES:\n" + "- Base conclusions ONLY on tool results, not assumptions.\n" + "- If a search returns no results, that is evidence the component is absent or not configured.\n" + "- DISTINGUISH where findings come from: main application code vs. dependency libraries.\n" + "- A component being present in dependencies does NOT mean the application uses it.\n" + "- Configuration in framework dependencies (e.g., Spring defaults) IS relevant evidence.\n" + "- Import in an intermediate library (e.g., Spring imports XStream, app imports Spring) IS relevant.\n" + "ANSWER QUALITY:\n" + "- Answer the SPECIFIC question asked with evidence. Do NOT just report what tools found.\n" + "- Always state: WHAT you checked, WHAT you found, and WHY it leads to your conclusion.\n" + "- If tool results conflict, state the conflict explicitly.\n" + "- When citing evidence, explain HOW it relates to the question." +) + +CU_AGENT_THOUGHT_INSTRUCTIONS = """ +1. Output valid JSON only. thought < 100 words. final_answer < 150 words. +2. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. +3. Docs Semantic Search, Code Keyword Search: use query field. +4. Configuration Scanner: use query field with keywords describing what to look for. +5. Import Usage Analyzer: use query field with the package/module name. +6. Do NOT call the same tool with the same input twice. Check KNOWLEDGE for "CALLED:" entries. +7. If Code Keyword Search returns no results and the query contains dots, retry with just the final component. +8. Before concluding, if the question involves a specific library or package, you MUST use Import Usage Analyzer to check how it is imported and used across sources. +9. GUIDELINE: Synthesize findings across documentation, code, configuration, and imports before drawing conclusions. + + +{{"thought": "Check configuration files for security settings related to the vulnerability", "mode": "act", "actions": {{"tool": "Configuration Scanner", "package_name": null, "function_name": null, "query": "deserialization allowlist denylist security", "tool_input": null, "reason": "Check if vulnerable feature is enabled or disabled in config"}}, "final_answer": null}} + + +{{"thought": "Search source code for imports of the vulnerable library", "mode": "act", "actions": {{"tool": "Code Keyword Search", "package_name": null, "function_name": null, "query": "import vulnerable_library", "tool_input": null, "reason": "Find where the package is imported in code"}}, "final_answer": null}} + + +{{"thought": "Check how the vulnerable library is imported and used across the codebase", "mode": "act", "actions": {{"tool": "Import Usage Analyzer", "package_name": null, "function_name": null, "query": "vulnerable_library", "tool_input": null, "reason": "Find all imports and usage sites of the vulnerable package"}}, "final_answer": null}} +""" \ No newline at end of file diff --git a/src/vuln_analysis/functions/cve_agent.py b/src/vuln_analysis/functions/cve_agent.py index 29dd006fd..ac2a1d00b 100644 --- a/src/vuln_analysis/functions/cve_agent.py +++ b/src/vuln_analysis/functions/cve_agent.py @@ -14,28 +14,32 @@ # limitations under the License. import asyncio -import contextvars -import logging -import typing from aiq.builder.builder import Builder from aiq.builder.framework_enum import LLMFrameworkEnum from aiq.builder.function_info import FunctionInfo from aiq.cli.register_workflow import register_function from aiq.data_models.function import FunctionBaseConfig -from langchain.agents import AgentExecutor -from langchain.agents import create_react_agent -from langchain.agents.agent import RunnableAgent + from langchain_core.exceptions import OutputParserException -from langchain_core.prompts import PromptTemplate +from langchain_core.messages import HumanMessage from pydantic import Field from vuln_analysis.data_models.state import AgentMorpheusEngineState -from vuln_analysis.utils.prompting import get_agent_prompt +from vuln_analysis.utils.error_handling_decorator import ToolRaisedException +from vuln_analysis.utils.intel_utils import build_critical_context, enrich_go_candidates, enrich_vulnerable_functions_from_patch +from vuln_analysis.runtime_context import ctx_state +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id +from nat.builder.context import Context -logger = logging.getLogger(__name__) +from vuln_analysis.functions.agent_registry import get_agent_class, get_all_agent_types +from vuln_analysis.functions.dispatcher import QuestionRouting, dispatch_question +# Import agent modules to trigger @register_agent decorators +import vuln_analysis.functions.reachability_agent # noqa: F401 +import vuln_analysis.functions.code_understanding_agent # noqa: F401 -ctx_state = contextvars.ContextVar("ctx_state", default="default_value") +logger = LoggingFactory.get_agent_logger(__name__) +AGENT_TRACER = Context.get() class CVEAgentExecutorToolConfig(FunctionBaseConfig, name="cve_agent_executor"): @@ -61,120 +65,211 @@ class CVEAgentExecutorToolConfig(FunctionBaseConfig, name="cve_agent_executor"): default=False, description= "Controls whether to return intermediate steps taken by the agent, and include them in the output file.") + transitive_search_tool_enabled: bool = Field(default=True, + description="Whether to enable transitive code search or not.") + cve_web_search_enabled: bool = Field(default=True, + description="Whether to enable CVE Web Search tool or not.") verbose: bool = Field(default=False, description="Set to true for verbose output") + context_window_token_limit: int = Field( + default=5000, + description="Estimated token threshold for pruning old messages in observation node." + ) + uber_jar_file_threshold: int = Field( + default=600, + description="Source file count above which a JAR is treated as an uber-JAR, " + "disabling same-artifact optimizations to prevent O(N^2) type resolution." + ) + + +async def _process_steps(agents: dict, routing_llm, steps, semaphore, max_iterations: int = 10, *, vuln_id: str): + workflow_state = ctx_state.get() + critical_context, candidate_packages, vulnerable_functions = build_critical_context(workflow_state.cve_intel) + + ecosystem = (workflow_state.original_input.input.image.ecosystem.value + if workflow_state.original_input.input.image.ecosystem else "") + if ecosystem == "go": + candidate_packages, vulnerable_functions = await enrich_go_candidates( + workflow_state.cve_intel, + workflow_state.original_input.input.image.source_info, + critical_context, candidate_packages, set(vulnerable_functions), + ) + + if not vulnerable_functions: + vf_set: set[str] = set() + pre_fetched_patch = workflow_state.patch_results.get(vuln_id) + await enrich_vulnerable_functions_from_patch( + workflow_state.cve_intel, critical_context, vf_set, ecosystem, + patch_result=pre_fetched_patch, + ) + if vf_set: + vulnerable_functions = sorted(vf_set) + + context_block = "\n".join(critical_context) + precomputed_intel = (critical_context, candidate_packages, vulnerable_functions) + + routings = await asyncio.gather( + *(dispatch_question(routing_llm, step, context_block) for step in steps), + return_exceptions=True, + ) + default_routing = QuestionRouting( + agent_type="reachability", + reason="Fallback: routing failed, defaulting to reachability agent.", + ) + for i, r in enumerate(routings): + if isinstance(r, Exception): + logger.error("dispatch_question failed for step %d, falling back to reachability: %s", i, r) + routings[i] = default_routing + routed_steps = list(zip(steps, routings)) + + has_reachability = any(r.agent_type == "reachability" for r in routings) + if not has_reachability and "reachability" in agents and candidate_packages: + pkg_name = candidate_packages[0].get("name", "") + if pkg_name: + synthetic_q = ( + f"Is the code from the vulnerable package '{pkg_name}' actually " + f"imported, called, or reachable from the application's own source code?" + ) + synthetic_routing = QuestionRouting( + agent_type="reachability", + reason="Injected: no checklist question targeted reachability — " + "verifying whether vulnerable code is actually used.", + ) + routed_steps.append((synthetic_q, synthetic_routing)) + logger.info( + "No reachability questions in checklist; injected synthetic question for '%s'", + pkg_name, + ) + + async def _execute_step(step, routing): + agent_type = routing.agent_type + actual_type = agent_type if agent_type in agents else "reachability" + compiled_graph = agents[actual_type] + tracker = get_agent_class(actual_type).create_rules_tracker() + + initial_state = { + "input": step, + "messages": [HumanMessage(content=step)], + "step": 0, + "max_steps": max_iterations, + "thought": None, + "observation": None, + "output": "waiting for the agent to respond", + "rules_tracker": tracker, + "precomputed_intel": precomputed_intel, + } + graph_config = {"recursion_limit": 50} + + with AGENT_TRACER.push_active_function( + "checklist_question", + input_data=f"[{actual_type}] {step[:80]}", + ): + if semaphore: + async with semaphore: + return await compiled_graph.ainvoke(initial_state, config=graph_config) + else: + return await compiled_graph.ainvoke(initial_state, config=graph_config) + raw_results = await asyncio.gather( + *(_execute_step(step, routing) for step, routing in routed_steps), + return_exceptions=True, + ) + questions = [step for step, _ in routed_steps] + return raw_results, questions -async def _create_agent(config: CVEAgentExecutorToolConfig, builder: Builder, - state: AgentMorpheusEngineState) -> AgentExecutor: - tools = builder.get_tools(tool_names=config.tool_names, wrapper_type=LLMFrameworkEnum.LANGCHAIN) - llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) - prompt = PromptTemplate.from_template(get_agent_prompt(config.prompt, config.prompt_examples)) - - # Filter tools that are not available - tools = [ - tool for tool in tools - if not ((tool.name == "Container Image Code QA System" and state.code_vdb_path is None) or - (tool.name == "Container Image Developer Guide QA System" and state.doc_vdb_path is None) or - (tool.name == "Lexical Search Container Image Code QA System" and state.code_index_path is None)) - ] - - agent = create_react_agent(llm=llm, - tools=tools, - prompt=prompt, - stop_sequence=["\nObservation:", "\n\tObservation:"]) - - agent_executor = AgentExecutor( - agent=agent, - tools=tools, - early_stopping_method="force", - handle_parsing_errors="Check your output and make sure it conforms, use the Action/Action Input syntax", - max_iterations=config.max_iterations, - return_intermediate_steps=config.return_intermediate_steps, - verbose=config.verbose) - - # Disable streaming for accurate token counts - if isinstance(agent_executor.agent, RunnableAgent): - agent_executor.agent.stream_runnable = False - - return agent_executor - - -async def _process_steps(agent, steps, semaphore): - - async def _process_step(step): - if semaphore: - async with semaphore: - return await agent.ainvoke({"input": step}) - else: - return await agent.ainvoke({"input": step}) - - return await asyncio.gather(*(_process_step(step) for step in steps)) - - -def _parse_intermediate_step(step: tuple[typing.Any, typing.Any]) -> dict[str, typing.Any]: - """ - Parse an agent intermediate step into an AgentIntermediateStep object. Return the dictionary representation for - compatibility with cudf. - """ - if len(step) != 2: - raise ValueError(f"Expected 2 values in each intermediate step but got {len(step)}.") - - action, output = step - - return {"tool_name": action.tool, "action_log": action.log, "tool_input": action.tool_input, "tool_output": output} +def _postprocess_results(results: list[tuple], replace_exceptions: bool, replace_exceptions_value: str | None, + checklist_questions: list[list]) -> list[list[dict]]: + """Post-process graph agent results into a uniform list of output dicts. -def _postprocess_results(results: list[list[dict]], replace_exceptions: bool, - replace_exceptions_value: str | None) -> tuple[list[list[str]], list[list[list]]]: - """ - Post-process results into lists of outputs and intermediate steps. Replace exceptions with placholder values if - config.replace_exceptions = True. + Replaces exceptions with placeholder values if replace_exceptions is True. + Each entry in *results* is a ``(raw_results, questions)`` tuple returned by + ``_process_steps``, where *questions* contains the actual question text + (including any synthetic questions that were injected). """ - - for i, answer_list in enumerate(results): + outputs = [[] for _ in range(len(results))] + for i, result_entry in enumerate(results): + if isinstance(result_entry, Exception): + logger.warning("_process_steps failed entirely for group %d: %s", i, result_entry) + continue + answer_list, questions = result_entry for j, answer in enumerate(answer_list): - # Handle exceptions returned by the agent - # OutputParserException is not a subclass of Exception, so we need to check for it separately - if isinstance(answer, (OutputParserException, Exception)): + if isinstance(answer, (ToolRaisedException, OutputParserException, Exception)): if replace_exceptions: - # If the agent encounters a parsing error or a server error after retries, replace the error - # with default values to prevent the pipeline from crashing - results[i][j]["output"] = replace_exceptions_value - results[i][j]["intermediate_step"] = None - logger.warning( - "Exception encountered in result[%d][%d]: %s. " - "Replacing with default output: \"%s\" and intermediate_steps: None", - i, - j, - answer, - replace_exceptions_value) - - # For successful agent responses, extract the output, and intermediate steps if available - else: - # intermediate_steps availability depends on config.return_intermediate_steps - if "intermediate_steps" in answer: - results[i][j]["intermediate_steps"] = [ - _parse_intermediate_step(step) for step in answer["intermediate_steps"] - ] - else: - results[i][j]["intermediate_steps"] = None + question_text = questions[j] if j < len(questions) else checklist_questions[i][j] + outputs[i].append({"input": question_text, "output": replace_exceptions_value, + "intermediate_steps": None, "cca_results": [], + "package_validated": None, + "is_reachability": None}) + if isinstance(answer, ToolRaisedException): + logger.warning( + "Tool execution exception in result[%d][%d], replacing with default output: %s", + i, j, type(answer).__name__) + else: + logger.warning( + "General exception in result[%d][%d]: %s, replacing with default output", + i, j, type(answer).__name__) - return results + else: + outputs[i].append({"input": answer["input"], "output": answer["output"], + "intermediate_steps": None, + "cca_results": answer.get("cca_results", []), + "package_validated": answer.get("package_validated"), + "is_reachability": answer.get("is_reachability")}) + return outputs @register_function(config_type=CVEAgentExecutorToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def cve_agent(config: CVEAgentExecutorToolConfig, builder: Builder): - semaphore = asyncio.Semaphore(config.max_concurrency) if config.max_concurrency else None async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: + trace_id.set(state.original_input.input.scan.id) + state.uber_jar_file_threshold = config.uber_jar_file_threshold ctx_state.set(state) - agent = await _create_agent(config, builder, state) - results = await asyncio.gather(*(_process_steps(agent, steps, semaphore) - for steps in state.checklist_plans.values())) - results = _postprocess_results(results, config.replace_exceptions, config.replace_exceptions_value) - state.checklist_results = dict(zip(state.checklist_plans.keys(), results)) + + checklist_plans = state.checklist_plans + + llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + + agents = {} + for agent_type in get_all_agent_types(): + agent_cls = get_agent_class(agent_type) + tools = agent_cls.get_tools(builder, config, state) + if tools: + agent_instance = agent_cls(tools=tools, llm=llm, config=config) + agents[agent_type] = await agent_instance.build_graph() + + routing_llm = llm.with_structured_output(QuestionRouting) + + results = await asyncio.gather( + *( + _process_steps( + agents, + routing_llm, + steps, + semaphore, + config.max_iterations, + vuln_id=vid, + ) + for vid, steps in checklist_plans.items() + ), + return_exceptions=True, + ) + results = _postprocess_results( + results, + config.replace_exceptions, + config.replace_exceptions_value, + list(checklist_plans.values()), + ) + state.checklist_results = dict(zip(checklist_plans.keys(), results)) + + with AGENT_TRACER.push_active_function("agent_finish", input_data={ + "cve_count": len(checklist_plans), + "scan_id": state.original_input.input.scan.id, + }): + pass + return state yield FunctionInfo.from_fn( diff --git a/src/vuln_analysis/functions/cve_build_agent.py b/src/vuln_analysis/functions/cve_build_agent.py new file mode 100644 index 000000000..ad52e3709 --- /dev/null +++ b/src/vuln_analysis/functions/cve_build_agent.py @@ -0,0 +1,853 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Level 2 Build Agent for Package Vulnerability Checker. + +Performs BuildCompilationCheck (Phase 1) and HardeningCheck (Phase 2) to +determine if vulnerable code identified by L1 is actually compiled into +the binary and whether hardening flags provide mitigation. +""" + +from pathlib import Path +from enum import StrEnum + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.data_models.checker_status import L2BuildResult + +from langgraph.graph import StateGraph, START, END +from langgraph.prebuilt import ToolNode +from langchain_core.messages import HumanMessage, AIMessage, SystemMessage, RemoveMessage + +from nat.builder.context import Context +from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput + +from vuln_analysis.functions.react_internals import ( + CheckerThought, + CodeFindings, + Observation, + FORCED_FINISH_PROMPT, + check_empty_output, + invoke_comprehension, + CheckerSearchTracker, +) + +from vuln_analysis.functions.build_agent_graph_defs import ( + BuildAgentState, + BuildHarvestReport, + harvest_build_data, + L2CompileVerdictExtraction, + L2HardeningVerdictExtraction, +) +from vuln_analysis.utils.rpm_checker_prompts import ( + L2_CONFIG_PROMPT_TEMPLATE, + L2_CONFIG_SYS_PROMPT, + L2_CONFIG_THOUGHT_INSTRUCTIONS, + L2_CONFIG_PROMPT_SPEC_ONLY_TEMPLATE, + L2_CONFIG_SPEC_ONLY_SYS_PROMPT, + L2_CONFIG_SPEC_ONLY_THOUGHT_INSTRUCTIONS, + L2_COMPREHENSION_PROMPT, + L2_MEMORY_UPDATE_PROMPT, + L2_HARDENING_PROMPT_TEMPLATE, + L2_HARDENING_SYS_PROMPT, + L2_HARDENING_THOUGHT_INSTRUCTIONS, + L2_COMPILATION_VERDICT_PROMPT, + L2_HARDENING_VERDICT_PROMPT, + L2_KERNEL_CONFIG_SYS_PROMPT, + L2_KERNEL_CONFIG_PROMPT_TEMPLATE, + L2_KERNEL_THOUGHT_INSTRUCTIONS, +) +from vuln_analysis.runtime_context import ctx_state +from vuln_analysis.utils.token_utils import truncate_tool_output, truncate_tool_output_list +import uuid +import tiktoken +logger = LoggingFactory.get_agent_logger(__name__) + + +class CVEBuildAgentConfig(FunctionBaseConfig, name="cve_build_agent"): + """ + Level 2 Build Agent. Analyzes build artifacts to determine if vulnerable + code is compiled into the binary and whether hardening flags mitigate. + + Phase 1: BuildCompilationCheck - Is vulnerable code compiled? + Phase 2: HardeningCheck - Do hardening flags mitigate the CVE? + """ + + base_checker_dir: str = Field( + default=".cache/am_cache/checker", + description="Root directory for checker-specific artifacts.", + ) + max_iterations: int = Field( + default=5, + description="The maximum number of iterations for the agent.", + ) + llm_name: str = Field(description="The LLM model to use with the L1 code agent.") + tool_names: list[str] = Field(default=[], description="The list of tools to provide to L1 code agent") + context_window_token_limit: int = Field(default=5000, description="Token limit for context window before pruning old messages.") + +def _build_tool_strategy(tool_names: list[str]) -> str: + """Generate tool usage guidance based on available tools.""" + strategies = [] + tool_names_lower = [t.lower().replace("_", " ") for t in tool_names] + + if any("grep" in t for t in tool_names_lower): + strategies.append("- Use Source Grep for exact code patterns from patch (function names, variable names, specific code)") + if any("keyword" in t or "search" in t for t in tool_names_lower): + strategies.append("- Use Code Keyword Search for broader concept searches when grep fails") + if any("read" in t for t in tool_names_lower): + strategies.append("- Use Read File to examine full context around matches") + + return "\n".join(strategies) if strategies else "Use available tools to search for vulnerable and fixed code patterns." + + +class L2InvestigationPhase(StrEnum): + CONFIGURATION = "configuration" + HARDENING = "hardening" + + +async def create_graph_build_agent( + config: CVEBuildAgentConfig, + builder: Builder, + state: AgentMorpheusEngineInput, + tracer, +): + """Build the L2 Build Agent LangGraph. + + Graph structure: + START -> data_harvest_node -> thought_node -+-> END (finish) + | + +-> tool_node -> observation_node -> thought_node + """ + # Node name constants + DATA_HARVEST_NODE = "data_harvest" + THOUGHT_NODE = "thought_node" + TOOL_NODE = "tool_node" + OBSERVATION_NODE = "observation_node" + FORCED_FINISH_NODE = "forced_finish" + INVESTIGATION_PHASE_NODE = "investigation_phase" + llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + tools = builder.get_tools(tool_names=config.tool_names, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + + thought_llm = llm.with_structured_output(CheckerThought) + comprehension_llm = llm.with_structured_output(CodeFindings) + observation_llm = llm.with_structured_output(Observation) + compilation_verdict_llm = llm.with_structured_output(L2CompileVerdictExtraction) + hardening_verdict_llm = llm.with_structured_output(L2HardeningVerdictExtraction) + tools_node = ToolNode(tools, handle_tool_errors=True) if tools else None + enabled_tool_names = [tool.name for tool in tools] + tool_descriptions_list = [t.name + ": " + t.description for t in tools] + tools_str = "\n".join(tool_descriptions_list) + tool_strategy = _build_tool_strategy(enabled_tool_names) + # Extract context from state (guaranteed by early exit checks in _arun) + ctx = state.info.checker_context + assert ctx is not None, "checker_context must exist (checked in _arun)" + assert ctx.l1_result is not None, "l1_result must exist (checked in _arun)" + # build_log_path is now optional - None means spec-only mode + assert ctx.source_key, "source_key must exist when artifacts exist" + + l1_result = ctx.l1_result + artifacts = ctx.artifacts + target_package = state.input.image.target_package + vuln_id = state.input.scan.vulns[0].vuln_id + + # Paths + source_key = ctx.source_key + checker_dir = Path(config.base_checker_dir) / source_key + + build_log_path = Path(artifacts.build_log_path) if artifacts.build_log_path else None + + # L1 results - use full VulnerabilityIntel for richer context + vulnerability_intel = l1_result.vulnerability_intel + vulnerability_intel_str = vulnerability_intel.format_for_prompt() if vulnerability_intel else "No intel available" + l1_preliminary_verdict = l1_result.preliminary_verdict + + # Extract CWE ID from intel (if available) + cwe_id = None + intel_list = state.info.intel + intel = intel_list[0] + if intel.nvd and intel.nvd.cwe_id: + cwe_id = intel.nvd.cwe_id + logger.info("build_agent: CWE ID from intel: %s", cwe_id) + + _tiktoken_enc = tiktoken.get_encoding("cl100k_base") + + investigation_stack: list[L2InvestigationPhase] = [] + investigation_stack.append(L2InvestigationPhase.HARDENING) + investigation_stack.append(L2InvestigationPhase.CONFIGURATION) + + search_tracker = CheckerSearchTracker() + + # ------------------------------------------------------------------------- + # L2 graph nodes + # ------------------------------------------------------------------------- + def _count_tokens(text: str) -> int: + """Count tokens using tiktoken cl100k_base encoding (~90-95% accurate for Llama 3.1).""" + try: + return len(_tiktoken_enc.encode(text)) + except Exception: + return len(text) // 4 + + def _estimate_tokens(runtime_prompt: str, messages: list, observation: Observation | None) -> int: + """Estimate the token count thought_node will send to the LLM.""" + parts = [runtime_prompt] + for msg in messages: + if hasattr(msg, "content") and isinstance(msg.content, str): + parts.append(msg.content) + if observation is not None: + for item in (observation.memory or []): + parts.append(item) + for item in (observation.results or []): + parts.append(item) + return _count_tokens("\n".join(parts)) + # ------------------------------------------------------------------------- + # Data Harvest Node + # ------------------------------------------------------------------------- + async def build_runtime_prompt(harvest_report: BuildHarvestReport) -> str: + """Generate the runtime prompt for the current investigation phase.""" + current_phase = investigation_stack[-1] + runtime_prompt = "" + if current_phase == L2InvestigationPhase.CONFIGURATION: + # Kernel packages use CONFIG-based checking (not build log grep) + if harvest_report.is_kernel: + logger.info("Using kernel-specific prompt template") + runtime_prompt = L2_KERNEL_CONFIG_PROMPT_TEMPLATE.format( + sys_prompt=L2_KERNEL_CONFIG_SYS_PROMPT, + vuln_id=vuln_id, + target_package=target_package.name, + vulnerability_intel=vulnerability_intel_str, + l1_preliminary_verdict=l1_preliminary_verdict, + kernel_config_path=harvest_report.kernel_config_path or "Not available", + kernel_source_root=harvest_report.kernel_source_root or "Not available", + tools=tools_str, + tool_instructions=L2_KERNEL_THOUGHT_INSTRUCTIONS, + ) + elif build_log_path: + # Full mode: build log + spec file (standard packages) + runtime_prompt = L2_CONFIG_PROMPT_TEMPLATE.format( + sys_prompt=L2_CONFIG_SYS_PROMPT, + vuln_id=vuln_id, + target_package=target_package.name, + vulnerability_intel=vulnerability_intel_str, + l1_preliminary_verdict=l1_preliminary_verdict, + disabled_features=harvest_report.disabled_features, + spec_disabled_features=harvest_report.spec_disabled_features, + enabled_features=harvest_report.enabled_features, + spec_enabled_features=harvest_report.spec_enabled_features, + linked_libraries=harvest_report.linked_libraries or "None", + built_subpackages=harvest_report.built_subpackages or "None", + excluded_subpackages=harvest_report.excluded_subpackages or "None", + kernel_config=harvest_report.kernel_config or "N/A (not a kernel package)", + tools=tools_str, + tool_instructions=L2_CONFIG_THOUGHT_INSTRUCTIONS, + ) + else: + # Spec-only mode: no build log available + logger.info("Using spec-only prompt template (no build log)") + runtime_prompt = L2_CONFIG_PROMPT_SPEC_ONLY_TEMPLATE.format( + sys_prompt=L2_CONFIG_SPEC_ONLY_SYS_PROMPT, + vuln_id=vuln_id, + target_package=target_package.name, + vulnerability_intel=vulnerability_intel_str, + l1_preliminary_verdict=l1_preliminary_verdict, + spec_disabled_features=harvest_report.spec_disabled_features, + spec_enabled_features=harvest_report.spec_enabled_features, + built_subpackages=harvest_report.built_subpackages or "None", + excluded_subpackages=harvest_report.excluded_subpackages or "None", + kernel_config=harvest_report.kernel_config or "N/A (not a kernel package)", + tools=tools_str, + tool_instructions=L2_CONFIG_SPEC_ONLY_THOUGHT_INSTRUCTIONS, + ) + return runtime_prompt + elif current_phase == L2InvestigationPhase.HARDENING: + runtime_prompt = L2_HARDENING_PROMPT_TEMPLATE.format( + sys_prompt=L2_HARDENING_SYS_PROMPT, + vuln_id=vuln_id, + target_package=target_package.name, + cwe_id=cwe_id, + expected_hardening_table=harvest_report.expected_hardening, + tools=tools_str, + tool_instructions=L2_HARDENING_THOUGHT_INSTRUCTIONS, + ) + return runtime_prompt + else: + raise ValueError(f"Unknown investigation phase: {current_phase}") + + async def data_harvest_node(state: BuildAgentState) -> dict: + """Harvest structured data from build log, spec, and source tree before the ReAct loop.""" + logger.info("data_harvest_node: starting") + + with tracer.push_active_function("data_harvest", input_data={}) as span: + # Find spec file if available + spec_path = None + if checker_dir and checker_dir.exists(): + spec_files = list((checker_dir / "source").glob("*.spec")) + spec_path = spec_files[0] if spec_files else None + + source_path = checker_dir / "source" if checker_dir else None + harvest_report = await harvest_build_data( + build_log_path=build_log_path, + spec_path=spec_path, + cwe_id=cwe_id, + source_path=source_path, + package_name=target_package.name if target_package else None, + arch=target_package.arch if target_package else None, + ) + + runtime_prompt = await build_runtime_prompt(harvest_report) + + affected_files_count = len(vulnerability_intel.affected_files) if vulnerability_intel else 0 + # Compute evidence sources based on available artifacts + evidence_sources: list[str] = [] + if build_log_path: + evidence_sources.append("build_log") + if spec_path: + evidence_sources.append("spec_file") + + span.set_output({ + "disabled_features_count": len(harvest_report.disabled_features), + "spec_disabled_features_count": len(harvest_report.spec_disabled_features), + "enabled_features_count": len(harvest_report.enabled_features), + "spec_enabled_features_count": len(harvest_report.spec_enabled_features), + "expected_hardening_count": len(harvest_report.expected_hardening), + "vulnerability_intel_files_count": affected_files_count, + "linked_libraries_count": len(harvest_report.linked_libraries), + "built_subpackages_count": len(harvest_report.built_subpackages), + "excluded_subpackages_count": len(harvest_report.excluded_subpackages), + "kernel_config_count": len(harvest_report.kernel_config), + "evidence_sources": evidence_sources, + }) + + return { + "harvest_report": harvest_report, + "vulnerability_intel_str": vulnerability_intel_str, + "l1_preliminary_verdict": l1_preliminary_verdict, + "runtime_prompt": runtime_prompt, + "evidence_sources": evidence_sources, + "messages": [AIMessage(content="Build data harvested, beginning analysis.")], + } + + async def thought_node(state: BuildAgentState) -> dict: + """ReAct step: LLM chooses tools or emits a final answer for the current investigation phase.""" + step_num = state.get("step", 0) + logger.info("thought_node: starting step %d", step_num) + + runtime_prompt = state.get("runtime_prompt") + _messages = [SystemMessage(content=runtime_prompt)] + state["messages"] + + with tracer.push_active_function("thought_node", input_data={}) as span: + obs = state.get("observation", None) + if obs is not None: + memory_list = obs.memory if obs.memory else ["No prior knowledge."] + recent_findings = obs.results if obs.results else ["No recent findings."] + memory_context = "\n".join(f"- {m}" for m in memory_list) + findings_context = "\n".join(f"- {f}" for f in recent_findings) + context_block = f"KNOWLEDGE:\n{memory_context}\nLATEST FINDINGS:\n{findings_context}" + _messages.append(SystemMessage(content=context_block)) + + response: CheckerThought = await thought_llm.ainvoke(_messages) + + duplicate_warning = None + if response.mode == "act" and response.actions: + tool_name = response.actions.tool + query = response.actions.query or "" + duplicate_warning = search_tracker.check_before_execute(tool_name, query) + + if duplicate_warning: + logger.warning( + "thought_node: duplicate search detected - tool=%s, query=%s", + tool_name, + query, + ) + _messages.append(HumanMessage(content=duplicate_warning)) + response = await thought_llm.ainvoke(_messages) + + if response.mode == "act" and response.actions: + retry_warning = search_tracker.check_before_execute( + response.actions.tool, + response.actions.query or "", + ) + if retry_warning: + logger.warning("thought_node: duplicate persists after re-prompt, forcing finish") + response = CheckerThought( + thought="Duplicate search loop detected - concluding with available evidence.", + mode="finish", + actions=None, + final_answer="Unable to gather additional evidence. Conclude based on KNOWLEDGE.", + ) + + if response.mode == "finish": + ai_message = AIMessage(content=response.final_answer or "Analysis complete.") + elif response.actions: + tool_name = response.actions.tool + arguments = response.actions.query + tool_call_id = str(uuid.uuid4()) + ai_message = AIMessage( + content=response.thought, + tool_calls=[{"name": tool_name, "args": {"query": arguments}, "id": tool_call_id}] + ) + else: + logger.warning("thought_node: mode='act' but actions is None, forcing finish") + response = CheckerThought( + thought=response.thought or "No tool action provided", + mode="finish", + actions=None, + final_answer=response.thought or "Analysis complete.", + ) + ai_message = AIMessage(content=response.final_answer) + + span.set_output({ + "thought": response.thought, + "mode": response.mode, + "actions": response.actions, + "final_answer": response.final_answer, + "duplicate_warning": duplicate_warning, + }) + + return { + "messages": [ai_message], + "thought": response, + "step": step_num + 1, + "max_steps": config.max_iterations, + } + + async def observation_node(state: BuildAgentState) -> dict: + """Process tool output: comprehension -> memory update for build analysis.""" + logger.info("observation_node: starting") + tool_message = state["messages"][-1] + last_thought = state.get("thought") + if not last_thought: + return { + "messages": [AIMessage(content="No thought found")], + } + if not last_thought.actions: + logger.warning("observation_node: thought has no actions (mode=%s), skipping", last_thought.mode) + return { + "messages": [AIMessage(content="No tool action to process")], + } + last_thought_text = last_thought.thought + tool_used = last_thought.actions.tool + tool_input_detail = last_thought.actions.query + previous_memory = state.get("observation").memory if state.get("observation") else ["No data gathered yet."] + + harvest_report = state.get("harvest_report") or BuildHarvestReport() + target_package_name = target_package.name if target_package else "unknown" + + with tracer.push_active_function("observation_node", input_data=f"tool used:{tool_used} + {tool_input_detail}") as span: + tool_output_for_llm = tool_message.content + + # Check for empty/error outputs - bypass LLM if so to prevent hallucination + empty_findings, _ = check_empty_output(tool_output_for_llm, tool_used, tool_input_detail) + if empty_findings: + # Build-specific: empty grep for file in logs = NOT_COMPILED evidence + if tool_used == "Source Grep" and "logs:" in tool_input_detail: + file_searched = tool_input_detail.replace("logs:", "") + empty_findings = CodeFindings( + findings=[ + f"Source Grep for '{file_searched}' returned EMPTY - file not in build log", + "This indicates the file was NOT compiled in this build" + ], + tool_outcome=f"CALLED: Source Grep with {tool_input_detail} -> EMPTY (not compiled)" + ) + code_findings = empty_findings + else: + # Step 1: Comprehension - split into chunks and process each + chunks = truncate_tool_output_list(tool_output_for_llm, tool_used, max_tokens=1000) + all_findings = [] + best_tool_outcome = "" + + for chunk in chunks: + comp_prompt = L2_COMPREHENSION_PROMPT.format( + vuln_id=vuln_id, + target_package=target_package_name, + vulnerability_intel=vulnerability_intel_str, + disabled_features=", ".join(harvest_report.disabled_features) if harvest_report.disabled_features else "None", + spec_disabled_features=", ".join(harvest_report.spec_disabled_features) if harvest_report.spec_disabled_features else "None", + enabled_features=", ".join(harvest_report.enabled_features) if harvest_report.enabled_features else "None", + spec_enabled_features=", ".join(harvest_report.spec_enabled_features) if harvest_report.spec_enabled_features else "None", + tool_used=tool_used, + tool_input=tool_input_detail, + last_thought=last_thought_text, + tool_output=chunk, + ) + chunk_findings: CodeFindings = await invoke_comprehension( + comprehension_llm, comp_prompt, tool_used, tool_input_detail, chunk, agent_label="L2", + ) + all_findings.extend(chunk_findings.findings) + # Keep tool_outcome from chunk with actual findings (not FAILED) + if not best_tool_outcome or ( + chunk_findings.findings and + not any("FAILED" in f for f in chunk_findings.findings) + ): + best_tool_outcome = chunk_findings.tool_outcome + + code_findings = CodeFindings( + findings=all_findings, + tool_outcome=best_tool_outcome or "No matches found" + ) + findings_text = "\n".join(f"- {f}" for f in code_findings.findings) + + # Step 2: Memory update - merge findings into cumulative memory + mem_prompt = L2_MEMORY_UPDATE_PROMPT.format( + vuln_id=vuln_id, + target_package=target_package_name, + previous_memory="\n".join(f"- {m}" for m in previous_memory) if isinstance(previous_memory, list) else previous_memory, + findings=findings_text, + tool_outcome=code_findings.tool_outcome, + ) + new_observation: Observation = await observation_llm.ainvoke([SystemMessage(content=mem_prompt)]) + + messages = state["messages"] + active_prompt = state.get("runtime_prompt") or "" + estimated = _estimate_tokens(active_prompt, messages, new_observation) + orig_estimated = estimated + prune_messages = [] + if estimated > config.context_window_token_limit: + with tracer.push_active_function("context_pruning", input_data={"estimated_tokens": estimated, "limit": config.context_window_token_limit}) as prune_span: + + for msg in messages: + prune_messages.append(RemoveMessage(id=msg.id)) + estimated -= _count_tokens(msg.content) if hasattr(msg, "content") and isinstance(msg.content, str) else 0 + if estimated <= config.context_window_token_limit: + break + logger.info( + "Context pruning: removed %d messages, estimated tokens now ~%d (limit %d)", + len(prune_messages), estimated, config.context_window_token_limit, + ) + prune_span.set_output({ + "pruning_triggered": len(prune_messages) > 0, + "messages_pruned": len(prune_messages), + "tokens_before": orig_estimated, + "tokens_after": estimated, + }) + + + span.set_output({ + "last_thought_text": last_thought_text, + "tool_output_for_llm": tool_output_for_llm[:500], + "findings": code_findings.findings, + "tool_outcome": code_findings.tool_outcome, + "new_memory": new_observation.memory, + "amount_of_orig_tokens": orig_estimated, + "amount_of_estimated_tokens": estimated, + }) + return { + "messages": prune_messages, + "observation": new_observation, + } + + async def forced_finish_node(state: BuildAgentState) -> dict: + """Force finish when max iterations reached. + + Invokes the LLM with FORCED_FINISH_PROMPT to generate a final answer + based on evidence gathered so far. + """ + step_num = state.get("step", 0) + with tracer.push_active_function("forced_finish_node", input_data=f"step:{step_num}") as span: + try: + active_prompt = state.get("runtime_prompt") or "" + messages = [SystemMessage(content=active_prompt)] + state["messages"] + messages.append(HumanMessage(content=FORCED_FINISH_PROMPT)) + + obs = state.get("observation") + if obs is not None and obs.memory: + memory_context = "\n".join(f"- {m}" for m in obs.memory) + messages.append(SystemMessage(content=f"KNOWLEDGE:\n{memory_context}")) + + response: CheckerThought = await thought_llm.ainvoke(messages) + + if response.mode == "finish" and response.final_answer: + ai_message = AIMessage(content=response.final_answer) + final_answer = response.final_answer + else: + final_answer = "Unable to determine compilation status within iteration limit." + ai_message = AIMessage(content=final_answer) + response = CheckerThought( + thought=response.thought or "Max steps exceeded", + mode="finish", + actions=None, + final_answer=final_answer, + ) + + span.set_output({"final_answer_length": len(final_answer), "step": step_num}) + return { + "messages": [ai_message], + "thought": response, + "step": step_num, + "max_steps": state.get("max_steps", config.max_iterations), + "observation": state.get("observation"), + "output": final_answer, + } + except Exception as e: + logger.exception("forced_finish_node failed at step %d", step_num) + span.set_output({"error": str(e), "exception_type": type(e).__name__, "step": step_num}) + raise + + async def should_continue(state: BuildAgentState) -> str: + """Route based on thought mode.""" + thought = state.get("thought") + if thought is not None and thought.mode == "finish": + return INVESTIGATION_PHASE_NODE + if state.get("step", 0) >= state.get("max_steps", config.max_iterations): + return FORCED_FINISH_NODE + return TOOL_NODE + + + async def is_investigation_finished(state: BuildAgentState) -> str: + """Check if the investigation is finished.""" + if len(investigation_stack) == 0: + return END + return THOUGHT_NODE + + async def investigation_phase_node(state: BuildAgentState) -> dict: + """Determine the next investigation phase.""" + if len(investigation_stack) == 0: + raise ValueError("Investigation stack is empty") + + final_answer = None + thought = state.get("thought") + if thought and thought.mode == "finish": + final_answer = thought.final_answer + + current_phase = investigation_stack[-1] + with tracer.push_active_function("investigation_phase_node", input_data=f"phase :{current_phase}") as span: + investigation_stack.pop() + if current_phase == L2InvestigationPhase.CONFIGURATION: + verdict: L2CompileVerdictExtraction = await compilation_verdict_llm.ainvoke([SystemMessage(content=L2_COMPILATION_VERDICT_PROMPT.format(final_answer=final_answer))]) + span.set_output({ + "compilation_status": verdict.compilation_status, + "confidence": verdict.confidence, + "reasoning": verdict.reasoning, + }) + if verdict.compilation_status == "not_compiled": + return {"L2CompileVerdict": verdict} + + # Check if this is a kernel package - skip hardening phase + # RHEL kernels always have hardening enabled, so checking provides low signal + preprocess_data = state.get("harvest_report") or BuildHarvestReport() + if preprocess_data.is_kernel: + logger.info("investigation_phase_node: kernel package - skipping hardening phase") + # Clear the investigation stack to prevent hardening phase + investigation_stack.clear() + span.set_output({ + "compilation_status": verdict.compilation_status, + "confidence": verdict.confidence, + "reasoning": verdict.reasoning, + "hardening_skipped": True, + "hardening_skip_reason": "kernel_package", + }) + return {"L2CompileVerdict": verdict} + + has_binary = bool( + artifacts + and artifacts.binary_rpm_path + and Path(artifacts.binary_rpm_path).exists() + ) + if not build_log_path and not has_binary: + logger.info( + "investigation_phase_node: no build log or binary RPM - skipping hardening phase", + ) + investigation_stack.clear() + span.set_output({ + "compilation_status": verdict.compilation_status, + "confidence": verdict.confidence, + "reasoning": verdict.reasoning, + "hardening_skipped": True, + "hardening_skip_reason": "no_build_or_binary_evidence", + }) + return {"L2CompileVerdict": verdict} + + else: + # Note: Architecture mismatch check is now handled early in L1 + # (cve_package_code_agent.py) using target_package.arch. The previous + # L2 check using build_architecture from build logs has been removed + # since build_architecture detection is no longer performed. + + # Normal path: proceed to hardening phase (non-kernel packages) + search_tracker.reset() + runtime_prompt = await build_runtime_prompt(preprocess_data) + messages = state["messages"] + prune_messages = [] + for msg in messages: + prune_messages.append(RemoveMessage(id=msg.id)) + span.set_output({ + "runtime_prompt": runtime_prompt, + }) + return { + "runtime_prompt": runtime_prompt, + "thought": None, + "observation": None, + "step": 0, + "messages": prune_messages, + "L2CompileVerdict": verdict, + } + else: + #state that run was hardening need to extract the hardening verdict + verdict: L2HardeningVerdictExtraction = await hardening_verdict_llm.ainvoke([SystemMessage(content=L2_HARDENING_VERDICT_PROMPT.format(final_answer=final_answer))]) + span.set_output({ + "hardening_status": verdict.hardening_status, + "confidence": verdict.confidence, + "reasoning": verdict.reasoning, + }) + return { + "L2HardeningVerdict": verdict, + } + # ------------------------------------------------------------------------- + # Build graph + # ------------------------------------------------------------------------- + + flow = StateGraph(BuildAgentState) + + flow.add_node(DATA_HARVEST_NODE, data_harvest_node) + flow.add_node(THOUGHT_NODE, thought_node) + flow.add_node(FORCED_FINISH_NODE, forced_finish_node) + flow.add_node(OBSERVATION_NODE, observation_node) + flow.add_node(TOOL_NODE, tools_node) + flow.add_node(INVESTIGATION_PHASE_NODE, investigation_phase_node) + + flow.add_edge(START, DATA_HARVEST_NODE) + flow.add_edge(DATA_HARVEST_NODE, THOUGHT_NODE) + edge_map = {INVESTIGATION_PHASE_NODE: INVESTIGATION_PHASE_NODE, FORCED_FINISH_NODE: FORCED_FINISH_NODE, TOOL_NODE: TOOL_NODE} + flow.add_conditional_edges(THOUGHT_NODE, should_continue, edge_map) + flow.add_edge(TOOL_NODE, OBSERVATION_NODE) + flow.add_edge(OBSERVATION_NODE, THOUGHT_NODE) + flow.add_edge(FORCED_FINISH_NODE, INVESTIGATION_PHASE_NODE) + flow.add_conditional_edges(INVESTIGATION_PHASE_NODE, is_investigation_finished, {END: END, THOUGHT_NODE: THOUGHT_NODE}) + + app = flow.compile() + return app + + +@register_function(config_type=CVEBuildAgentConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def cve_build_agent(config: CVEBuildAgentConfig, builder: Builder): + """Level 2 Build Agent entry point.""" + + async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Run L2 build analysis and populate l2_result on checker_context.""" + trace_id.set(message.input.scan.id) + tracer = Context.get() + + # Set ctx_state for tools + from types import SimpleNamespace + workflow_state = SimpleNamespace(original_input=message, info=message.info) + ctx_state.set(workflow_state) + + logger.info("build_agent: starting L2 investigation") + + ctx = message.info.checker_context + if not ctx or not ctx.l1_result: + logger.warning("build_agent: no L1 result available, skipping L2") + return message + + if not ctx.artifacts: + logger.warning("build_agent: no artifacts available, skipping L2") + return message + + has_build_log = bool(ctx.artifacts.build_log_path) + if not has_build_log: + logger.info("build_agent: no build log, will use spec/build system files only") + + # Build and run the graph + build_agent_graph = await create_graph_build_agent(config, builder, message, tracer) + initial_state: BuildAgentState = { + "messages": [HumanMessage(content="Begin L2 build analysis")], + "step": 0, + "max_steps": config.max_iterations, + } + + with tracer.push_active_function("l2_build_agent_graph", input_data=initial_state["messages"][0].content): + # Each phase: (max_iterations * 3 react nodes) + data_harvest/forced_finish/investigation_phase + # Two phases (CONFIG + HARDENING) when code is compiled + steps_per_phase = (config.max_iterations * 3) + 4 + recursion_limit = steps_per_phase * 2 + 5 # buffer for edge cases + result = await build_agent_graph.ainvoke( + initial_state, + config={"recursion_limit": recursion_limit}, + ) + + logger.info("build_agent: L2 investigation finished") + + # Extract verdict from result + compile_verdict = result.get("L2CompileVerdict") or None + if compile_verdict is None: + logger.error("build_agent: L2CompileVerdict is none should never happen") + return message + hardening_verdict = result.get("L2HardeningVerdict") or None + hardening_reason = None + hardening_flags = [] + if compile_verdict.compilation_status == "not_compiled": + hardening_relevant = False + l2_override_verdict = "not_vulnerable" + elif hardening_verdict is None: + # Hardening skipped (kernel, no build log/binary, etc.) — defer to L1/code agent + hardening_relevant = False + l2_override_verdict = None + else: + hardening_relevant = True + hardening_reason = hardening_verdict.reasoning + hardening_flags = hardening_verdict.hardening_flags or [] + if hardening_verdict.hardening_status == "mitigated": + l2_override_verdict = "vulnerable_mitigated" + else: + l2_override_verdict = None + + # Build L2 result + evidence_sources = result.get("evidence_sources") or [] + l2_result = L2BuildResult( + compilation_status=compile_verdict.compilation_status, + compilation_confidence=compile_verdict.confidence, + compilation_evidence=compile_verdict.reasoning, + hardening_relevant=hardening_relevant, + hardening_flags=hardening_flags, + hardening_rationale=hardening_reason, + l2_override_verdict=l2_override_verdict, + evidence_sources=evidence_sources, + ) + + with tracer.push_active_function( + "l2_agent_finish", + input_data={"compilation_status": l2_result.compilation_status}, + ) as span: + span.set_output({ + "compilation_status": l2_result.compilation_status, + "compilation_confidence": l2_result.compilation_confidence, + "l2_override_verdict": l2_result.l2_override_verdict, + }) + + # Store result on checker_context + if message.info.checker_context is not None: + message.info.checker_context.l2_result = l2_result + else: + logger.warning("build_agent: checker_context is None, cannot store l2_result") + + logger.info( + "build_agent: L2 result - status=%s, confidence=%.2f, override=%s", + l2_result.compilation_status, + l2_result.compilation_confidence, + l2_result.l2_override_verdict, + ) + + return message + + yield FunctionInfo.from_fn( + _arun, + description="Level 2 Build Agent: analyzes build artifacts for compilation status and hardening", + ) diff --git a/src/vuln_analysis/functions/cve_calculate_intel_score.py b/src/vuln_analysis/functions/cve_calculate_intel_score.py new file mode 100644 index 000000000..5f562e4d2 --- /dev/null +++ b/src/vuln_analysis/functions/cve_calculate_intel_score.py @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio + +import aiohttp +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) + +class CVECalculateIntelScoreConfig(FunctionBaseConfig, name="cve_calculate_intel_score"): + """ + Defines a function that calculates the intel quality generating a score for it. + """ + llm_name: str = Field(description="The LLM model to use with the CVE agent.") + generate_intel_score: bool = Field(default=True, description="Whether to generate a CVE intel score for the agent morpheus analysis or not.") + intel_low_score: int = Field(default=51, description="The intel low score threshold to stop analysis.") + insist_analysis: bool = Field(default=False, description="Whether to continue the analysis even when the intel score is below 'intel_low_score' threshold.") + +@register_function(config_type=CVECalculateIntelScoreConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def cve_calculate_intel_score(config: CVECalculateIntelScoreConfig, builder: Builder): # pylint: disable=unused-argument + + from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput + from vuln_analysis.utils.intel_source_score import IntelScorer + + async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + if config.generate_intel_score: + intel_scorer = IntelScorer(config, builder) + + message.info.intel = [await intel_scorer.calculate_intel_score(intel) for intel in message.info.intel] + + return message + + yield FunctionInfo.from_fn(_arun, + input_schema=AgentMorpheusEngineInput, + description="Calculates the CVE source score.") diff --git a/src/vuln_analysis/functions/cve_check_vuln_deps.py b/src/vuln_analysis/functions/cve_check_vuln_deps.py index e769266df..08dc972af 100644 --- a/src/vuln_analysis/functions/cve_check_vuln_deps.py +++ b/src/vuln_analysis/functions/cve_check_vuln_deps.py @@ -14,7 +14,6 @@ # limitations under the License. import asyncio -import logging import aiohttp from aiq.builder.builder import Builder @@ -24,7 +23,10 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -logger = logging.getLogger(__name__) +from exploit_iq_commons.data_models.common import AnalysisType +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id + +logger = LoggingFactory.get_agent_logger(__name__) class CVEVulnerableDepsChecksConfig(FunctionBaseConfig, name="cve_check_vuln_deps"): @@ -32,21 +34,36 @@ class CVEVulnerableDepsChecksConfig(FunctionBaseConfig, name="cve_check_vuln_dep Defines a function that cross-references every entry in the SBOM for known vulnerabilities. """ retry_on_client_errors: bool = Field(default=True, description="Whether to retry if HTTP client cannot connect") + skip: bool = Field(default=False, + description="Whether or not the vulnerability dependency checker should be skipped.") @register_function(config_type=CVEVulnerableDepsChecksConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def cve_check_vuln_deps(config: CVEVulnerableDepsChecksConfig, builder: Builder): # pylint: disable=unused-argument - from vuln_analysis.data_models.cve_intel import CveIntel - from vuln_analysis.data_models.dependencies import VulnerableDependencies - from vuln_analysis.data_models.dependencies import VulnerableSBOMPackage - from vuln_analysis.data_models.input import AgentMorpheusEngineInput + from exploit_iq_commons.data_models.cve_intel import CveIntel + from exploit_iq_commons.data_models.dependencies import VulnerableDependencies + from exploit_iq_commons.data_models.dependencies import VulnerableSBOMPackage + from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput from vuln_analysis.utils.vulnerable_dependency_checker import VulnerableDependencyChecker async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: - sbom = message.info.sbom.packages + trace_id.set(message.input.scan.id) + + if message.input.image.analysis_type == AnalysisType.SOURCE: + logger.info("Source code analysis requested. Skipping vulnerable dependency check.") + # Don't overwrite vulnerable_dependencies - it may have been set by verify_vuln_package + return message + image = f"{message.input.image.name}:{message.input.image.tag}" + + if config.skip: + logger.info("`config.skip` is set to True. Skipping vulnerable dependency check for image %s.", image) + message.info.vulnerable_dependencies = None + return message + + sbom = message.info.sbom.packages if not sbom: logger.warning("No SBOM packages found for image %s. Skipping vulnerable dependency check.", image) @@ -69,7 +86,6 @@ async def _inner(): async def _calc_dep(cve_intel: CveIntel): vuln_id = cve_intel.vuln_id - vuln_deps = [] vuln_package_intel_sources = [] diff --git a/src/vuln_analysis/functions/cve_checker_report.py b/src/vuln_analysis/functions/cve_checker_report.py new file mode 100644 index 000000000..130d34d2c --- /dev/null +++ b/src/vuln_analysis/functions/cve_checker_report.py @@ -0,0 +1,1225 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +CVE Checker Report Generation Function. + +This module provides the report generation node for the L1/L2 pipeline. +It consumes L1InvestigationResult (and optionally L2BuildResult) from +checker_context and produces the final AgentMorpheusOutput. +""" + +import re +import warnings +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput +from exploit_iq_commons.data_models.checker_status import ( + EnumIdentifyResult, + L1InvestigationResult, + L2BuildResult, + PackageCheckerStatus, +) + +from nat.builder.context import Context +from vuln_analysis.data_models.output import ( + AgentMorpheusEngineOutput, + AgentMorpheusOutput, + JustificationOutput, + OutputPayload, +) +from vuln_analysis.functions.code_agent_graph_defs import ( + CodeAgentReport, + CodeSnippet, + DownstreamSearchReport, + GitSearchReport, + ParsedPatch, + UpstreamSearchReport, + generate_code_agent_report, +) + +logger = LoggingFactory.get_agent_logger(__name__) + + +_StatusLiteral = Literal["TRUE", "FALSE", "UNKNOWN"] + +_JUSTIFICATION_LABEL_TO_STATUS: dict[str, _StatusLiteral] = { + "code_not_present": "FALSE", + "code_not_reachable": "FALSE", + "protected_by_mitigating_control": "FALSE", + "protected_by_compiler": "FALSE", + "requires_environment": "FALSE", + "vulnerable": "TRUE", + "uncertain": "UNKNOWN", + "conflicting_evidence": "UNKNOWN", +} + + +_POLICY_MAX_RPM_LIST_ITEMS = 5 +_POLICY_RHSA_STATEMENT_CAP = 400 + +_POLICY_MAX_PACKAGE_STATE_ITEMS = 8 + +_BUILD_LOG_EXCERPT_MAX_LINES = 8 + +_EVIDENCE_CHAIN_ITEM_CAP = 8 + + +def _detect_intel_conflict(ctx, final_label: str) -> bool: + """Detect when final verdict contradicts RHSA 'not affected' assessment. + + Returns True if RHSA assessed the package as "not affected" but the + code/build analysis concluded it is vulnerable. This conflict requires + human review to determine if it's an RHSA error or an analysis bug. + """ + if not ctx or not ctx.identify_result: + return False + if ctx.identify_result.is_target_package_affected != EnumIdentifyResult.NO: + return False + if ctx.identify_result.rhsa_fix_state != "not affected": + return False + return final_label == "vulnerable" + + +def _md_section_header(title: str) -> list[str]: + """UI-friendly section break: horizontal rule + level-2 heading (no tables).""" + return ["---", "", f"## {title}", ""] + + +def _md_note(text: str) -> list[str]: + """Callout line for interpretation; renders as blockquote in most markdown UIs.""" + return [f"> **Note:** {text}", ""] + + +def _md_check_result(passed: bool) -> str: + return "**Pass**" if passed else "**Fail**" + + +def _first_nonempty_line(text: str, *, max_len: int = 140) -> str: + for raw in text.splitlines(): + line = raw.strip() + if line: + if len(line) > max_len: + return line[: max_len - 1] + "…" + return line + return "" + + +def _spec_declaration_summary(directives: list[str]) -> str: + for directive in directives: + stripped = directive.strip() + if stripped.startswith("Patch"): + return stripped + return directives[0].strip() if directives else "" + + +def _is_redundant_patch_evidence(evidence: str) -> bool: + """Skip LLM bullets that repeat the patch/spec/build tri-state checks.""" + ev_lower = evidence.lower() + redundant_phrases = ( + "target package patch available", + "target patch file referenced in spec", + "referenced in spec", + "build log evidence", + "patch file exists", + "applied in build", + "patch file on target", + ) + return any(phrase in ev_lower for phrase in redundant_phrases) + + +def _categorize_evidence_chain( + evidence_chain: list[str], +) -> tuple[list[str], list[str], list[str], list[str]]: + patch_evidence: list[str] = [] + build_evidence: list[str] = [] + code_evidence: list[str] = [] + other_evidence: list[str] = [] + + patch_keywords = ("patch", "spec", "patchn", "directive", "target", "reference", "rebase") + build_keywords = ("build", "applied", "log", "compilation") + code_keywords = ("code agent", "code analysis", "function", "vulnerable", "source search") + + for ev in evidence_chain[:_EVIDENCE_CHAIN_ITEM_CAP]: + ev_lower = ev.lower() + if any(kw in ev_lower for kw in code_keywords): + code_evidence.append(ev) + elif any(kw in ev_lower for kw in build_keywords): + if not _is_redundant_patch_evidence(ev): + build_evidence.append(ev) + elif any(kw in ev_lower for kw in patch_keywords): + if _is_redundant_patch_evidence(ev): + continue + patch_evidence.append(ev) + else: + other_evidence.append(ev) + + return patch_evidence, build_evidence, code_evidence, other_evidence + + +def _target_nvr(blocks: "ReportBlocks") -> str: + version_release = ( + f"{blocks.package_version}-{blocks.package_release}" + if blocks.package_release + else blocks.package_version + ) + return f"{blocks.package_name}-{version_release} ({blocks.package_arch})" + + +def _derive_investigation_mode( + downstream_report: DownstreamSearchReport | None, + upstream_report: UpstreamSearchReport | None, +) -> str: + if downstream_report and downstream_report.is_patch_file_available: + return "patch_available" + if upstream_report and upstream_report.is_code_fixed_by_rebase == "yes": + return "rebase" + if upstream_report and upstream_report.fixed_parsed_patch: + return "upstream_patch" + return "no_patch" + + +def _derive_arch_gate_reason( + l1_agent_answer: str | None, + affected_bitness: str | None, + package_arch: str, +) -> str | None: + if l1_agent_answer: + answer_lower = l1_agent_answer.lower() + if "bitness mismatch" in answer_lower or ( + "architecture" in answer_lower and "mismatch" in answer_lower + ): + return l1_agent_answer.strip() + if not affected_bitness or affected_bitness == "both": + return None + arch_lower = package_arch.lower() + is_64 = any(token in arch_lower for token in ("x86_64", "amd64", "aarch64", "64")) + is_32 = any(token in arch_lower for token in ("i686", "i386", "32")) + if affected_bitness == "32-bit" and is_64 and not is_32: + return f"CVE affects 32-bit only; target package is {package_arch}." + if affected_bitness == "64-bit" and is_32 and not is_64: + return f"CVE affects 64-bit only; target package is {package_arch}." + return None + + +def _target_fix_search_interpretation(blocks: "ReportBlocks") -> str: + checks = ( + blocks.is_patch_file_available, + blocks.is_patch_in_spec_file, + blocks.is_patch_applied_in_build, + ) + passed = sum(1 for c in checks if c) + if passed == 3: + if blocks.justification_label == "vulnerable": + return ( + "Target fix search (3/3): patch file, spec reference, and build log all match, " + "but vulnerable patterns remain in source—see target source check below." + ) + return ( + "Target fix search (3/3): CVE patch file present, referenced in the spec, " + "and observed in the build log." + ) + if passed == 0: + return ( + "Target fix search (0/3): no CVE-named patch file, spec reference, " + "or build-time application found on the target package." + ) + gaps: list[str] = [] + if not blocks.is_patch_file_available: + gaps.append("no CVE patch file on target") + if not blocks.is_patch_in_spec_file: + gaps.append("not referenced in target spec") + if not blocks.is_patch_applied_in_build: + gaps.append("not observed in target build log") + return ( + f"Target fix search ({passed}/3 checks passed): " + f"{'; '.join(gaps)}." + ) + + +def _summarize_parsed_patch_lines(parsed_patch: ParsedPatch | None) -> list[str]: + if not parsed_patch: + return [] + lines: list[str] = [] + if parsed_patch.patch_filename: + lines.append(f"- **Patch file:** `{parsed_patch.patch_filename}`") + for pf in parsed_patch.files[:2]: + lines.append(f"- **File in patch:** `{pf.clean_target_path}`") + for hunk in pf.hunks[:1]: + if hunk.removed_lines: + lines.append(f" - Removed: `{hunk.removed_lines[0].strip()}`") + if hunk.added_lines: + lines.append(f" - Added: `{hunk.added_lines[0].strip()}`") + return lines + + +def _build_commit_url(repo_url: str, commit_hash: str) -> str | None: + """Build a viewable commit URL from repo URL and commit hash.""" + from vuln_analysis.utils.web_patch_fetcher import build_patch_url_from_repo + + patch_url = build_patch_url_from_repo(repo_url, commit_hash) + if patch_url: + return patch_url.removesuffix(".patch") + return None + + +def _format_target_fix_search_md(blocks: "ReportBlocks") -> str: + lines: list[str] = [] + lines.extend(_md_section_header(f"CVE fix search on target package ({_target_nvr(blocks)})")) + + patch_detail = f"`{blocks.patch_file_name}`" if blocks.patch_file_name else "_not found_" + spec_detail = ( + f"`{_spec_declaration_summary(blocks.spec_patch_directives)}`" + if blocks.spec_patch_directives + else ("_CVE mentioned in spec_" if blocks.is_patch_in_spec_file else "_not referenced_") + ) + build_detail = _first_nonempty_line(blocks.build_log_evidence) or "_not observed in build log_" + if build_detail and not build_detail.startswith("`") and build_detail != "_not observed in build log_": + build_detail = f"`{build_detail}`" + + lines.append( + f"- **Patch file on target:** {_md_check_result(blocks.is_patch_file_available)} — {patch_detail}" + ) + lines.append( + f"- **Referenced in spec:** {_md_check_result(blocks.is_patch_in_spec_file)} — {spec_detail}" + ) + lines.append( + f"- **Applied in build:** {_md_check_result(blocks.is_patch_applied_in_build)} — {build_detail}" + ) + lines.append("") + lines.extend(_md_note(_target_fix_search_interpretation(blocks))) + + if blocks.spec_patch_directives: + declarations = [d for d in blocks.spec_patch_directives if d.strip().startswith("Patch")] + applications = [d for d in blocks.spec_patch_directives if d.strip().startswith("%patch")] + if declarations: + lines.append("- **Spec patch declaration:**") + for decl in declarations: + lines.append(f" - `{decl.strip()}`") + if applications: + lines.append("- **Spec patch application:**") + for app in applications: + lines.append(f" - `{app.strip()}`") + if blocks.spec_changelog_cve_lines.strip(): + lines.append("- **Spec changelog (CVE):**") + for changelog_line in blocks.spec_changelog_cve_lines.strip().splitlines(): + stripped = changelog_line.strip() + if stripped: + lines.append(f" - `{stripped}`") + if blocks.spec_version_line or blocks.spec_source0_line: + if blocks.spec_version_line: + lines.append(f"- **Spec version line:** `{blocks.spec_version_line.strip()}`") + if blocks.spec_source0_line: + lines.append(f"- **Spec Source0 line:** `{blocks.spec_source0_line.strip()}`") + lines.append( + "> **Note:** Version/Source0 describe the upstream snapshot named in the spec; " + "they may differ from the tree used for source review." + ) + lines.append("") + return "\n".join(lines).strip() + + +def _format_external_fix_clues_md(blocks: "ReportBlocks") -> str: + lines: list[str] = [] + lines.extend(_md_section_header("Fix clues from advisories and reference builds")) + + if blocks.investigation_mode == "patch_available": + lines.append( + "- **Status:** Not needed — a CVE-named patch file was found on the target package." + ) + lines.append("") + return "\n".join(lines).strip() + + if not blocks.upstream_search_ran: + lines.append( + "- **Status:** Not evaluated — external reference search did not run " + "(target patch workflow did not require it)." + ) + lines.append("") + return "\n".join(lines).strip() + + upstream = blocks.upstream_report + clue_lines: list[str] = [] + + if blocks.fixed_rpm_hints: + shown = blocks.fixed_rpm_hints[:_POLICY_MAX_RPM_LIST_ITEMS] + clue_lines.append("- **Known-fixed packages (intel):**") + for hint in shown: + clue_lines.append(f" - `{hint}`") + + if upstream: + if upstream.reference_package_nvr: + clue_lines.append( + f"- **Reference package NVR:** `{upstream.reference_package_nvr}`" + ) + if upstream.fixed_srpm_file_name: + clue_lines.append( + f"- **Reference patch source:** `{upstream.fixed_srpm_file_name}`" + ) + clue_lines.extend(_summarize_parsed_patch_lines(upstream.fixed_parsed_patch)) + if upstream.osv_result: + osv = upstream.osv_result + if osv.patch_url: + clue_lines.append(f"- **Fetched patch URL:** {osv.patch_url}") + if osv.fixed_commit: + clue_lines.append(f"- **Fixed commit:** `{osv.fixed_commit}`") + if osv.source: + clue_lines.append(f"- **Patch source:** {osv.source}") + if upstream.is_code_fixed_by_rebase == "yes": + clue_lines.append("- **Rebase on target:** Spec/changelog suggests fix via version rebase.") + if upstream.spec_file_log_change.strip(): + excerpt = _first_nonempty_line(upstream.spec_file_log_change, max_len=200) + if excerpt: + clue_lines.append(f" - `{excerpt}`") + if upstream.reason_code_fixed_by_rebase.strip(): + clue_lines.append(f" - {upstream.reason_code_fixed_by_rebase.strip()}") + + git = blocks.git_search_report + if git and git.search_success and git.best_result: + result = git.best_result + clue_lines.append("- **Git commit search:**") + clue_lines.append(f" - **Repository:** `{git.repo_url}`") + # debug information + #clue_lines.append(f" - **Commit:** `{result.commit_hash_short}`") + + commit_url = _build_commit_url(git.repo_url, result.commit_hash) + if commit_url: + clue_lines.append(f" - **Commit URL:** {commit_url}") + + # debug information + #clue_lines.append(f" - **Search method:** {result.search_method}") + #clue_lines.append(f" - **Confidence:** {result.confidence:.0%}") + + if result.commit_message: + msg_preview = result.commit_message[:100] + if len(result.commit_message) > 100: + msg_preview += "..." + clue_lines.append(f" - **Message:** {msg_preview}") + + if git.parsed_patch: + patch_lines = _summarize_parsed_patch_lines(git.parsed_patch) + clue_lines.extend(f" {line}" for line in patch_lines) + + if clue_lines: + lines.extend(clue_lines) + else: + lines.append( + "- **Status:** No external fix clue found — advisories and reference builds " + "did not yield a usable patch for comparison." + ) + lines.append("") + return "\n".join(lines).strip() + + +def _format_target_source_check_md(blocks: "ReportBlocks") -> str: + _, _, code_evidence, _ = _categorize_evidence_chain(blocks.evidence_chain) + lines: list[str] = [] + lines.extend(_md_section_header("Target source check")) + + if blocks.l1_agent_answer: + lines.append(f"- **Code agent summary:** {blocks.l1_agent_answer.strip()}") + + if blocks.affected_files: + lines.append("- **Affected source files:**") + for f in blocks.affected_files[:10]: + lines.append(f" - `{f}`") + elif blocks.arch_gate_reason: + lines.append( + f"- **Status:** Source review limited — {blocks.arch_gate_reason}" + ) + else: + lines.append("- **Affected source files:** _not determined_") + + mitigated = blocks.justification_label in ( + "protected_by_mitigating_control", + "protected_by_compiler", + "code_not_present", + "code_not_reachable", + "requires_environment", + ) + vulnerable_snippets = _filter_review_snippets(blocks.vulnerable_snippets) + fix_snippets = _filter_review_snippets(blocks.fix_snippets) + + if vulnerable_snippets: + title = "Pattern before fix (reference or target)" if mitigated else "Vulnerable pattern on target" + lines.append(f"- **{title}:**") + lines.extend(blocks._format_snippet_bullets(vulnerable_snippets)) + elif not blocks.arch_gate_reason: + lines.append("- **Vulnerable pattern on target:** _not shown — no source excerpt collected_") + + if fix_snippets: + lines.append("- **Fix pattern (reference or expected on target):**") + lines.extend(blocks._format_snippet_bullets(fix_snippets)) + elif vulnerable_snippets or blocks.l1_agent_answer: + lines.append("- **Fix pattern:** _not shown — no fix excerpt collected_") + + for ev in code_evidence[:2]: + if not _is_redundant_patch_evidence(ev): + lines.append(f"- {ev}") + + lines.append("") + return "\n".join(lines).strip() + + +def _format_target_build_check_md(blocks: "ReportBlocks") -> str: + lines: list[str] = [] + lines.extend(_md_section_header("Target build check")) + + l2 = blocks.l2_result + if l2 is None: + lines.append("- **Status:** Not evaluated — build agent did not run for this scan.") + lines.append("") + return "\n".join(lines).strip() + + lines.append(f"- **Compilation:** {l2.compilation_status or 'unknown'}") + if l2.compilation_evidence: + lines.append(f"- **Compilation evidence:** {l2.compilation_evidence.strip()}") + if l2.hardening_flags: + flags_str = ", ".join(l2.hardening_flags[:5]) + lines.append(f"- **Compiler hardening:** `{flags_str}`") + if l2.hardening_rationale: + lines.append(f"- **Hardening rationale:** {l2.hardening_rationale.strip()}") + if l2.l2_override_verdict: + lines.append(f"- **Build agent override:** `{l2.l2_override_verdict}`") + elif l2.hardening_relevant is False: + lines.append("- **Hardening relevant to CVE:** no") + lines.append("") + return "\n".join(lines).strip() + + +def _filter_review_snippets(snippets: list[CodeSnippet]) -> list[CodeSnippet]: + """Prefer primary source files over build-system noise when both are present.""" + primary = [ + s for s in snippets + if s.file_path.endswith((".c", ".h", ".cpp", ".cc", ".cxx")) + and "/test/" not in s.file_path + and "CMakeLists" not in s.file_path + and "Makefile" not in s.file_path + ] + return primary if primary else snippets + + +def _is_reference_tree_path(file_path: str) -> bool: + path_lower = file_path.lower() + return "_patched" in path_lower or "/patched/" in path_lower + + +@dataclass +class ReportBlocks: + """Formatted report blocks - each piece of data formatted once for UI output.""" + + # Package info + package_name: str + package_version: str + package_release: str + package_arch: str + + # CVE info + cve_id: str + cve_description: str + + # Verdict + justification_label: str + executive_summary: str + + # Evidence + evidence_chain: list[str] + affected_files: list[str] + + # Extracted facts from downstream search + patch_file_name: str + spec_patch_directives: list[str] + build_log_evidence: str + spec_changelog_cve_lines: str + spec_version_line: str + spec_source0_line: str + is_patch_file_available: bool + is_patch_in_spec_file: bool + is_patch_applied_in_build: bool + + # Code snippets + vulnerable_snippets: list[CodeSnippet] + fix_snippets: list[CodeSnippet] + + # Investigation context (upstream / L1 / L2 / git search) + upstream_report: UpstreamSearchReport | None = None + git_search_report: GitSearchReport | None = None + l1_agent_answer: str | None = None + investigation_mode: str | None = None + arch_gate_reason: str | None = None + l2_result: L2BuildResult | None = None + fixed_rpm_hints: list[str] = field(default_factory=list) + upstream_search_ran: bool = False + + @property + def package_header_md(self) -> str: + """Format package metadata as Markdown header.""" + version_release = f"{self.package_version}-{self.package_release}" if self.package_release else self.package_version + return f"**Package:** `{self.package_name}-{version_release}` ({self.package_arch})" + + @property + def evidence_chain_md(self) -> str: + """Investigation narrative for the details field (four sections, always present).""" + sections = [ + _format_target_fix_search_md(self), + _format_external_fix_clues_md(self), + _format_target_source_check_md(self), + _format_target_build_check_md(self), + ] + return "\n\n".join(s for s in sections if s).strip() + + @property + def affected_files_md(self) -> str: + """Format affected files as Markdown list.""" + if not self.affected_files: + return "" + lines = ["**Affected Files:**"] + for f in self.affected_files[:10]: + lines.append(f"- `{f}`") + return "\n".join(lines) + + @property + def details_md(self) -> str: + """Source snippets are rendered inside evidence_chain_md (target source check).""" + return "" + + def _format_snippet_bullets(self, snippets: list[CodeSnippet]) -> list[str]: + lines: list[str] = [] + for snippet in snippets: + display_path = snippet.file_path + if len(display_path) > 72: + display_path = Path(display_path).name + line_part = f" (line {snippet.line_number})" if snippet.line_number else "" + lines.append(f"### `{display_path}`{line_part}") + if _is_reference_tree_path(snippet.file_path): + lines.append( + "> **Note:** Excerpt path is from a reference/patched tree, not the target SRPM source." + ) + lines.append("") + lang = infer_language_from_path(snippet.file_path) + code_body = "\n".join(snippet.code.strip().splitlines()[:12]) + if code_body: + lines.append(f"```{lang}") + lines.append(code_body) + lines.append("```") + lines.append("") + return lines + +_NO_PATCH_EVIDENCE_PHRASE = ( + "no CVE fix evidence was found on the target (no patch file, spec reference, " + "or build-time application)" +) + +_REASON_BY_LABEL: dict[str, str] = { + "vulnerable": ( + "Labeled vulnerable because the code agent confirmed vulnerable patterns " + "remain in source." + ), + "protected_by_mitigating_control": ( + "Labeled protected by mitigating control because a downstream patch or fix pattern " + "addresses the vulnerability in this package." + ), + "protected_by_compiler": ( + "Labeled protected by compiler because the build agent found compiler hardening " + "that mitigates exploitation of this flaw." + ), + "code_not_present": ( + "Labeled code not present because the build agent determined the vulnerable code " + "is not compiled into this package." + ), + "code_not_reachable": ( + "Labeled code not reachable because the vulnerable code path is not reachable " + "in this package build." + ), + "requires_environment": ( + "Labeled requires environment because exploitation depends on runtime or architecture " + "conditions not met on this target." + ), + "uncertain": ( + "Labeled uncertain because patch, spec, build, and source evidence are " + "insufficient or conflicting." + ), + "conflicting_evidence": ( + "Labeled conflicting evidence because RHSA assessed this package as 'not affected' " + "but code analysis found vulnerable patterns. Manual review required to determine " + "if this is an RHSA assessment error or an analysis false positive." + ), +} + + +def _all_patch_checks_failed(blocks: ReportBlocks) -> bool: + return not ( + blocks.is_patch_file_available + or blocks.is_patch_in_spec_file + or blocks.is_patch_applied_in_build + ) + + +def _format_justification_reason(blocks: ReportBlocks, label_override: str | None = None) -> str: + """One-sentence VEX label rationale (outcome voice); evidence lives in details. + + Args: + blocks: Report blocks containing investigation data + label_override: Optional label to use instead of blocks.justification_label + (used when label is changed due to conflict detection) + """ + label = label_override if label_override else blocks.justification_label + if label == "vulnerable": + if _all_patch_checks_failed(blocks): + return ( + f"Labeled vulnerable because {_NO_PATCH_EVIDENCE_PHRASE}, and the code agent " + "confirmed vulnerable patterns remain in source." + ) + return _REASON_BY_LABEL["vulnerable"] + if label == "uncertain": + if _all_patch_checks_failed(blocks): + return ( + f"Labeled uncertain because {_NO_PATCH_EVIDENCE_PHRASE}, and source evidence " + "is insufficient or conflicting." + ) + return _REASON_BY_LABEL["uncertain"] + return _REASON_BY_LABEL.get(label, _REASON_BY_LABEL["uncertain"]) + + +_BUILD_AGENT_ABSENCE_PHRASES = ( + "build agent context is not present", + "l2 context is not present", + "l2_build_context is not present", + "based solely on the code agent analysis", + "verdict is based solely on the code agent", + "because the build agent context is not present", + "build agent did not run", + "no build agent context", +) + + +def _strip_build_agent_absence_boilerplate(text: str) -> str: + """Remove LLM sentences that misstate build-agent availability.""" + parts = re.split(r"(?<=[.!?])\s+", text.strip()) + kept = [ + part + for part in parts + if part and not any(phrase in part.lower() for phrase in _BUILD_AGENT_ABSENCE_PHRASES) + ] + return " ".join(kept).strip() + + +NON_CODE_LANGUAGES = frozenset({"markdown", "yaml", "json", "toml", "xml", "rst", "text", "ini", "csv"}) + + +def infer_language_from_path(file_path: str) -> str: + """Infer programming language hint from file extension for syntax highlighting.""" + ext_map = { + ".c": "c", + ".h": "c", + ".cpp": "cpp", + ".hpp": "cpp", + ".cc": "cpp", + ".cxx": "cpp", + ".py": "python", + ".go": "go", + ".rs": "rust", + ".java": "java", + ".js": "javascript", + ".ts": "typescript", + ".rb": "ruby", + ".sh": "bash", + ".md": "markdown", + ".rst": "rst", + ".txt": "text", + ".yml": "yaml", + ".yaml": "yaml", + ".json": "json", + ".xml": "xml", + ".toml": "toml", + ".cfg": "ini", + ".ini": "ini", + ".csv": "csv", + } + return ext_map.get(Path(file_path).suffix.lower(), "") + + +def _build_details_md(blocks: ReportBlocks) -> str | None: + """Audit packet: investigation narrative (four sections, includes source excerpts).""" + investigation = blocks.evidence_chain_md.strip() + return investigation or None + + +def _build_report_blocks( + message: AgentMorpheusEngineInput, + code_agent_report: CodeAgentReport, + cve_description: str, + downstream_report: DownstreamSearchReport | None, + upstream_report: UpstreamSearchReport | None = None, + l1_result: L1InvestigationResult | None = None, + l2_result: L2BuildResult | None = None, + identify_result=None, + git_search_report: GitSearchReport | None = None, +) -> ReportBlocks: + """Extract and format all report data into blocks.""" + target_package = message.input.image.target_package + + vulnerable_snippets = [s for s in code_agent_report.code_snippets if s.snippet_type == "vulnerable"] + fix_snippets = [s for s in code_agent_report.code_snippets if s.snippet_type == "fix"] + + patch_file_name = "" + spec_patch_directives: list[str] = [] + build_log_evidence = "" + spec_changelog_cve_lines = "" + spec_version_line = "" + spec_source0_line = "" + is_patch_file_available = False + is_patch_in_spec_file = False + is_patch_applied_in_build = False + + if downstream_report: + patch_file_name = downstream_report.patch_file_name or "" + spec_patch_directives = downstream_report.spec_patch_directives_for_cve or [] + build_log_evidence = downstream_report.build_log_patch_applied or "" + spec_changelog_cve_lines = downstream_report.spec_changelog_cve_lines or "" + spec_version_line = downstream_report.spec_version_line or "" + spec_source0_line = downstream_report.spec_source0_line or "" + is_patch_file_available = downstream_report.is_patch_file_available + is_patch_in_spec_file = downstream_report.is_patch_in_spec_file + is_patch_applied_in_build = downstream_report.is_patch_applied_in_build + + investigation_mode = _derive_investigation_mode(downstream_report, upstream_report) + upstream_search_ran = not ( + downstream_report is not None and downstream_report.is_patch_file_available + ) + + l1_agent_answer = l1_result.l1_agent_answer if l1_result else None + affected_bitness = None + if l1_result and l1_result.vulnerability_intel: + affected_bitness = l1_result.vulnerability_intel.affected_bitness + + package_arch = target_package.arch or "x86_64" if target_package else "x86_64" + arch_gate_reason = _derive_arch_gate_reason(l1_agent_answer, affected_bitness, package_arch) + + fixed_rpm_hints: list[str] = [] + if identify_result and identify_result.fixed_rpm_list: + fixed_rpm_hints = list(identify_result.fixed_rpm_list[:_POLICY_MAX_RPM_LIST_ITEMS]) + + return ReportBlocks( + package_name=target_package.name if target_package else "unknown", + package_version=target_package.version or "" if target_package else "", + package_release=target_package.release or "" if target_package else "", + package_arch=package_arch, + cve_id=message.input.scan.vulns[0].vuln_id if message.input.scan.vulns else "", + cve_description=cve_description, + justification_label=code_agent_report.justification_label, + executive_summary=code_agent_report.executive_summary, + evidence_chain=list(code_agent_report.evidence_chain), + affected_files=list(code_agent_report.affected_files), + patch_file_name=patch_file_name, + spec_patch_directives=spec_patch_directives, + build_log_evidence=build_log_evidence, + spec_changelog_cve_lines=spec_changelog_cve_lines, + spec_version_line=spec_version_line, + spec_source0_line=spec_source0_line, + is_patch_file_available=is_patch_file_available, + is_patch_in_spec_file=is_patch_in_spec_file, + is_patch_applied_in_build=is_patch_applied_in_build, + vulnerable_snippets=vulnerable_snippets, + fix_snippets=fix_snippets, + upstream_report=upstream_report, + git_search_report=git_search_report, + l1_agent_answer=l1_agent_answer, + investigation_mode=investigation_mode, + arch_gate_reason=arch_gate_reason, + l2_result=l2_result, + fixed_rpm_hints=fixed_rpm_hints, + upstream_search_ran=upstream_search_ran, + ) + + +def _format_policy_context_for_report( + *, + target_nvr: str, + identify_result, + intel, +) -> str: + """Build a context block for the LLM prompt covering NVR posture and RHSA excerpts.""" + lines: list[str] = [] + + if target_nvr: + lines.append(f"**Scanned target NVR:** `{target_nvr}`") + + if identify_result: + affected = identify_result.affected_rpm_list or [] + fixed = identify_result.fixed_rpm_list or [] + + if affected: + shown = affected[:_POLICY_MAX_RPM_LIST_ITEMS] + suffix = f" (+ {len(affected) - len(shown)} more)" if len(affected) > len(shown) else "" + lines.append(f"**Affected NVRs from identify:** {', '.join(f'`{n}`' for n in shown)}{suffix}") + lines.append(f" - is_target_package_affected: `{identify_result.is_target_package_affected.value}`") + + if fixed: + shown = fixed[:_POLICY_MAX_RPM_LIST_ITEMS] + suffix = f" (+ {len(fixed) - len(shown)} more)" if len(fixed) > len(shown) else "" + lines.append(f"**Fixed NVRs from identify:** {', '.join(f'`{n}`' for n in shown)}{suffix}") + lines.append(f" - is_target_package_fixed: `{identify_result.is_target_package_fixed.value}`") + + rhsa = None + if intel and len(intel) > 0: + rhsa = intel[0].rhsa + + if rhsa: + if rhsa.statement: + stmt = rhsa.statement + if len(stmt) > _POLICY_RHSA_STATEMENT_CAP: + stmt = stmt[:_POLICY_RHSA_STATEMENT_CAP] + " …" + lines.append(f"**RHSA statement excerpt:** {stmt}") + + if rhsa.upstream_fix: + lines.append(f"**RHSA upstream_fix:** `{rhsa.upstream_fix}`") + + pkg_states = rhsa.package_state or [] + if pkg_states: + lines.append("**RHSA package_state:**") + for ps in pkg_states[:_POLICY_MAX_PACKAGE_STATE_ITEMS]: + parts = [] + if ps.product_name: + parts.append(ps.product_name) + if ps.package_name: + parts.append(f"pkg={ps.package_name}") + if ps.fix_state: + parts.append(f"fix_state={ps.fix_state}") + if parts: + lines.append(f" - {' | '.join(parts)}") + if len(pkg_states) > _POLICY_MAX_PACKAGE_STATE_ITEMS: + lines.append(f" - (+ {len(pkg_states) - _POLICY_MAX_PACKAGE_STATE_ITEMS} more)") + + return "\n".join(lines) + + +def _apply_l2_verdict( + report: CodeAgentReport, + l2_result: L2BuildResult, +) -> CodeAgentReport: + """Apply L2 Build Agent verdict overrides to the CodeAgentReport. + + .. deprecated:: + This function is deprecated. L2 results are now passed directly to + `generate_code_agent_report()` so the LLM can synthesize L1 and L2 + findings into a cohesive narrative. This function will be removed + in a future release. + """ + warnings.warn( + "_apply_l2_verdict is deprecated. L2 results are now integrated " + "directly into the LLM prompt via generate_code_agent_report().", + DeprecationWarning, + stacklevel=2, + ) + if l2_result.l2_override_verdict is None: + return report + + updated_fields = {} + + if l2_result.l2_override_verdict == "not_vulnerable": + evidence = l2_result.compilation_evidence or "" + if "Architecture mismatch" in evidence: + # Architecture-based not affected - vulnerability cannot occur on this platform + updated_fields["justification_label"] = "requires_environment" + updated_fields["executive_summary"] = ( + f"{report.executive_summary}\n\n" + f"**L2 Override:** {evidence} " + f"Vulnerability condition cannot occur on this architecture." + ) + elif l2_result.compilation_status == "not_compiled": + updated_fields["justification_label"] = "code_not_present" + updated_fields["executive_summary"] = ( + f"{report.executive_summary}\n\n" + f"**L2 Override:** Vulnerable code is NOT compiled into the binary. " + f"Evidence: {evidence or 'Build analysis confirmed exclusion.'}" + ) + else: + updated_fields["justification_label"] = "code_not_reachable" + updated_fields["executive_summary"] = ( + f"{report.executive_summary}\n\n" + f"**L2 Override:** Code determined not vulnerable by L2 analysis." + ) + + elif l2_result.l2_override_verdict == "vulnerable_mitigated": + if l2_result.hardening_relevant and l2_result.hardening_flags: + updated_fields["justification_label"] = "protected_by_compiler" + flags_str = ", ".join(l2_result.hardening_flags[:5]) + updated_fields["executive_summary"] = ( + f"{report.executive_summary}\n\n" + f"**L2 Override:** Vulnerability mitigated by compiler hardening flags: {flags_str}. " + f"Rationale: {l2_result.hardening_rationale or 'Hardening flags provide protection.'}" + ) + else: + updated_fields["justification_label"] = "protected_by_mitigating_control" + updated_fields["executive_summary"] = ( + f"{report.executive_summary}\n\n" + f"**L2 Override:** Vulnerability mitigated by build-time controls." + ) + + if updated_fields: + evidence = list(report.evidence_chain) + evidence.append(f"L2 Build Agent: {l2_result.l2_override_verdict}") + if l2_result.compilation_evidence: + evidence.append(f"L2 compilation evidence: {l2_result.compilation_evidence}") + if l2_result.hardening_rationale: + evidence.append(f"L2 hardening rationale: {l2_result.hardening_rationale}") + updated_fields["evidence_chain"] = evidence + + return report.model_copy(update=updated_fields) + + return report + + +def _build_analysis( + message: AgentMorpheusEngineInput, + code_agent_report: CodeAgentReport, + intel_score: int, + cve_description: str = "", + downstream_report: DownstreamSearchReport | None = None, + upstream_report: UpstreamSearchReport | None = None, + l1_result: L1InvestigationResult | None = None, + l2_result: L2BuildResult | None = None, +) -> list[AgentMorpheusEngineOutput]: + """Build the final analysis output from the code agent report using ReportBlocks. + + - summary: LLM executive_summary (verdict, reconciliation, technical context) + - justification.reason: one-sentence template from label and patch booleans + - details: investigation narrative markdown (target fix search through build check) + """ + ctx = message.info.checker_context if message.info else None + + git_search_report: GitSearchReport | None = None + if l1_result and l1_result.git_search_report: + git_search_report = GitSearchReport.model_validate(l1_result.git_search_report) + + blocks = _build_report_blocks( + message, + code_agent_report, + cve_description, + downstream_report, + upstream_report=upstream_report, + l1_result=l1_result, + l2_result=l2_result, + identify_result=ctx.identify_result if ctx else None, + git_search_report=git_search_report, + ) + + label = blocks.justification_label + status: _StatusLiteral = _JUSTIFICATION_LABEL_TO_STATUS.get(label, "UNKNOWN") + + # Check for conflict between RHSA "not affected" assessment and analysis verdict + if _detect_intel_conflict(ctx, label): + logger.warning( + "Intel conflict detected: RHSA says 'not affected' but analysis concluded 'vulnerable'. " + "Flagging for human review." + ) + label = "conflicting_evidence" + status = "UNKNOWN" + ctx.status = PackageCheckerStatus.CONFLICTING_INTEL_EVIDENCE + + summary = _strip_build_agent_absence_boilerplate(blocks.executive_summary.strip()) + reason = _format_justification_reason(blocks, label_override=label if label != blocks.justification_label else None) + details = _build_details_md(blocks) + + return [ + AgentMorpheusEngineOutput( + vuln_id=intel.vuln_id, + checklist=[], + summary=summary, + justification=JustificationOutput( + label=label, + reason=reason, + status=status, + ), + intel_score=intel_score, + cvss=None, + details=details, + ) + for intel in (message.info.intel if message.info and message.info.intel else []) + ] + + +class CVECheckerReportConfig(FunctionBaseConfig, name="cve_checker_report"): + """Configuration for the CVE Checker Report generation function.""" + base_checker_dir: str = Field( + default=".cache/am_cache/checker", + description="Root directory for checker-specific artifacts.", + ) + llm_name: str = Field(description="The LLM model to use for report generation.") + + +@register_function(config_type=CVECheckerReportConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def cve_checker_report(config: CVECheckerReportConfig, builder: Builder): + """Report generation function for the L1/L2 checker pipeline.""" + + async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusOutput: + """Generate the final checker report from L1 (and optionally L2) results.""" + trace_id.set(message.input.scan.id) + tracer = Context.get() + + logger.info("cve_checker_report: starting report generation") + + ctx = message.info.checker_context + if ctx is None or ctx.l1_result is None: + logger.error("cve_checker_report: no L1 result available") + return AgentMorpheusOutput( + input=message.input, + info=message.info, + output=OutputPayload( + analysis=[ + AgentMorpheusEngineOutput( + vuln_id=intel.vuln_id, + checklist=[], + summary="Rpm scanning investigation did not produce results.", + justification=JustificationOutput( + label="uncertain", + reason="Rpm scanning investigation did not produce results.", + status="UNKNOWN", + ), + intel_score=0, + cvss=None, + details=None, + ) + for intel in (message.info.intel if message.info and message.info.intel else []) + ], + vex=None, + ), + ) + + l1_result = ctx.l1_result + l2_result = ctx.l2_result + + downstream_report: DownstreamSearchReport | None = None + upstream_report: UpstreamSearchReport | None = None + git_search_report: GitSearchReport | None = None + + if l1_result.downstream_report: + downstream_report = DownstreamSearchReport.model_validate(l1_result.downstream_report) + if l1_result.upstream_report: + upstream_report = UpstreamSearchReport.model_validate(l1_result.upstream_report) + if l1_result.git_search_report: + git_search_report = GitSearchReport.model_validate(l1_result.git_search_report) + + vuln_id = message.input.scan.vulns[0].vuln_id + target_package = message.input.image.target_package + target_package_name = target_package.name if target_package else "unknown" + intel = message.info.intel + + descriptions: list[tuple[str, str]] = [] + if intel: + a_intel = intel[0] + if a_intel.ghsa: + cve_text = a_intel.ghsa.description or a_intel.ghsa.summary or "" + if cve_text: + descriptions.append(("ghsa", cve_text)) + if a_intel.ubuntu and a_intel.ubuntu.description: + descriptions.append(("ubuntu", a_intel.ubuntu.description)) + + version = (target_package.version or "") if target_package else "" + release = (target_package.release or "") if target_package else "" + target_nvr = f"{target_package_name}-{version}-{release}" if target_package_name else "" + + policy_context = _format_policy_context_for_report( + target_nvr=target_nvr, + identify_result=ctx.identify_result, + intel=intel, + ) + + llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + + with tracer.push_active_function("generate_code_agent_report", input_data={"vuln_id": vuln_id}): + code_agent_report: CodeAgentReport = await generate_code_agent_report( + llm=llm, + vuln_id=vuln_id, + target_package=target_package_name, + descriptions=descriptions, + downstream_report=downstream_report, + upstream_report=upstream_report, + git_search_report=git_search_report, + l1_agent_answer=l1_result.l1_agent_answer, + tracer=tracer, + policy_context=policy_context, + l2_result=l2_result, + ) + + source_key = ctx.source_key + if source_key: + report_dir = Path(config.base_checker_dir) / source_key / "report" + report_dir.mkdir(parents=True, exist_ok=True) + suffix = f"-{target_package_name}" if target_package_name else "" + if version: + suffix += f"-{version}" + if release: + suffix += f"-{release}" + report_path = report_dir / f"L1_report_{vuln_id}{suffix}.md" + report_path.write_text(code_agent_report.to_markdown( + vuln_id=vuln_id, + target_package=target_package_name, + version=version, + release=release, + downstream_report=downstream_report, + )) + logger.info("cve_checker_report: wrote report to %s", report_path) + + with tracer.push_active_function( + "report_finish", + input_data={ + "justification_label": code_agent_report.justification_label, + "has_l2_override": l2_result is not None and l2_result.l2_override_verdict is not None, + }, + ) as span: + span.set_output({ + "executive_summary": code_agent_report.executive_summary, + "affected_files": code_agent_report.affected_files, + }) + intel_score = intel[0].intel_score + + cve_description = "" + if descriptions: + cve_description = descriptions[0][1] + + return AgentMorpheusOutput( + input=message.input, + info=message.info, + output=OutputPayload( + analysis=_build_analysis( + message, + code_agent_report, + intel_score, + cve_description=cve_description, + downstream_report=downstream_report, + upstream_report=upstream_report, + l1_result=l1_result, + l2_result=l2_result, + ), + vex=None, + ), + ) + + yield FunctionInfo.from_fn( + _arun, + description="Generate final checker report from L1/L2 investigation results", + ) diff --git a/src/vuln_analysis/functions/cve_checker_segmentation.py b/src/vuln_analysis/functions/cve_checker_segmentation.py new file mode 100644 index 000000000..bae7f31ee --- /dev/null +++ b/src/vuln_analysis/functions/cve_checker_segmentation.py @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +from pathlib import Path + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from langchain.docstore.document import Document +from exploit_iq_commons.utils.document_embedding import MultiLanguageRecursiveCharacterTextSplitter,ExtendedLanguageParser +from langchain_community.document_loaders.generic import GenericLoader +from langchain.text_splitter import RecursiveCharacterTextSplitter +logger = LoggingFactory.get_agent_logger(__name__) + +_BUILD_FILE_NAMES = {"Makefile", "GNUmakefile", "configure"} +LANG_PARSER_EXTENSIONS = {".c", ".h", ".cpp", ".hpp", ".py", ".go", ".java"} +TEXT_FILE_EXTENSIONS = {".spec", ".conf", ".cfg", ".sh", ".m4",".ac", ".am", ".in", ".txt", ".md", ".rst"} +class RpmDocumentEmbedding: + def __init__(self, source_dir: Path, chunk_size: int = 800, chunk_overlap: int = 160): + self.source_dir = source_dir + self.lang_splitter = MultiLanguageRecursiveCharacterTextSplitter( + chunk_size=chunk_size, chunk_overlap=chunk_overlap, + ) + self.text_splitter = RecursiveCharacterTextSplitter( + chunk_size=1000, chunk_overlap=200, + ) + + def load_and_chunk_code(self) -> list[Document]: + loader = GenericLoader.from_filesystem( + self.source_dir, + glob="**/*", + suffixes=list(LANG_PARSER_EXTENSIONS), + parser=ExtendedLanguageParser(), + ) + try: + documents = loader.load() + except Exception as e: + logger.warning("LanguageParser failed on %s: %s", self.source_dir, e) + return [] + return self.lang_splitter.split_documents(documents) + + def load_and_chunk_files(self) -> list[Document]: + documents: list[Document] = [] + for root, _, files in os.walk(self.source_dir): + for file in files: + if any(file.endswith(ext) for ext in TEXT_FILE_EXTENSIONS) or file in _BUILD_FILE_NAMES: + file_path = os.path.join(root, file) + try: + with open(file_path, "r") as f: + content = f.read() + documents.append(Document(page_content=content, metadata={"source": file_path})) + except Exception as e: + logger.warning("Error reading %s: %s", file_path, e) + continue + return self.text_splitter.split_documents(documents) + + def load_and_chunk_all(self) -> list[Document]: + documents = self.load_and_chunk_code() + documents.extend(self.load_and_chunk_files()) + return documents + + + + +class CVECheckerSegmentationConfig(FunctionBaseConfig, name="cve_checker_segmentation"): + """ + Builds a scoped Tantivy lexical code index from extracted RPM source files. + Reads source directories populated by source_acquisition, indexes them, + and sets info.vdb.code_index_path for downstream checker nodes. + """ + base_checker_dir: str = Field( + default=".cache/am_cache/checker", + description="Root directory for checker-specific artifacts.", + ) + base_code_index_dir: str = Field( + default=".cache/am_cache/code_index", + description="Base directory for Tantivy code index storage.", + ) + include_extensions: list[str] = Field( + default=[ + ".c", ".h", ".cpp", ".hpp", ".py", ".go", ".java", + ".spec", ".patch", ".conf", ".cfg", ".sh", ".m4", + ".ac", ".am", ".in", ".txt", ".md", ".rst", + ], + description="File extensions to include when building the code index.", + ) + + +@register_function(config_type=CVECheckerSegmentationConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def cve_checker_segmentation(config: CVECheckerSegmentationConfig, builder: Builder): + from exploit_iq_commons.data_models.info import AgentMorpheusInfo + from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput + from vuln_analysis.utils.full_text_search import FullTextSearch + + async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + if not message.info.checker_context or not message.info.checker_context.source_key: + logger.info("checker_segmentation: no checker_context.source_keys, skipping indexing") + return message + + source_key = message.info.checker_context.source_key + if not source_key: + logger.info("checker_segmentation: no source_key, skipping indexing") + return message + + index_path = FullTextSearch.get_index_directory(config.base_code_index_dir, source_key) + + if index_path.exists(): + logger.info("checker_segmentation: cache hit on code index: %s", index_path) + else: + start = time.time() + fts = FullTextSearch(cache_path=str(index_path)) + + source_dir = Path(config.base_checker_dir) / source_key / "source" + if not source_dir.is_dir(): + logger.warning("checker_segmentation: source dir missing: %s", source_dir) + return message + + logger.info("checker_segmentation: indexing source dir %s", source_dir) + document_embedding = RpmDocumentEmbedding(source_dir=source_dir) + documents = document_embedding.load_and_chunk_all() + + + fts.add_documents_from_langchain_chunks(documents) + + elapsed = time.time() - start + logger.info("checker_segmentation: indexing completed in %.2fs at %s", elapsed, index_path) + + message.info.vdb = AgentMorpheusInfo.VdbPaths(code_index_path=str(index_path)) + return message + + yield FunctionInfo.from_fn( + _arun, + description="Build scoped Tantivy code index from extracted checker sources", + ) + + +def _index_build_files(fts, source_dir: Path) -> None: + """Walk source_dir for extensionless build files and add them to the index.""" + docs: list[tuple[str, str]] = [] + for root, _, files in os.walk(source_dir): + for fname in files: + if fname in _BUILD_FILE_NAMES: + fpath = os.path.join(root, fname) + try: + with open(fpath, "r", encoding="utf-8", errors="replace") as f: + docs.append((fpath, f.read())) + except Exception as exc: + logger.warning("checker_segmentation: error reading %s: %s", fpath, exc) + if docs: + fts.add_documents(docs) + logger.info("checker_segmentation: indexed %d build files from %s", len(docs), source_dir) diff --git a/src/vuln_analysis/functions/cve_checklist.py b/src/vuln_analysis/functions/cve_checklist.py index 23453c9d0..ab6f3c985 100644 --- a/src/vuln_analysis/functions/cve_checklist.py +++ b/src/vuln_analysis/functions/cve_checklist.py @@ -14,7 +14,6 @@ # limitations under the License. import asyncio -import logging import pandas as pd from aiq.builder.builder import Builder @@ -22,9 +21,12 @@ from aiq.builder.function_info import FunctionInfo from aiq.cli.register_workflow import register_function from aiq.data_models.function import FunctionBaseConfig +from aiq.data_models.component_ref import FunctionRef from pydantic import Field +from exploit_iq_commons.utils import data_utils +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id -logger = logging.getLogger(__name__) +logger = LoggingFactory.get_agent_logger(__name__) class CVEChecklistToolConfig(FunctionBaseConfig, name="cve_checklist"): @@ -32,6 +34,10 @@ class CVEChecklistToolConfig(FunctionBaseConfig, name="cve_checklist"): Defines a function that generates tailored, context-sensitive task checklist for impact analysis. """ llm_name: str = Field(description="The LLM model to use") + agent_name: FunctionRef = Field( + default="cve_agent_executor", + description="Name of agent function to get tool configuration from" + ) prompt: str | None = Field( default=None, description= @@ -48,22 +54,30 @@ async def cve_checklist(config: CVEChecklistToolConfig, builder: Builder): llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) - async def generate_checklist_for_cve(cve_intel): + # Get agent tool configuration + agent_config = builder.get_function_config(config.agent_name) + agent_tool_names = agent_config.tool_names if hasattr(agent_config, 'tool_names') else None + + async def generate_checklist_for_cve(cve_intel, ecosystem: str = ""): checklist = await generate_checklist(prompt=config.prompt, llm=llm, input_dict=cve_intel, - enable_llm_list_parsing=False) + tool_names=agent_tool_names, + enable_llm_list_parsing=False, + ecosystem=ecosystem) checklist = await _parse_list([checklist]) return cve_intel["vuln_id"], checklist[0] async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: - intel_df = pd.json_normalize([x.model_dump(mode="json") for x in state.cve_intel], sep="_") + trace_id.set(state.original_input.input.scan.id) + ecosystem = state.original_input.input.image.ecosystem.value if state.original_input and state.original_input.input.image.ecosystem else "" + intel_df = data_utils.merge_intel_and_plugin_data_convert_to_dataframe(state.cve_intel) workflow_cve_intel = intel_df.to_dict(orient='records') - results = await asyncio.gather(*(generate_checklist_for_cve(cve_intel) for cve_intel in workflow_cve_intel)) + results = await asyncio.gather(*(generate_checklist_for_cve(cve_intel, ecosystem=ecosystem) for cve_intel in workflow_cve_intel)) state.checklist_plans = dict(results) return state diff --git a/src/vuln_analysis/functions/cve_clone_and_deps.py b/src/vuln_analysis/functions/cve_clone_and_deps.py new file mode 100644 index 000000000..a7e2117ab --- /dev/null +++ b/src/vuln_analysis/functions/cve_clone_and_deps.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Early-phase node: clone repositories and install dependencies. + +This node runs early in the pipeline (before verify_vuln_package) to prepare +the codebase. Expensive VDB/indexing work is deferred to cve_segmentation, +which can be skipped if verify_vuln_package determines no CVE is vulnerable. +""" + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.data_models.common import AnalysisType +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.utils.credential_client import credential_context +from exploit_iq_commons.utils.dep_tree import detect_ecosystem + +logger = LoggingFactory.get_agent_logger(__name__) + + +class CVECloneAndDepsConfig(FunctionBaseConfig, name="cve_clone_and_deps"): + """ + Configuration for the clone-and-deps node. + + This node clones source repositories and installs dependencies early in + the pipeline, before vulnerability verification. VDB/indexing is deferred. + """ + + base_git_dir: str = Field( + default=".cache/am_cache/git", + description="Directory for storing cloned git repositories.", + ) + base_pickle_dir: str = Field( + default=".cache/am_cache/pickle", + description="Directory for storing pickle cache files.", + ) + base_rpm_dir: str = Field( + default=".cache/am_cache/rpms", + description="Directory for storing RPM files.", + ) + ignore_errors: bool = Field( + default=False, + description="Whether to ignore errors during clone/install.", + ) + + +@register_function(config_type=CVECloneAndDepsConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def clone_and_deps(config: CVECloneAndDepsConfig, builder: Builder): + from exploit_iq_commons.data_models.info import AgentMorpheusInfo + from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput + from exploit_iq_commons.data_models.input import AgentMorpheusInput + from exploit_iq_commons.data_models.input import ManualSBOMInfoInput + from exploit_iq_commons.utils.document_embedding import DocumentEmbedding + from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager + + rpm_manager = RPMDependencyManager.get_instance() + rpm_manager.set_rpm_cache_dir(config.base_rpm_dir) + + embedder = DocumentEmbedding( + embedding=None, + git_directory=config.base_git_dir, + pickle_cache_directory=config.base_pickle_dir, + ) + + async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput: + """ + Clone repositories and install dependencies. + + This is the first phase of the split pipeline. VDB/indexing happens + later in segmentation (if the package turns out to be vulnerable). + """ + trace_id.set(message.scan.id) + source_infos = message.image.source_info + sbom_infos = message.image.sbom_info + failure_reason = "" + + try: + logger.debug( + "clone_and_deps: credential_id=%r scan_id=%s", + message.credential_id, + message.scan.id, + ) + + with credential_context(message.credential_id): + # Configure RPM manager for IMAGE analysis + if message.image.analysis_type == AnalysisType.IMAGE and isinstance( + sbom_infos, ManualSBOMInfoInput + ): + rpm_manager.sbom = sbom_infos.packages + image = f"{message.image.name}:{message.image.tag}" + rpm_manager.container_image = image + + # Clone and install dependencies for each source + code_sources = [si for si in source_infos if si.type == "code"] + for si in code_sources: + try: + embedder.clone_and_install_dependencies(si) + except Exception as e: + logger.warning( + "Error cloning/installing for source %s: %s", + si.git_repo, + e, + exc_info=True, + ) + if not config.ignore_errors: + raise + + # Detect ecosystem from cloned repo manifests if not provided + if message.image.ecosystem is None and code_sources: + repo_path = embedder.get_repo_path(code_sources[0]) + detected = detect_ecosystem(repo_path) + if detected is not None: + message.image.ecosystem = detected + logger.info( + "Detected ecosystem '%s' from repo manifests", + detected.value, + ) + + except Exception as e: + failure_reason = f"Clone/install failed: {e}" + logger.error( + "clone_and_deps failed for image '%s': %s", + message.image.name, + e, + exc_info=True, + ) + if not config.ignore_errors: + raise + + if failure_reason: + message.failure_reason = failure_reason + + info = AgentMorpheusInfo() + return AgentMorpheusEngineInput(input=message, info=info) + + yield FunctionInfo.from_fn( + _arun, + input_schema=AgentMorpheusInput, + description="Clone repositories and install dependencies (early phase).", + ) diff --git a/src/vuln_analysis/functions/cve_fetch_intel.py b/src/vuln_analysis/functions/cve_fetch_intel.py index 8141c4a39..b21b9238a 100644 --- a/src/vuln_analysis/functions/cve_fetch_intel.py +++ b/src/vuln_analysis/functions/cve_fetch_intel.py @@ -14,7 +14,7 @@ # limitations under the License. import asyncio -import logging +import typing import aiohttp from aiq.builder.builder import Builder @@ -24,7 +24,11 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -logger = logging.getLogger(__name__) +from vuln_analysis.data_models.plugin import PluginConfig + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id + +logger = LoggingFactory.get_agent_logger(__name__) class CVEFetchIntelConfig(FunctionBaseConfig, name="cve_fetch_intel"): @@ -32,21 +36,27 @@ class CVEFetchIntelConfig(FunctionBaseConfig, name="cve_fetch_intel"): Defines a function that fetches details about CVEs from NIST and CVE Details websites. """ retry_on_client_errors: bool = Field(default=True, description="Whether to retry if HTTP client cannot connect") + intel_plugin_config: PluginConfig = Field(description="Intel Plugin configuration") + rpm_user_type: str = Field( + default="external", + description="User profile: 'internal' (Red Hat VPN) enables OSIDB intel, 'external' disables it." + ) @register_function(config_type=CVEFetchIntelConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def cve_fetch_intel(config: CVEFetchIntelConfig, builder: Builder): # pylint: disable=unused-argument - from vuln_analysis.data_models.input import AgentMorpheusEngineInput + from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput from vuln_analysis.utils.intel_retriever import IntelRetriever async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: - async def _inner(): - async with aiohttp.ClientSession() as session: - - intel_retriever = IntelRetriever(session=session, retry_on_client_errors=config.retry_on_client_errors) + trace_id.set(message.input.scan.id) + intel_retriever = IntelRetriever(session=session, + retry_on_client_errors=config.retry_on_client_errors, + plugins_config=[config.intel_plugin_config], + rpm_user_type=config.rpm_user_type) intel_coros = [intel_retriever.retrieve(vuln_id=cve.vuln_id) for cve in message.input.scan.vulns] diff --git a/src/vuln_analysis/functions/cve_fetch_patches.py b/src/vuln_analysis/functions/cve_fetch_patches.py new file mode 100644 index 000000000..a78b5acbe --- /dev/null +++ b/src/vuln_analysis/functions/cve_fetch_patches.py @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import contextlib + +import aiohttp +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id + +logger = LoggingFactory.get_agent_logger(__name__) + + +class CVEFetchPatchesConfig(FunctionBaseConfig, name="cve_fetch_patches"): + """Fetches vulnerability fix patches from intel references and OSV for the details UI.""" + llm_name: str | None = Field(default=None, description="Optional LLM for Chromium CL disambiguation") + max_concurrency: int | None = Field(ge=1, default=5, description="Maximum number of concurrent patch fetch operations") + + +@register_function(config_type=CVEFetchPatchesConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def cve_fetch_patches(config: CVEFetchPatchesConfig, builder: Builder): + + from vuln_analysis.data_models.state import AgentMorpheusEngineState + from vuln_analysis.utils.intel_utils import extract_commit_url_candidates + from vuln_analysis.utils.web_patch_fetcher import fetch_patch_for_cve + + llm = ( + await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + if config.llm_name else None + ) + + async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: + trace_id.set(state.original_input.input.scan.id) + + intel_map = {intel.vuln_id: intel for intel in state.cve_intel} + semaphore = asyncio.Semaphore(config.max_concurrency) if config.max_concurrency else None + + async with aiohttp.ClientSession() as session: + + async def _throttled_fetch(vuln_id, intel): + async with semaphore or contextlib.nullcontext(): + candidates = extract_commit_url_candidates(intel) + cve_description = None + if intel.nvd and intel.nvd.cve_description: + cve_description = intel.nvd.cve_description + return await fetch_patch_for_cve( + session=session, + candidates=candidates, + vuln_id=vuln_id, + cve_description=cve_description, + llm=llm, + ) + + tasks = { + vuln_id: _throttled_fetch(vuln_id, intel) + for vuln_id, intel in intel_map.items() + } + + results = await asyncio.gather(*tasks.values(), return_exceptions=True) + + for vuln_id, result in zip(tasks.keys(), results): + if isinstance(result, Exception): + logger.warning("Patch fetch failed for %s: %s", vuln_id, result) + state.patch_results[vuln_id] = None + else: + state.patch_results[vuln_id] = result + + return state + + yield FunctionInfo.from_fn( + _arun, + input_schema=AgentMorpheusEngineState, + description="Fetches vulnerability fix patches from intel references and OSV API.", + ) diff --git a/src/vuln_analysis/functions/cve_file_output.py b/src/vuln_analysis/functions/cve_file_output.py index 2fe53e69f..6e41d198c 100644 --- a/src/vuln_analysis/functions/cve_file_output.py +++ b/src/vuln_analysis/functions/cve_file_output.py @@ -13,15 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging - from aiq.builder.builder import Builder from aiq.builder.function_info import FunctionInfo from aiq.cli.register_workflow import register_function from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -logger = logging.getLogger(__name__) +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class CVEFileOutputConfig(FunctionBaseConfig, name="cve_file_output"): diff --git a/src/vuln_analysis/functions/cve_generate_cvss.py b/src/vuln_analysis/functions/cve_generate_cvss.py new file mode 100644 index 000000000..8343d1c1b --- /dev/null +++ b/src/vuln_analysis/functions/cve_generate_cvss.py @@ -0,0 +1,326 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +from vuln_analysis.runtime_context import ctx_state +import json +import typing +import re + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from langchain.agents import AgentExecutor +from langchain.agents import create_react_agent +from langchain.agents.agent import RunnableAgent +from langchain_core.exceptions import OutputParserException +from langchain_core.prompts import PromptTemplate +from pydantic import Field +from cvss import CVSS3 +from langchain.agents.mrkl.output_parser import MRKLOutputParser + +from vuln_analysis.data_models.state import AgentMorpheusEngineState +from vuln_analysis.tools.tool_names import ToolNames +from vuln_analysis.utils.prompting import get_cvss_prompt + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) + +OUTPUT_CONTAIN_BOTH_ACTION_AND_FINAL_ANSWER = "Parsing LLM output produced both a final answer and a parse-able action" +OUTPUT_CONTAIN_PARSING_ERROR = "Could not parse" + +CVSS_VECTOR_STRING = "vector_string" +CVSS_SCORE = "score" + +DEFAULT_CVSS_METRICS_LIST = [ + { + "metric_name": "Attack Vector", + "metric_abbreviation": "AV", + "question": "Where can the attack be executed from?", + "values": [ + {"value": "Network", "abbreviation": "N", "definition": "Remote attack over the internet.", "least_impact": False}, + {"value": "Adjacent", "abbreviation": "A", "definition": "Requires local network access.", "least_impact": False}, + {"value": "Local", "abbreviation": "L", "definition": "Requires local access to the system.", "least_impact": False}, + {"value": "Physical", "abbreviation": "P", "definition": "Requires physical access.", "least_impact": True} + ] + }, + { + "metric_name": "Attack Complexity", + "metric_abbreviation": "AC", + "question": "How difficult is the attack to execute?", + "values": [ + {"value": "Low", "abbreviation": "L", "definition": "The attack is straightforward.", "least_impact": False}, + {"value": "High", "abbreviation": "H", "definition": "The attack requires special conditions.", "least_impact": True} + ] + }, + { + "metric_name": "Privileges Required", + "metric_abbreviation": "PR", + "question": "What level of access does the attacker need?", + "values": [ + {"value": "None", "abbreviation": "N", "definition": "No privileges needed.", "least_impact": False}, + {"value": "Low", "abbreviation": "L", "definition": "Basic user privileges needed.", "least_impact": False}, + {"value": "High", "abbreviation": "H", "definition": "Administrative privileges needed.", "least_impact": True} + ] + }, + { + "metric_name": "User Interaction", + "metric_abbreviation": "UI", + "question": "Does the attack require user action?", + "values": [ + {"value": "None", "abbreviation": "N", "definition": "No user interaction needed.", "least_impact": False}, + {"value": "Required", "abbreviation": "R", "definition": "Requires user action.", "least_impact": True} + ] + }, + { + "metric_name": "Scope", + "metric_abbreviation": "S", + "question": "Does the attack affect other components or systems?", + "values": [ + {"value": "Unchanged", "abbreviation": "U", + "definition": "The impact is limited to the vulnerable system.", "least_impact": True}, + {"value": "Changed", "abbreviation": "C", + "definition": "The impact extends beyond the vulnerable system.", "least_impact": False} + ] + }, + { + "metric_name": "Confidentiality Impact", + "metric_abbreviation": "C", + "question": "Does the attack expose sensitive data?", + "values": [ + {"value": "None", "abbreviation": "N", "definition": "No impact on data confidentiality.", "least_impact": True}, + {"value": "Low", "abbreviation": "L", "definition": "Partial exposure of non-sensitive data.", "least_impact": False}, + {"value": "High", "abbreviation": "H", "definition": "Full exposure of sensitive data.", "least_impact": False} + ] + }, + { + "metric_name": "Integrity Impact", + "metric_abbreviation": "I", + "question": "Can the attacker modify critical data?", + "values": [ + {"value": "None", "abbreviation": "N", "definition": "No impact on data integrity.", "least_impact": True}, + {"value": "Low", "abbreviation": "L", "definition": "Limited unauthorized modifications.", "least_impact": False}, + {"value": "High", "abbreviation": "H", "definition": "Complete control over data integrity.", "least_impact": False} + ] + }, + { + "metric_name": "Availability Impact", + "metric_abbreviation": "A", + "question": "Does the attack affect system uptime?", + "values": [ + {"value": "None", "abbreviation": "N", "definition": "No impact on system availability.", "least_impact": True}, + {"value": "Low", "abbreviation": "L", "definition": "Minor service degradation.", "least_impact": False}, + {"value": "High", "abbreviation": "H", "definition": "Complete system failure.", "least_impact": False} + ] + } +] + +class CVEGenerateCvssToolConfig(FunctionBaseConfig, name="cve_generate_cvss"): + """ + Defines a function that generates and calculates the CVSS (Common Vulnerability Scoring System) + vector string and score for the vulnerability analysis. + + This class evaluates the severity of a given vulnerability by determining its impact and exploitability + based on key CVSS metrics - Attack Vector (AV), Attack Complexity (AC), Privileges Required (PR), + User Interaction (UI), Scope (S), Confidentiality (C), Integrity (I), and Availability (A). + """ + llm_name: str = Field(description="The LLM model to use with the CVE agent.") + tool_names: list[str] = Field(default=[], description="The list of tools to provide to CVE agent.") + max_concurrency: int | None = Field(ge=1, + default=None, + description="The maximum number of concurrent agent invocations.") + max_iterations: int = Field(default=10, description="The maximum number of iterations for the agent.") + prompt: str | None = Field( + default=None, + description= + "Manually set the prompt for the specific model in the configuration. The prompt can either be passed in as a " + "string of text or as a path to a text file containing the desired prompting.") + prompt_examples: bool = Field(default=True, description="Whether to include examples in agent prompt.") + replace_exceptions: bool = Field(default=False, + description="Whether to replace exception message with custom message.") + replace_exceptions_value: str | None = Field(default=None, description="Message if replace_exceptions is true") + return_intermediate_steps: bool = Field( + default=False, + description= + "Controls whether to return intermediate steps taken by the agent, and include them in the output file.") + verbose: bool = Field(default=False, description="Set to true for verbose output") + skip: bool = Field(default=False, + description="Whether or not the CVSS generator should be skipped.") + +def _make_parse_error_handler(is_openai: bool): + def handle_parse_error(exception: OutputParserException) -> str: + if is_openai: + logger.debug(f"Exception intercepted in notify {exception}") + exception_string = str(exception) + if OUTPUT_CONTAIN_BOTH_ACTION_AND_FINAL_ANSWER in exception_string: + return (f"Check your output and make sure it conforms! Do not output an action " + f"and a final answer at the same time") + if OUTPUT_CONTAIN_PARSING_ERROR in exception_string: + return ("Check your output and make sure it conforms. " + "Do not execute any tool. " + "Do not repeat Metric or Question. " + "Resume from your last step. " + "Begin your next token with 'Thought:'. Fix format only: " + "- NEVER output `Action: None`.\n." + "- Do NOT call `Container Image Analysis Data` again; reuse its previous Observation.\n" + "- Final Thought must end with: Selected: ().\n" + "- Final Answer must be: : (reuse the same abbreviation).") + return "Check your output and make sure it conforms" + return handle_parse_error + +async def _create_agent(config: CVEGenerateCvssToolConfig, builder: Builder, + state: AgentMorpheusEngineState) -> AgentExecutor: + tools = builder.get_tools(tool_names=config.tool_names, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + is_openai = "openai" in llm.__class__.__module__.lower() + + # Filter tools that are not available based on state + tools = [ + tool for tool in tools + if not ((tool.name == ToolNames.CODE_SEMANTIC_SEARCH and state.code_vdb_path is None) or + (tool.name == ToolNames.DOCS_SEMANTIC_SEARCH and state.doc_vdb_path is None) or + (tool.name == ToolNames.CODE_KEYWORD_SEARCH and state.code_index_path is None)) + ] + + # Get prompt (examples now embedded in template) + prompt = PromptTemplate.from_template( + get_cvss_prompt(config.prompt, config.prompt_examples) + ) + + error_handler = _make_parse_error_handler(is_openai) + + agent = create_react_agent(llm=llm, + tools=tools, + prompt=prompt, + output_parser=MRKLOutputParser()) + + agent_executor = AgentExecutor( + agent=agent, + tools=tools, + early_stopping_method="force", + handle_parsing_errors=error_handler, + max_iterations=config.max_iterations, + return_intermediate_steps=config.return_intermediate_steps, + verbose=config.verbose) + + # Disable streaming for accurate token counts + if isinstance(agent_executor.agent, RunnableAgent): + agent_executor.agent.stream_runnable = False + + return agent_executor + + +async def _run_for_vuln(agent: AgentExecutor, + state: AgentMorpheusEngineState, + vuln_id: str, + semaphore: asyncio.Semaphore | None) -> list: + state_copy = state.model_copy(deep=True) + state_copy.current_vuln_id = vuln_id + token = ctx_state.set(state_copy) + + # Uncomment to enable default to least impact value for non-vulnerable components + # is_vulnerable = state.justifications[vuln_id]['justification_label'] == 'vulnerable' + + try: + async def _invoke_metric(metric: dict): + # Uncomment to enable default to least impact value for non-vulnerable components + # if (not is_vulnerable) and (metric['metric_abbreviation'] != 'AV'): + # least_impact_value = next((v for v in metric['values'] if v.get('least_impact')), None) + # abbr = least_impact_value['abbreviation'] if least_impact_value else "" + # return {"output": f"{metric['metric_abbreviation']}:{abbr}"} + + if semaphore: + async with semaphore: + return await agent.ainvoke({"input": json.dumps(metric, ensure_ascii=False)}) + return await agent.ainvoke({"input": json.dumps(metric, ensure_ascii=False)}) + + return await asyncio.gather(*(_invoke_metric(metric) for metric in DEFAULT_CVSS_METRICS_LIST), + return_exceptions=True) + finally: + ctx_state.reset(token) + + +def _postprocess_results(results: list, + replace_exceptions: bool, + replace_exceptions_value: str | None) -> tuple[str, str]: + """ + Post-process per-metric results for a single CVE into a CVSS vector string and score. + Replace exceptions with placeholder values if configured. + """ + outputs: list[str] = [] + + pattern = r'\s*[A-Z\s]+\s*:\s*[A-Z\s]+\s*' + for answer in results: + # Handle exceptions returned by the agent + # OutputParserException is not a subclass of Exception, so we need to check for it separately + if isinstance(answer, (OutputParserException, Exception)): + if replace_exceptions: + return "", replace_exceptions_value + else: + return "", f"Failed to generate a CVSS score for this analysis: {str(answer)}" + + output = answer["output"] + if re.fullmatch(pattern, output): + output = output.replace(" ", "") + outputs.append(output) + + vector_string = f"CVSS:3.1/" + "/".join(outputs) + + try: + cvss_obj = CVSS3(vector_string) + score = str(cvss_obj.scores()[0]) # get base score + except Exception as e: + return vector_string, f'Failed to generate a CVSS score from vector string. Error: {e}' + + return vector_string, score + +@register_function(config_type=CVEGenerateCvssToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def cve_generate_cvss(config: CVEGenerateCvssToolConfig, builder: Builder): + + semaphore = asyncio.Semaphore(config.max_concurrency) if config.max_concurrency else None + + async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: + + if config.skip: + logger.info("`config.skip` is set to True. Skipping CVSS generation.") + return state + + agent = await _create_agent(config, builder, state) + + results = await asyncio.gather(*( + _run_for_vuln(agent, state, vuln_id, semaphore) + for vuln_id in state.checklist_results.keys() + ), return_exceptions=True) + + cvss_results = {} + + for vuln_id, result in zip(state.checklist_results.keys(), results): + vector_string, score_or_error = _postprocess_results(result, config.replace_exceptions, config.replace_exceptions_value) + + cvss_results[vuln_id] = { + CVSS_VECTOR_STRING: vector_string, + CVSS_SCORE: score_or_error + } + + state.cvss_results = cvss_results + + return state + + yield FunctionInfo.from_fn( + _arun, + input_schema=AgentMorpheusEngineState, + description=("Generates the CVSS (Common Vulnerability Scoring System) score and vector string for the vulnerability analysis results.")) diff --git a/src/vuln_analysis/functions/cve_generate_vdbs.py b/src/vuln_analysis/functions/cve_generate_vdbs.py index c3134a09d..354631eea 100644 --- a/src/vuln_analysis/functions/cve_generate_vdbs.py +++ b/src/vuln_analysis/functions/cve_generate_vdbs.py @@ -14,7 +14,6 @@ # limitations under the License. import json -import logging import os import time from pathlib import Path @@ -27,8 +26,23 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -logger = logging.getLogger(__name__) +from exploit_iq_commons.data_models.common import AnalysisType +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.utils.credential_client import credential_context +from exploit_iq_commons.utils.dep_tree import detect_ecosystem +from vuln_analysis.tools.tool_names import ToolNames +logger = LoggingFactory.get_agent_logger(__name__) + +class GitCloneError(Exception): + """Raised when cloning or updating a repo for document collection fails.""" + + +class EcosystemNotDetectedError(Exception): + """Raised when no ecosystem can be detected from cloned repo manifests.""" + + def __init__(self, message: str = "No ecosystem detected from repo manifests") -> None: + super().__init__(message) class CVEGenerateVDBsToolConfig(FunctionBaseConfig, name="cve_generate_vdbs"): """ @@ -45,6 +59,11 @@ class CVEGenerateVDBsToolConfig(FunctionBaseConfig, name="cve_generate_vdbs"): description="The directory used for storing vector database files.") base_code_index_dir: str = Field(default=".cache/am_cache/code_index", description="The directory used for storing code index files.") + base_pickle_dir: str = Field(default=".cache/am_cache/pickle", + description="The directory used for storing code index files.") + base_rpm_dir: str = Field(default=".cache/am_cache/rpms", + description="The directory used for storing rpm files.") + ignore_errors: bool = Field(default=False, description="Whether to ignore errors during generation.") ignore_code_embedding: bool = Field(default=False, description="Whether to ignore building code vector database.") ignore_code_index: bool = Field(default=False, description="Whether to ignore building code index.") @@ -52,36 +71,46 @@ class CVEGenerateVDBsToolConfig(FunctionBaseConfig, name="cve_generate_vdbs"): @register_function(config_type=CVEGenerateVDBsToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def generate_vdb(config: CVEGenerateVDBsToolConfig, builder: Builder): - - from vuln_analysis.data_models.info import AgentMorpheusInfo - from vuln_analysis.data_models.input import AgentMorpheusEngineInput - from vuln_analysis.data_models.input import AgentMorpheusInput - from vuln_analysis.data_models.input import SourceDocumentsInfo + from exploit_iq_commons.data_models.info import AgentMorpheusInfo + from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput + from exploit_iq_commons.data_models.input import AgentMorpheusInput + from exploit_iq_commons.data_models.input import SourceDocumentsInfo from vuln_analysis.functions.cve_agent import CVEAgentExecutorToolConfig - from vuln_analysis.utils.document_embedding import DocumentEmbedding + from exploit_iq_commons.utils.document_embedding import DocumentEmbedding from vuln_analysis.utils.full_text_search import FullTextSearch - from vuln_analysis.utils.git_utils import get_repo_from_path + from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager + from exploit_iq_commons.data_models.input import ManualSBOMInfoInput + from exploit_iq_commons.utils.standard_library_cache import StandardLibraryCache + agent_config = builder.get_function_config(config.agent_name) assert isinstance(agent_config, CVEAgentExecutorToolConfig) # Update config based on tools available in agent config - if "Container Image Code QA System" not in agent_config.tool_names: + if ToolNames.CODE_SEMANTIC_SEARCH not in agent_config.tool_names: logger.info("Container Image Code QA System tool is not enabled, setting ignore_code_embedding to True") config.ignore_code_embedding = True - if "Lexical Search Container Image Code QA System" not in agent_config.tool_names: + if ToolNames.CODE_KEYWORD_SEARCH not in agent_config.tool_names: logger.info( "Lexical Search Container Image Code QA System tool is not enabled, setting ignore_code_index to True") config.ignore_code_index = True embedding = await builder.get_embedder(embedder_name=config.embedder_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + + # Configure RPM singleton with cache directory from config + rpm_manager = RPMDependencyManager.get_instance() + rpm_manager.set_rpm_cache_dir(config.base_rpm_dir) + cache_std = StandardLibraryCache.get_instance() + cache_std.set_cache_directory(config.base_pickle_dir) embedder = DocumentEmbedding(embedding=embedding, vdb_directory=config.base_vdb_dir, - git_directory=config.base_git_dir) + git_directory=config.base_git_dir, + pickle_cache_directory=config.base_pickle_dir) - def _create_code_index(source_infos: list[SourceDocumentsInfo], embedder: DocumentEmbedding, output_path: Path): + def _create_code_index(source_infos: list[SourceDocumentsInfo], embedder: DocumentEmbedding, + output_path: Path) -> bool : logger.info("Collecting documents from git repos. Source Infos: %s", json.dumps([x.model_dump(mode="json") for x in source_infos])) @@ -97,11 +126,14 @@ def _create_code_index(source_infos: list[SourceDocumentsInfo], embedder: Docume documents.extend(embedder.collect_documents(si)) except Exception as e: logger.warning("Error collecting documents for source info %s: %s", si, e) - continue + logger.error("Failed to clone repository %s--%s", si.git_repo, e) + raise GitCloneError(f"Failed to clone repository {si.git_repo}") + # Warn and return early if no documents were collected if len(documents) == 0: logger.warning("No documents were collected, skipping code indexing for source infos: '%s'", source_infos) + return False # Apply chunking on the source documents chunked_documents = embedder._chunk_documents(documents) @@ -115,17 +147,21 @@ def _create_code_index(source_infos: list[SourceDocumentsInfo], embedder: Docume indexing_start_time = time.time() full_text_search.add_documents_from_langchain_chunks(chunked_documents) logger.info("Completed code indexing in %.2f seconds for '%s'", time.time() - indexing_start_time, output_path) + return True def _build_code_index(source_infos: list[SourceDocumentsInfo]) -> Path | None: code_index_path: Path | None = None # Filter to only code sources source_infos = [si for si in source_infos if si.type == 'code'] - + documents_exists: bool = True if source_infos: # Use DocumentEmbedding class with embedding=None for its utility functions - embedder = DocumentEmbedding(embedding=None) + embedder = DocumentEmbedding(embedding=None, + vdb_directory=config.base_vdb_dir, + git_directory=config.base_git_dir, + pickle_cache_directory=config.base_pickle_dir) # Determine code index path for either loading from cache or creating new index # Need to add support for configurable base path @@ -133,11 +169,13 @@ def _build_code_index(source_infos: list[SourceDocumentsInfo]) -> Path | None: base_path=config.base_code_index_dir, hash_value=embedder.hash_source_documents_info(source_infos)) if (not code_index_path.exists() or os.environ.get("MORPHEUS_ALWAYS_REBUILD_VDB", "0") == "1"): - _create_code_index(source_infos, embedder, code_index_path) + documents_exists = _create_code_index(source_infos, embedder, code_index_path) else: logger.info("Cache hit on code index. Loading existing code index: %s", code_index_path) - - return code_index_path + if documents_exists: + return code_index_path + else: + return None async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput: """ @@ -160,63 +198,90 @@ async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput: vdb_code_path = None vdb_doc_path = None code_index_path = None + ecosystem_detected = True base_image = message.image.name source_infos = message.image.source_info - + sbom_infos = message.image.sbom_info + failure_reason = "" try: - - # Build VDBs - vdb_code_path, vdb_doc_path = embedder.build_vdbs(source_infos, config.ignore_code_embedding) - - if (vdb_code_path is None): - # Only log warning if we're not ignoring code embeddings - if (not config.ignore_code_embedding): - logger.warning(("Failed to generate code VDB for image '%s'. " + trace_id.set(message.scan.id) + logger.debug("_arun: received credential_id=%r scan_id=%s", message.credential_id, message.scan.id) + # Build VDBs (credential_id is propagated via async context) + with credential_context(message.credential_id): + logger.debug("_arun: credential_context entered, credential_id=%r", message.credential_id) + vdb_code_path, vdb_doc_path = embedder.build_vdbs(source_infos, config.ignore_code_embedding) + + if (vdb_code_path is None): + # Only log warning if we're not ignoring code embeddings + if (not config.ignore_code_embedding): + logger.warning(("Failed to generate code VDB for image '%s'. " + "Ensure the source repositories are setup correctly"), + base_image) + + if (vdb_doc_path is None): + logger.warning(("Failed to generate documentation VDB for image '%s'. " "Ensure the source repositories are setup correctly"), base_image) - if (vdb_doc_path is None): - logger.warning(("Failed to generate documentation VDB for image '%s'. " - "Ensure the source repositories are setup correctly"), - base_image) - - # Build code index if not ignored - if (not config.ignore_code_index): - code_index_path = _build_code_index(source_infos) - - if code_index_path is None: - logger.warning(("Failed to generate code index for image '%s'. " - "Ensure the source repositories are setup correctly"), - base_image) - - # Replace ref with specific commit hash for each source info - for si in source_infos: - try: - repo = get_repo_from_path(config.base_git_dir, si.git_repo) - si.ref = repo.commit().hexsha - except ValueError as e: - logger.warning("Failed to get commit hash for %s/%s: %s", config.base_git_dir, si.git_repo, e) - continue + # Build code index if not ignored + if not config.ignore_code_index: + logger.info("analysis type: %s", message.image.analysis_type) + logger.debug("_arun: building code index, credential_id in ctx=%r", message.credential_id) + if message.image.analysis_type == AnalysisType.IMAGE and isinstance(sbom_infos, ManualSBOMInfoInput): + RPMDependencyManager.get_instance().sbom = sbom_infos.packages + image = f"{message.image.name}:{message.image.tag}" + RPMDependencyManager.get_instance().container_image = image + + code_index_path = _build_code_index(source_infos) + + if code_index_path is None: + logger.warning(("Failed to generate code index for image '%s'. " + "Ensure the source repositories are setup correctly"), + base_image) + + # Detect ecosystem from cloned repo manifests if not provided by the client + if message.image.ecosystem is None: + code_sources = [si for si in source_infos if si.type == 'code'] + if code_sources: + repo_path = embedder.get_repo_path(code_sources[0]) + detected = detect_ecosystem(repo_path) + if detected is not None: + message.image.ecosystem = detected + logger.info("Detected ecosystem '%s' from repo manifests", detected.value) + else: + ecosystem_detected = False + raise EcosystemNotDetectedError() + + except EcosystemNotDetectedError: + failure_reason = "No ecosystem detected from repo manifests" + logger.error( + "No ecosystem detected from repo manifests for image '%s', source code info: %s", + base_image, + source_infos, + ) + logger.warning("Ignoring VDB generation error and continuing with missing VDBs.") except Exception as e: + failure_reason = f"Failure to build VDB , Error: {e}" logger.error("Failure to build VDB for image, '%s', with source code info: %s\nError: %s", base_image, source_infos, e, exc_info=True) - if not config.ignore_errors: - raise - logger.warning("Ignoring VDB generation error and continuing with missing VDBs.") vdb_code_path = str(vdb_code_path) if vdb_code_path is not None else None vdb_doc_path = str(vdb_doc_path) if vdb_doc_path is not None else None code_index_path = str(code_index_path) if code_index_path is not None else None + code_index_success = True if code_index_path is not None and ecosystem_detected else False + if not code_index_success: + message.failure_reason = failure_reason info = AgentMorpheusInfo(vdb=AgentMorpheusInfo.VdbPaths( code_vdb_path=vdb_code_path, doc_vdb_path=vdb_doc_path, code_index_path=code_index_path)) + message.code_index_success = code_index_success return AgentMorpheusEngineInput(input=message, info=info) diff --git a/src/vuln_analysis/functions/cve_generate_vex.py b/src/vuln_analysis/functions/cve_generate_vex.py new file mode 100644 index 000000000..a7d2ee4b9 --- /dev/null +++ b/src/vuln_analysis/functions/cve_generate_vex.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid + +from aiq.builder.builder import Builder +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field +from vuln_analysis.data_models.state import AgentMorpheusEngineState +from vuln_analysis.utils.vex.vex_generator_loader import load_vex_generator + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) + +class CVEGenerateVexConfig(FunctionBaseConfig, name="cve_generate_vex"): + """ + Defines a function that generates a custom VEX (Vendor Exploitability eXchange) document for vulnerable components following the OpenVEX specification. + """ + + skip: bool = Field(default=False, description="Whether or not the VEX generator should be skipped.") + vex_format: str = Field(default="csaf", description="VEX document format to generate.") + +@register_function(config_type=CVEGenerateVexConfig) +async def cve_generate_vex(config: CVEGenerateVexConfig, builder: Builder): + + async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: + if config.skip: + logger.info("`config.skip` is set to True. Skipping VEX generation.") + return state + + if not any(justification.get("justification_label") == "vulnerable" for justification in state.justifications.values()): + logger.info("No vulnerable CVE(s) found. Generating VEX with known_not_affected status.") + + try: + generator = load_vex_generator(config.vex_format) + vex_doc = generator.generate(state) + if vex_doc: + tracking = vex_doc.get("document", {}).get("tracking") + if tracking and tracking.get("id"): + tracking["id"] = f"{tracking['id']}-{uuid.uuid4()}" + state.vex = vex_doc + except ValueError as e: + logger.error("VEX generator initialization failed: %s", e) + except Exception as e: + logger.error("VEX document generation failed: %s", e) + + return state + + yield FunctionInfo.from_fn(_arun, + input_schema=AgentMorpheusEngineState, + description="Generates a custom VEX document for vulnerable components.") diff --git a/src/vuln_analysis/functions/cve_http_output.py b/src/vuln_analysis/functions/cve_http_output.py index f80ac7729..792b0f107 100644 --- a/src/vuln_analysis/functions/cve_http_output.py +++ b/src/vuln_analysis/functions/cve_http_output.py @@ -12,17 +12,75 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -import logging - +import base64 +from dataclasses import dataclass +from http import HTTPStatus +from datetime import datetime from aiq.builder.builder import Builder from aiq.builder.function_info import FunctionInfo from aiq.cli.register_workflow import register_function from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -logger = logging.getLogger(__name__) +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.data_models.checker_status import ( + CHECKER_FAILURE_ERROR_TYPES, + PACKAGE_CHECKER_STATUS_DESCRIPTIONS, +) +from exploit_iq_commons.data_models.common import PipelineMode, TypedBaseModel +from exploit_iq_commons.data_models.input import SourceDocumentsInfo +from vuln_analysis.data_models.job import Job, LocalDateTime +import typing +from typing import Any, TYPE_CHECKING +import os +import re + +if TYPE_CHECKING: + from vuln_analysis.data_models.output import AgentMorpheusOutput, FailureReport + +logger = LoggingFactory.get_agent_logger(__name__) + + + +# Explicit env var names to never export +ENV_BLACKLIST = frozenset({ + "GHSA_API_KEY", "NVD_API_KEY", "NVIDIA_API_KEY", "SERPAPI_API_KEY" +}) + +# Regex (case-insensitive) for key names to exclude +ENV_KEY_PATTERN = re.compile( + r"password|token|api\.key|api_key|api-key|secret|credential|" + r"private-key|private_key|auth|pwd|pass_|_pass|connection_string|" + r"connection-string|pat|certificate", + re.IGNORECASE +) + +def get_filtered_env_vars() -> str: + """Filter os.environ and return comma-delimited key=value pairs (safe for Job.env_vars).""" + pairs = [] + for key, value in os.environ.items(): + if key in ENV_BLACKLIST: + continue + if ENV_KEY_PATTERN.search(key): + continue + pairs.append(f"{key}={value}") + return ",".join(pairs) + +def encode_in_base64(v: str) -> str: + if not v: + return "" # Return safe empty string + encoded_bytes = base64.b64encode(v.encode("utf-8")) + return encoded_bytes.decode("ascii") + + + +class MLOpsConfig(TypedBaseModel[typing.Literal["mlops"]]): + mlops_url: str = Field(description="URL to send CVE workflow output") + auth_type: str = Field(default="None", description="Type of auth - bearer, basic or None") + token_path: str | None = Field(default=None, description="Path to token file containing auth token") + verify_path: str | None = Field(default=None, description="Path to certificate to validate the token key found in ") + enable_verify: bool = Field(default=False, description="Enable verify") class CVEHttpOutputConfig(FunctionBaseConfig, name="cve_http_output"): """ @@ -30,6 +88,72 @@ class CVEHttpOutputConfig(FunctionBaseConfig, name="cve_http_output"): """ url: str = Field(description="URL to send CVE workflow output") endpoint: str = Field(description="Endpoint to send CVE workflow output") + auth_type: str = Field(default="disabled", description="Type of auth - bearer, basic or disabled") + token: str | None = Field(default=None, description="Token to authenticate when sending CVE workflow output") + token_path: str | None = Field(default=None, description="Path to token file containing auth token") + verify_path: str | None = Field(default=None, description="Path to certificate to validate the token key found in ") + username: str | None = Field(default=None, description="Username to authenticate with for basic auth") + password: str | None = Field(default=None, description="Password to authenticate with for basic auth") + failure_endpoint: str = Field(default="/api/v1/reports/failed", + description="Endpoint to send failure reports when code indexing fails") + enable_mlops: bool = Field(default=False, description="Enable MLOps") + mlops_config: MLOpsConfig = Field(..., description="MLOps configuration") + + +@dataclass +class OutputPayload: + """Encapsulates the HTTP output payload details.""" + json: str + url: str + skip_mlops: bool + + +def _build_output_payload( + message: "AgentMorpheusOutput", + config: CVEHttpOutputConfig, + default_json: str, +) -> OutputPayload: + """ + Determine the payload to send - either the full output or a failure report. + + Returns an OutputPayload with the appropriate JSON, URL, and skip_mlops flag. + """ + from vuln_analysis.data_models.output import FailureReport + + default_url = config.url + config.endpoint + failure_url = config.url + config.failure_endpoint + + if message.input.code_index_success is False: + repo_url = message.input.image.source_info[0].git_repo if message.input.image.source_info else "unknown" + report = FailureReport( + scan_id=message.input.scan.id, + error_type="processing-error", + error_message=message.input.failure_reason if message.input.failure_reason else "Unknown failure reason", + ) + logger.info(f"Code index failed for scan {message.input.scan.id}, sending failure report to {failure_url}") + return OutputPayload(json=report.model_dump_json(by_alias=True), url=failure_url, skip_mlops=True) + + checker_ctx = message.info.checker_context + if checker_ctx and checker_ctx.status in CHECKER_FAILURE_ERROR_TYPES: + error_type = CHECKER_FAILURE_ERROR_TYPES[checker_ctx.status] + error_msg = PACKAGE_CHECKER_STATUS_DESCRIPTIONS.get( + checker_ctx.status, + f"Checker failed with status {checker_ctx.status}" + ) + cve_id = message.input.scan.vulns[0].vuln_id if message.input.scan.vulns else "unknown" + pkg_name = message.input.image.target_package.name if message.input.image.target_package else "unknown" + report = FailureReport( + scan_id=message.input.scan.id, + error_type=error_type, + error_message=f"{error_msg} (CVE: {cve_id}, package: {pkg_name})", + ) + logger.info( + f"Checker early exit for scan {message.input.scan.id} with status {checker_ctx.status}, " + f"sending failure report to {failure_url}" + ) + return OutputPayload(json=report.model_dump_json(by_alias=True), url=failure_url, skip_mlops=True) + + return OutputPayload(json=default_json, url=default_url, skip_mlops=False) @register_function(config_type=CVEHttpOutputConfig) @@ -39,21 +163,145 @@ async def output_to_http(config: CVEHttpOutputConfig, builder: Builder): # pyli from vuln_analysis.utils import http_utils async def _arun(message: AgentMorpheusOutput) -> AgentMorpheusOutput: + trace_id.set(message.input.scan.id) + + model_json = message.model_dump_json(by_alias=True) - url = config.url + config.endpoint - headers = {'Content-type': 'application/json'} + + # Save JSON for debugging - compare with local markdown reports + #from pathlib import Path + #debug_output_dir = Path(".cache/am_cache/checker_json_output") + #debug_output_dir.mkdir(parents=True, exist_ok=True) + #vuln_id = message.input.scan.vulns[0].vuln_id if message.input.scan.vulns else "unknown" + #json_file = debug_output_dir / f"{message.input.scan.id}_{vuln_id}.json" + #json_file.write_text(model_json) + #logger.info(f"Saved JSON output to {json_file}") + + headers = {'Content-type': 'application/json', 'traceId': trace_id.get()} + auth_header = get_auth_header(config) + if auth_header is not None: + headers['Authorization'] = auth_header + verify = config.verify_path if config.verify_path else True + + payload = _build_output_payload(message, config, model_json) try: - http_utils.request_with_retry(request_kwargs={ - "url": url, "method": "POST", "data": model_json.encode('utf-8'), "headers": headers - }) + logger.info(f"Sending output to {payload.url}") + http_utils.request_with_retry( + request_kwargs={ + "url": payload.url, + "method": "POST", + "data": payload.json.encode('utf-8'), + "headers": headers, + "verify": verify, + }, + accept_status_codes=(HTTPStatus.OK, HTTPStatus.CREATED, HTTPStatus.ACCEPTED), + ) + if config.enable_mlops and not payload.skip_mlops: + mlops_url = None + try: + job = _extract_job_data(message) + job_json = job.model_dump_json(by_alias=True) + http_params = _http_params_override(config, config.mlops_config, headers) + is_verify = http_params["verify"] + if is_verify: + verify = http_params["verify_path"] + mlops_url = http_params["url"] + http_utils.request_with_retry(request_kwargs={ + "url": mlops_url, "method": "POST", "data": job_json.encode('utf-8'), "headers": http_params["headers"], "verify": verify + }, accept_status_codes=(HTTPStatus.OK, HTTPStatus.CREATED, HTTPStatus.ACCEPTED)) + except Exception as mlops_e: + logger.error('Unable to send job to MLOps API at %s. Error: %s', mlops_url, mlops_e) except Exception as e: - logger.error('Unable to send output response to %s. Error: %s', url, e) + logger.error('Unable to send output response to %s. Error: %s', payload.url, e) else: - logger.info('Successfully sent output to %s', url) + logger.info('Successfully sent output to %s', payload.url) return message + + def _extract_job_data(message: AgentMorpheusOutput) -> Job: + agent_config = builder.get_workflow_config() + job_id = message.input.scan.id + start_time = datetime.fromisoformat(message.input.scan.started_at.replace('Z', '+00:00')) + end_time = datetime.fromisoformat(message.input.scan.completed_at.replace('Z', '+00:00')) + duration = (end_time - start_time).total_seconds() + executionTimestamp = LocalDateTime(root=start_time) + duration_in_seconds = str(duration) + cve = message.input.scan.vulns[0].vuln_id + app_language = message.input.image.ecosystem + component = message.input.image.name + component_version = message.input.image.tag + env_vars = get_filtered_env_vars() + intel = message.info.intel[0].model_dump_json(by_alias=True) + intel_b64 = encode_in_base64(intel) + agent_config_b64 = encode_in_base64(agent_config.model_dump_json()) + message_output = message.output.model_dump() + agent_git_commit=os.environ.get("AGENT_GIT_COMMIT") + agent_git_tag=os.environ.get("AGENT_GIT_TAG") + return Job(job_id=job_id, execution_start_timestamp=executionTimestamp, duration_in_seconds=duration_in_seconds, + cve=cve, app_language=app_language, component=component, component_version=component_version, + agent_git_commit=agent_git_commit, agent_git_tag=agent_git_tag, + agent_config_b64=agent_config_b64, concrete_intel_sources_b64=intel_b64, + executed_from_background_process=False, batch_id="", success_status=True, + env_vars=env_vars, job_output=message_output) yield FunctionInfo.from_fn(_arun, input_schema=AgentMorpheusOutput, description=("Sends CVE workflow output to HTTP endpoint.")) + + + + +def get_auth_header(http_config: CVEHttpOutputConfig | None) -> str | None: + match http_config.auth_type: + case "basic": + if http_config.username and http_config.password: + credentials = f"{http_config.username}:{http_config.password}" + encoded_creds = base64.b64encode(credentials.encode('utf-8')).decode('utf-8') + return f"Basic {encoded_creds}" + return None + case "bearer": + if http_config.token: + return f"Bearer {http_config.token}" + elif http_config.token_path: + try: + with open(http_config.token_path, 'r') as file: + return f"Bearer {file.read().strip()}" + except Exception as e: + logger.warn(f"Unable to read OAuth token: {e}") + return None + case None: + return None + + +def _http_params_override(http_config: CVEHttpOutputConfig, mlops_config: MLOpsConfig,http_headers) -> dict[str, Any]: + + url = http_config.url + "/api/v1/jobs/one" + if mlops_config.auth_type == "None": + headers = {'Content-type': 'application/json'} + return { + "headers": headers, + "verify": False, + "url": url + } + if http_config.auth_type.lower() == "bearer" and mlops_config.auth_type.lower() == "bearer": + return { + "headers": http_headers, + "url": url, + "verify": mlops_config.enable_verify, + "verify_path": http_config.verify_path + } + clone_http = http_config.copy() + clone_http.auth_type = "bearer" + clone_http.token_path = mlops_config.token_path + auth_header = get_auth_header(clone_http) + headers = {'Content-type': 'application/json'} + if auth_header is not None: + headers['Authorization'] = auth_header + return { + "url": url, + "headers": headers, + "verify": mlops_config.enable_verify, + "verify_path": mlops_config.verify_path + } + diff --git a/src/vuln_analysis/functions/cve_justify.py b/src/vuln_analysis/functions/cve_justify.py index 50a49abaf..8124b4222 100644 --- a/src/vuln_analysis/functions/cve_justify.py +++ b/src/vuln_analysis/functions/cve_justify.py @@ -14,7 +14,6 @@ # limitations under the License. import asyncio -import logging from aiq.builder.builder import Builder from aiq.builder.framework_enum import LLMFrameworkEnum @@ -23,7 +22,9 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -logger = logging.getLogger(__name__) +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id + +logger = LoggingFactory.get_agent_logger(__name__) class CVEJustifyToolConfig(FunctionBaseConfig, name="cve_justify"): @@ -53,8 +54,8 @@ async def justify_cve(summary): return justification_text.content async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: + trace_id.set(state.original_input.input.scan.id) results = await asyncio.gather(*(justify_cve(summary) for summary in state.final_summaries.values())) - parsed_justification = await asyncio.gather(jp._parse_justification(results)) # format justification output diff --git a/src/vuln_analysis/functions/cve_package_code_agent.py b/src/vuln_analysis/functions/cve_package_code_agent.py new file mode 100644 index 000000000..035396c69 --- /dev/null +++ b/src/vuln_analysis/functions/cve_package_code_agent.py @@ -0,0 +1,1656 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from pathlib import Path +from typing import Literal + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.data_models.checker_status import L1InvestigationResult, EnumIdentifyResult + +from langgraph.graph import StateGraph, START, END +from langgraph.prebuilt import ToolNode +from langchain_core.messages import HumanMessage, AIMessage, SystemMessage, RemoveMessage + +from nat.builder.context import Context +from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput +from vuln_analysis.functions.code_agent_graph_defs import ( + CodeAgentState, + DownstreamSearchReport, + UpstreamSearchReport, + ParsedPatch, + downstream_search_preprocss, + upstream_search_preprocess, + extract_l1_verdict, + VulnerabilityIntel, + format_patch_data_for_intel, + get_relevant_hunks, + ReferenceHints, + AdvisoryContent, + GitSearchReport, + CONFIDENCE_THRESHOLD_MIN, +) +from vuln_analysis.utils.rpm_checker_prompts import ( + L1_AGENT_SYS_PROMPT_PATCH_AVAILABLE, + L1_AGENT_SYS_PROMPT_REBASE_FIX, + L1_AGENT_SYS_PROMPT_REBASE_NO_PATCH, + L1_AGENT_PROMPT_TEMPLATE, + L1_AGENT_PROMPT_TEMPLATE_NO_PATCH, + L1_AGENT_THOUGHT_INSTRUCTIONS, + L1_AGENT_THOUGHT_REBASE_INSTRUCTIONS, + L1_AGENT_THOUGHT_CVE_DESC_INSTRUCTIONS, + select_upstream_prompt_and_instructions, + L1_COMPREHENSION_PROMPT, + L1_MEMORY_UPDATE_PROMPT, + L1_EMPTY_RESULT_CLASSIFICATION_PROMPT, + VULNERABILITY_INTEL_EXTRACTION_PROMPT, +) +from vuln_analysis.tools.brew_downloader import BrewDownloader, BrewDownloaderError, resolve_brew_profile +from vuln_analysis.utils.package_identifier import _extract_dist_tag, extract_nvd_version_info + +from vuln_analysis.functions.react_internals import CheckerThought, CodeFindings, Observation, FORCED_FINISH_PROMPT, check_empty_output, invoke_comprehension +from vuln_analysis.utils.intel_utils import extract_commit_url_candidates, extract_advisory_urls +from vuln_analysis.utils.vulnerability_intel_sanitizer import VulnerabilityIntelSanitizer +from vuln_analysis.utils.reference_fetcher import ReferenceFetcher +from vuln_analysis.utils.reference_parser import ReflectiveReferenceParser, ParserConfig +from vuln_analysis.utils.token_utils import truncate_tool_output, truncate_tool_output_list +from vuln_analysis.runtime_context import ctx_state + +logger = LoggingFactory.get_agent_logger(__name__) + +import uuid +import tiktoken +import aiohttp + +# NEVRA = Name-Epoch-Version-Release-Architecture, the standard RPM package naming format. +_RPM_NEVRA_RE = re.compile(r"^(.+?)-(?:(\d+):)?(\d\S*?)-(\S+)$") + +# Architecture mappings for early CVE applicability checks +ARCH_TO_BITNESS = { + "x86_64": "64-bit", + "amd64": "64-bit", + "i686": "32-bit", + "i386": "32-bit", + "aarch64": "64-bit", + "arm64": "64-bit", + "ppc64le": "64-bit", + "s390x": "64-bit", +} +ARCH_TO_FAMILY = { + "x86_64": "x86", + "amd64": "x86", + "i686": "x86", + "i386": "x86", + "aarch64": "arm", + "arm64": "arm", + "ppc64le": "ppc", + "s390x": "s390", +} + + +def _parse_fix_info_from_context(ctx, target_name: str, target_release: str | None = None) -> dict: + """Extract {name, version, release} from checker_context.identify_result.fixed_rpm_list. + + Handles both epoch and non-epoch NEVRAs: + - With epoch: libpq-0:13.20-1.el8_6.x86_64 + - Without epoch: libpq-13.20-1.el8_6.x86_64 + + Finds the NEVRA that matches the target package name AND EL dist tag. + Falls back to the first name match if no EL-matching entry is found. + Returns an empty dict if no match is found. + """ + if not ctx or not ctx.identify_result or not ctx.identify_result.fixed_rpm_list: + return {} + + target_dist = _extract_dist_tag(target_release) if target_release else None + fallback: dict = {} + + for nevra in ctx.identify_result.fixed_rpm_list: + m = _RPM_NEVRA_RE.match(nevra) + if not m: + continue + name = m.group(1) + if name.lower() != target_name.lower(): + continue + + version = m.group(3) + release_arch = m.group(4) + release = release_arch.rsplit(".", 1)[0] if "." in release_arch else release_arch + clean_nevra = f"{name}-{version}-{release_arch}" + result = {"nevra": clean_nevra, "name": name, "version": version, "release": release} + + # Store first name match as fallback + if not fallback: + fallback = result + + # Check if EL tag matches - if so, return immediately + candidate_dist = _extract_dist_tag(release_arch) + if target_dist and candidate_dist and target_dist == candidate_dist: + return result + + # No EL-matching entry found, return fallback (first name match) + return fallback + + +def _build_tool_strategy(tool_names: list[str]) -> str: + """Generate tool usage guidance based on available tools.""" + strategies = [] + tool_names_lower = [t.lower().replace("_", " ") for t in tool_names] + + if any("grep" in t for t in tool_names_lower): + strategies.append("- Use Source Grep for exact code patterns from patch (function names, variable names, specific code)") + if any("keyword" in t or "search" in t for t in tool_names_lower): + strategies.append("- Use Code Keyword Search for broader concept searches when grep fails") + if any("read" in t for t in tool_names_lower): + strategies.append("- Use Read File to examine full context around matches") + + return "\n".join(strategies) if strategies else "Use available tools to search for vulnerable and fixed code patterns." + + +_DISCOVERED_FILE_PATTERN = re.compile(r' in ([./\w-]+\.(?:c|h|cpp|hpp|cc|cxx|hxx))(?::\d+)?') + + +def _extract_discovered_files_from_memory(memory: list[str]) -> set[str]: + """Extract file basenames from L1 observation memory entries. + + Parses entries like: + - "VULNERABLE_CODE_FOUND: mod_proxy_ftp in ./httpd-2.4/.../mod_proxy_ftp.c" + - "CVE_CODE_FOUND: function in /path/to/file.h:123" + + Returns set of unique file basenames for build log matching (e.g., 'mod_proxy_ftp.c'). + """ + file_basenames = set() + + for entry in memory: + match = _DISCOVERED_FILE_PATTERN.search(entry) + if match: + full_path = match.group(1) + basename = Path(full_path).name + file_basenames.add(basename) + + return file_basenames + + +# --------------------------------------------------------------------------- +# Policy context formatting for L1 reports (Feedback-2 gap coverage) +# --------------------------------------------------------------------------- + +_POLICY_MAX_RPM_LIST_ITEMS = 5 +_POLICY_RHSA_STATEMENT_CAP = 400 +_POLICY_MAX_PACKAGE_STATE_ITEMS = 8 + + +def _format_policy_context_for_l1_report( + *, + target_nvr: str, + identify_result, + intel, +) -> str: + """Build a context block for the LLM prompt covering NVR posture and RHSA excerpts. + + Returns an empty string if no meaningful context is available. + """ + lines: list[str] = [] + + # 1. Scanned target NVR + if target_nvr: + lines.append(f"**Scanned target NVR:** `{target_nvr}`") + + # 2. PackageIdentifyResult: affected/fixed lists + if identify_result: + affected = identify_result.affected_rpm_list or [] + fixed = identify_result.fixed_rpm_list or [] + + if affected: + shown = affected[:_POLICY_MAX_RPM_LIST_ITEMS] + suffix = f" (+ {len(affected) - len(shown)} more)" if len(affected) > len(shown) else "" + lines.append(f"**Affected NVRs from identify:** {', '.join(f'`{n}`' for n in shown)}{suffix}") + lines.append(f" - is_target_package_affected: `{identify_result.is_target_package_affected.value}`") + if identify_result.conclusion_reason: + lines.append(f" - conclusion_reason: `{identify_result.conclusion_reason}`") + + if fixed: + shown = fixed[:_POLICY_MAX_RPM_LIST_ITEMS] + suffix = f" (+ {len(fixed) - len(shown)} more)" if len(fixed) > len(shown) else "" + lines.append(f"**Fixed NVRs from identify:** {', '.join(f'`{n}`' for n in shown)}{suffix}") + lines.append(f" - is_target_package_fixed: `{identify_result.is_target_package_fixed.value}`") + + # 3. RHSA excerpts (if present) + rhsa = None + if intel and len(intel) > 0: + rhsa = intel[0].rhsa + + if rhsa: + # Statement excerpt + if rhsa.statement: + stmt = rhsa.statement + if len(stmt) > _POLICY_RHSA_STATEMENT_CAP: + stmt = stmt[:_POLICY_RHSA_STATEMENT_CAP] + " …" + lines.append(f"**RHSA statement excerpt:** {stmt}") + + # Upstream fix + if rhsa.upstream_fix: + lines.append(f"**RHSA upstream_fix:** `{rhsa.upstream_fix}`") + + # Package state (compact table-like bullets) + pkg_states = rhsa.package_state or [] + if pkg_states: + lines.append("**RHSA package_state:**") + for ps in pkg_states[:_POLICY_MAX_PACKAGE_STATE_ITEMS]: + parts = [] + if ps.product_name: + parts.append(ps.product_name) + if ps.package_name: + parts.append(f"pkg={ps.package_name}") + if ps.fix_state: + parts.append(f"fix_state={ps.fix_state}") + if parts: + lines.append(f" - {' | '.join(parts)}") + if len(pkg_states) > _POLICY_MAX_PACKAGE_STATE_ITEMS: + lines.append(f" - (+ {len(pkg_states) - _POLICY_MAX_PACKAGE_STATE_ITEMS} more)") + + return "\n".join(lines) + + +def _merge_hints(existing: ReferenceHints, new: ReferenceHints) -> ReferenceHints: + """Merge two ReferenceHints objects, keeping first revision and deduplicating lists.""" + return ReferenceHints( + revision_hint=existing.revision_hint if existing.revision_hint else new.revision_hint, + function_hints=list(set(existing.function_hints + new.function_hints)), + file_hints=list(set(existing.file_hints + new.file_hints)), + branch_hints=list(set(existing.branch_hints + new.branch_hints)), + version_hints=list(set(existing.version_hints + new.version_hints)), + sources_processed=existing.sources_processed + new.sources_processed, + hint_strength=new.hint_strength if new.confidence > existing.confidence else existing.hint_strength, + confidence=max(existing.confidence, new.confidence), + ) + + +def merge_reference_hints_to_intel( + intel: VulnerabilityIntel, + hints: ReferenceHints, +) -> VulnerabilityIntel: + """Merge extracted reference hints into VulnerabilityIntel. + + Reference mining hints are lower confidence than patch-derived intel, + so they are appended (not prepended) to existing lists. + + Args: + intel: Existing VulnerabilityIntel (may have data from patch analysis) + hints: ReferenceHints from reference mining + + Returns: + Updated VulnerabilityIntel with merged hints + """ + existing_funcs = set(intel.vulnerable_functions) + new_funcs = [f for f in hints.function_hints if f not in existing_funcs] + intel.vulnerable_functions = intel.vulnerable_functions + new_funcs + + existing_files = set(intel.affected_files) + new_files = [f for f in hints.file_hints if f not in existing_files] + intel.affected_files = intel.affected_files + new_files + + if hints.revision_hint and hints.revision_hint not in intel.search_keywords: + intel.search_keywords = intel.search_keywords + [hints.revision_hint] + + if not intel.fixed_version and hints.version_hints: + intel.fixed_version = hints.version_hints[0] + + return intel + + +class CVEPackageCodeAgentConfig(FunctionBaseConfig, name="cve_package_code_agent"): + """ + Level 1 Package Code Agent. Investigates each CVE using extracted source + code and the scoped Tantivy code index built by checker_segmentation. + + Phases: Identify -> Locate -> Verify (see HLD-standalone-checker.md §5). + """ + base_checker_dir: str = Field( + default=".cache/am_cache/checker", + description="Root directory for checker-specific artifacts.", + ) + base_code_index_dir: str = Field( + default=".cache/am_cache/code_index", + description="Base directory for Tantivy code index storage.", + ) + base_rpm_dir: str = Field( + default=".cache/am_cache/rpms", + description="Shared RPM cache directory (for BrewDownloader).", + ) + rpm_user_type: str = Field( + default="internal", + description=( + "Brew profile for reference-patch SRPM lookup: internal or external. " + "Overridden by RPM_USER_TYPE environment variable when set." + ), + ) + llm_name: str = Field(description="The LLM model to use with the L1 code agent.") + tool_names: list[str] = Field(default=[], description="The list of tools to provide to L1 code agent") + max_iterations: int = Field(default=10, description="The maximum number of iterations for the agent.") + context_window_token_limit: int = Field(default=5000, description="Token limit for context window before pruning old messages.") + + # Reference Mining configuration + reference_mining_enabled: bool = Field( + default=True, + description="Enable reference mining when no upstream patch is found", + ) + reference_mining_max_urls: int = Field( + default=3, + description="Maximum number of advisory URLs to process", + ) + reference_mining_fetch_timeout: int = Field( + default=30, + description="HTTP timeout per URL in seconds", + ) + reference_mining_chunk_overlap: int = Field( + default=200, + description="Token overlap between chunks to preserve context", + ) + reference_mining_prompt_overhead: int = Field( + default=2500, + description="Tokens reserved for system prompt + CVE context + output", + ) + reference_mining_max_content_bytes: int = Field( + default=500_000, + description="Maximum advisory content to fetch per URL (bytes)", + ) + + # Repository Resolution configuration + repo_resolution_validate_urls: bool = Field( + default=True, + description="Validate repo URLs are accessible before returning", + ) + + # Git Search configuration + git_search_enabled: bool = Field( + default=True, + description="Enable git commit search node", + ) + git_search_cache_dir: str = Field( + default=".cache/am_cache/checker/upstream_repos", + description="Directory to cache cloned repositories", + ) + git_search_clone_depth: int = Field( + default=1000, + description="Default depth for shallow clones", + ) + git_search_fetch_timeout_seconds: int = Field( + default=120, + description="Timeout for git clone/fetch operations", + ) + git_search_min_confidence: float = Field( + default=0.50, + description="Minimum confidence to use git search result", + ) + git_search_max_diff_size: int = Field( + default=50000, + description="Maximum diff size in characters", + ) + + +async def create_graph_code_agent(config: CVEPackageCodeAgentConfig, builder: Builder, state: AgentMorpheusEngineInput, tracer): + # Node name constants + THOUGHT_NODE = "think_node" + TOOL_NODE = "tool" + FORCED_FINISH_NODE = "forced_finish" + OBSERVATION_NODE = "observation_node" + DOWNSTREAM_SEARCH_NODE = "downstream_search" + GATHER_MORE_INFO_NODE = "gather_more_info" + REFERENCE_MINING_NODE = "reference_mining" + REPO_RESOLUTION_NODE = "repo_resolution_node" + GIT_SEARCH_NODE = "git_search_node" + L1_AGENT_NODE = "L1_agent" + + llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + tools = builder.get_tools(tool_names=config.tool_names, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + + thought_llm = llm.with_structured_output(CheckerThought) + structured_comprehension_llm = llm.with_structured_output(CodeFindings) + observation_llm = llm.with_structured_output(Observation) + vulnerability_intel_llm = llm.with_structured_output(VulnerabilityIntel) + # Get tool names after filtering for dynamic guidance + enabled_tool_names = [tool.name for tool in tools] + tool_descriptions_list = [t.name + ": " + t.description for t in tools] + tools_node = ToolNode(tools, handle_tool_errors=True) + tool_strategy = _build_tool_strategy(enabled_tool_names) + tools_str = "\n".join(tool_descriptions_list) + + vuln_id = state.input.scan.vulns[0].vuln_id + ctx = state.info.checker_context + intel = state.info.intel + target_package = state.input.image.target_package + source_key = ctx.source_key + + _tiktoken_enc = tiktoken.get_encoding("cl100k_base") + + def _count_tokens(text: str) -> int: + """Count tokens using tiktoken cl100k_base encoding (~90-95% accurate for Llama 3.1).""" + try: + return len(_tiktoken_enc.encode(text)) + except Exception: + return len(text) // 4 + + def _estimate_tokens(runtime_prompt: str, messages: list, observation: Observation | None) -> int: + """Estimate the token count thought_node will send to the LLM.""" + parts = [runtime_prompt] + for msg in messages: + if hasattr(msg, "content") and isinstance(msg.content, str): + parts.append(msg.content) + if observation is not None: + for item in (observation.memory or []): + parts.append(item) + for item in (observation.results or []): + parts.append(item) + return _count_tokens("\n".join(parts)) + # -- Locate setup: fix info + BrewDownloader + paths ----------------------- + aIntel = intel[0] + fix_info = _parse_fix_info_from_context(ctx, target_package.name, target_package.release) + checker_dir = Path(config.base_checker_dir) / source_key + source_dir = checker_dir / "source" + patch_dir = checker_dir / "patch" + + brew_downloader = None + if fix_info: + try: + brew_downloader = BrewDownloader( + resolve_brew_profile(config.rpm_user_type), + config.base_rpm_dir, + str(checker_dir), + ) + brew_downloader.connect() + except BrewDownloaderError as e: + logger.warning("locate: BrewDownloader init failed (%s), diff path unavailable", e) + brew_downloader = None + + descriptions: list[tuple[str, str]] = [] + + if aIntel.osidb: + if aIntel.osidb.statement: + descriptions.append(("osidb_statement", aIntel.osidb.statement)) + if aIntel.osidb.cve_description: + descriptions.append(("osidb", aIntel.osidb.cve_description)) + if aIntel.osidb.comment_zero and not aIntel.osidb.cve_description: + descriptions.append(("osidb_vendor", aIntel.osidb.comment_zero)) + else: + if aIntel.ghsa: + cve_text = aIntel.ghsa.description or aIntel.ghsa.summary or "" + if cve_text: + descriptions.append(("ghsa", cve_text)) + if aIntel.ubuntu and aIntel.ubuntu.description: + descriptions.append(("ubuntu", aIntel.ubuntu.description)) + + cve_description = "\n".join(f"[{src}] {txt}" for src, txt in descriptions) + + + async def L1_agent(state: CodeAgentState) -> dict: + logger.info("L1_agent: starting") + downstream_report = state.get("downstream_report") + upstream_report = state.get("upstream_report") + git_search_report = state.get("git_search_report") + # Extract potential commit URLs from intel references for analysis + commit_url_candidates = extract_commit_url_candidates(aIntel) + with tracer.push_active_function("Initial_Intelligence_Gathering", input_data={"commit_url_candidates": commit_url_candidates}) as span: + + if downstream_report and downstream_report.is_patch_file_available: + parsed_patch = downstream_report.parsed_patch + patch_data = format_patch_data_for_intel(parsed_patch) + elif upstream_report and upstream_report.is_fixed_srpm_is_needed: + parsed_patch = upstream_report.fixed_parsed_patch + patch_data = format_patch_data_for_intel(parsed_patch) + elif git_search_report and git_search_report.parsed_patch: + parsed_patch = git_search_report.parsed_patch + patch_data = format_patch_data_for_intel(parsed_patch) + logger.info( + "L1_agent: Using discovered patch from git search (commit=%s, confidence=%.2f)", + git_search_report.best_result.commit_hash_short if git_search_report.best_result else "unknown", + git_search_report.best_result.confidence if git_search_report.best_result else 0, + ) + else: + parsed_patch = None + patch_data = "" + + # Extract vendor mitigations from intel (OSIDB preferred, fallback to RHSA) + vendor_mitigations = "" + if aIntel.osidb and aIntel.osidb.mitigation: + vendor_mitigations = aIntel.osidb.mitigation + elif aIntel.rhsa: + mitigation = getattr(aIntel.rhsa, 'mitigation', None) + if mitigation: + mit_text = mitigation.get('value', '') if isinstance(mitigation, dict) else str(mitigation) + if mit_text: + vendor_mitigations = mit_text + + vul_prompt = VULNERABILITY_INTEL_EXTRACTION_PROMPT.format( + vuln_id=vuln_id, + target_package=target_package.name, + cve_description=cve_description, + vendor_mitigations=vendor_mitigations or "No vendor mitigations available.", + patch_data=patch_data, + ) + vulnerability_intel: VulnerabilityIntel = await vulnerability_intel_llm.ainvoke( + [SystemMessage(content=vul_prompt)], + ) + vulnerability_intel = VulnerabilityIntelSanitizer(parsed_patch).apply( + vulnerability_intel + ) + # Prepend component_names to search_keywords for file discovery prioritization + # This ensures L1 Agent sees module names (e.g., "mod_http2") as first search patterns + if vulnerability_intel.component_names: + vulnerability_intel.search_keywords = ( + vulnerability_intel.component_names + vulnerability_intel.search_keywords + ) + # Preserve vendor mitigations in the intel object if not already extracted + if vendor_mitigations and not vulnerability_intel.known_mitigations: + vulnerability_intel.known_mitigations = vendor_mitigations + + if downstream_report: + vulnerability_intel.is_downstream_patch_available = downstream_report.is_patch_file_available + vulnerability_intel.is_patch_applied_in_build = downstream_report.is_patch_applied_in_build + vulnerability_intel.patch_file_name = downstream_report.patch_file_name or "" + + # Populate version range context from Identify phase + if ctx and ctx.identify_result: + identify_result = ctx.identify_result + if identify_result.is_target_package_affected == EnumIdentifyResult.YES: + vulnerability_intel.target_version_in_vulnerable_range = True + elif identify_result.is_target_package_affected == EnumIdentifyResult.NO: + vulnerability_intel.target_version_in_vulnerable_range = False + + affected_range, fixed_ver = extract_nvd_version_info(aIntel, target_package.name) + if affected_range: + vulnerability_intel.affected_version_range = affected_range + if fixed_ver: + vulnerability_intel.fixed_version = fixed_ver + + # Always set target version for LLM reasoning (not dependent on identify phase) + vulnerability_intel.target_package_version = target_package.version + + # Merge reference hints if available from reference mining + reference_hints = state.get("reference_hints") + if reference_hints: + vulnerability_intel = merge_reference_hints_to_intel( + intel=vulnerability_intel, + hints=reference_hints, + ) + logger.info( + "L1_agent: Merged reference hints - %d functions, %d files, revision=%s", + len(reference_hints.function_hints), + len(reference_hints.file_hints), + reference_hints.revision_hint, + ) + + # Early architecture check - skip further investigation if CVE doesn't apply + target_arch = target_package.arch + target_bitness = ARCH_TO_BITNESS.get(target_arch) + target_family = ARCH_TO_FAMILY.get(target_arch) + + arch_mismatch_reason = None + if target_bitness and vulnerability_intel.affected_bitness != "both": + if target_bitness != vulnerability_intel.affected_bitness: + arch_mismatch_reason = ( + f"Bitness mismatch: CVE affects {vulnerability_intel.affected_bitness}, " + f"target is {target_bitness} ({target_arch})" + ) + + if not arch_mismatch_reason and target_family: + if vulnerability_intel.affected_architectures is not None: + if target_family not in vulnerability_intel.affected_architectures: + arch_mismatch_reason = ( + f"Architecture mismatch: CVE affects {vulnerability_intel.affected_architectures}, " + f"target is {target_family} ({target_arch})" + ) + + if arch_mismatch_reason: + logger.info( + "L1_agent: %s - %s. Skipping further investigation.", + vuln_id, arch_mismatch_reason + ) + span.set_output({ + "vulnerability_intel": vulnerability_intel.model_dump(), + "arch_mismatch_reason": arch_mismatch_reason, + }) + return { + "vulnerability_intel": vulnerability_intel, + "arch_mismatch_reason": arch_mismatch_reason, + "runtime_prompt": "", + "messages": [], + } + + span.set_output({ + "vulnerability_intel": vulnerability_intel.model_dump(), + }) + + # Use case 1: Downstream patch file is available + if downstream_report and downstream_report.is_patch_file_available: + runtime_prompt = L1_AGENT_PROMPT_TEMPLATE.format( + sys_prompt=L1_AGENT_SYS_PROMPT_PATCH_AVAILABLE, + vuln_id=vuln_id, + target_package=target_package.name, + vulnerability_intel=vulnerability_intel.format_for_prompt(), + tools=tools_str, + tool_selection_strategy=tool_strategy, + tool_instructions=L1_AGENT_THOUGHT_INSTRUCTIONS, + ) + + span.set_output({ + "mode": "patch_available", + "patch_filename": downstream_report.patch_file_name, + }) + # Use case 2: code is fixed by rebase + elif upstream_report and upstream_report.is_code_fixed_by_rebase == "yes": + + if upstream_report.is_fixed_srpm_is_needed and upstream_report.fixed_parsed_patch: + # Has patch context - use patch-based verification + runtime_prompt = L1_AGENT_PROMPT_TEMPLATE.format( + sys_prompt=L1_AGENT_SYS_PROMPT_REBASE_FIX, + vuln_id=vuln_id, + target_package=target_package.name, + vulnerability_intel=vulnerability_intel.format_for_prompt(), + tools=tools_str, + tool_selection_strategy=tool_strategy, + tool_instructions=L1_AGENT_THOUGHT_REBASE_INSTRUCTIONS, + ) + + span.set_output({ + "mode": "rebase_fix_verification", + "spec_log_change": upstream_report.spec_file_log_change[:200] if upstream_report.spec_file_log_change else "", + }) + else: + # No patch context - use CVE description-based verification + runtime_prompt = L1_AGENT_PROMPT_TEMPLATE_NO_PATCH.format( + sys_prompt=L1_AGENT_SYS_PROMPT_REBASE_NO_PATCH, + vuln_id=vuln_id, + target_package=target_package.name, + vulnerability_intel=vulnerability_intel.format_for_prompt(), + tools=tools_str, + tool_selection_strategy=tool_strategy, + tool_instructions=L1_AGENT_THOUGHT_CVE_DESC_INSTRUCTIONS, + ) + + span.set_output({ + "mode": "rebase_fix_cve_description", + "spec_log_change": upstream_report.spec_file_log_change[:200] if upstream_report.spec_file_log_change else "", + }) + # use case 3: in target patch was not found but patch is found in the rpm that was mention in cve that is fixed + elif upstream_report and upstream_report.fixed_parsed_patch: + sys_prompt, tool_instructions = select_upstream_prompt_and_instructions( + vulnerability_intel.vulnerable_patterns + ) + runtime_prompt = L1_AGENT_PROMPT_TEMPLATE.format( + sys_prompt=sys_prompt, + vuln_id=vuln_id, + target_package=target_package.name, + vulnerability_intel=vulnerability_intel.format_for_prompt(), + tools=tools_str, + tool_selection_strategy=tool_strategy, + tool_instructions=tool_instructions, + ) + + span.set_output({ + "mode": "upstream_patch_verification", + "patch_filename": upstream_report.fixed_srpm_file_name, + "prompt_variant": "case_b" if not vulnerability_intel.vulnerable_patterns else "case_a", + }) + # use case 4: Fix commit discovered via git search + elif git_search_report and git_search_report.parsed_patch: + sys_prompt, tool_instructions = select_upstream_prompt_and_instructions( + vulnerability_intel.vulnerable_patterns + ) + runtime_prompt = L1_AGENT_PROMPT_TEMPLATE.format( + sys_prompt=sys_prompt, + vuln_id=vuln_id, + target_package=target_package.name, + vulnerability_intel=vulnerability_intel.format_for_prompt(), + tools=tools_str, + tool_selection_strategy=tool_strategy, + tool_instructions=tool_instructions, + ) + + span.set_output({ + "mode": "git_search_patch_verification", + "commit_hash": git_search_report.best_result.commit_hash_short if git_search_report.best_result else "", + "confidence": git_search_report.best_result.confidence if git_search_report.best_result else 0, + "search_method": git_search_report.best_result.search_method if git_search_report.best_result else "", + "prompt_variant": "case_b" if not vulnerability_intel.vulnerable_patterns else "case_a", + }) + else: + # Use case 4: Default prompt - no patch context, use VulnerabilityIntel from CVE description + runtime_prompt = L1_AGENT_PROMPT_TEMPLATE_NO_PATCH.format( + sys_prompt=L1_AGENT_SYS_PROMPT_REBASE_NO_PATCH, + vuln_id=vuln_id, + target_package=target_package.name, + vulnerability_intel=vulnerability_intel.format_for_prompt(), + tools=tools_str, + tool_selection_strategy=tool_strategy, + tool_instructions=L1_AGENT_THOUGHT_CVE_DESC_INSTRUCTIONS, + ) + span.set_output({ + "mode": "no_patch", + }) + + messages = state.get("messages", []) + remove_messages = [RemoveMessage(id=msg.id) for msg in messages if msg.id] + + return { + "runtime_prompt": runtime_prompt, + "vulnerability_intel": vulnerability_intel, + "messages": remove_messages, + } + + async def should_continue_downstream(state: CodeAgentState) -> str: + downstream_report = state.get("downstream_report") + if downstream_report and downstream_report.is_patch_file_available: + return "L1_agent" + else: + return "gather_more_info" + + async def downstream_search(state: CodeAgentState) -> dict: + logger.info("downstream_search: starting") + + + build_log = ctx.artifacts.build_log_path if ctx and ctx.artifacts else None + with tracer.push_active_function("downstream_search", input_data={}) as span: + report: DownstreamSearchReport = await downstream_search_preprocss( + llm=llm, + vuln_id=vuln_id, + descriptions=descriptions, + source_path=Path(source_dir), + build_log_path=Path(build_log) if build_log else None, + tracer=tracer, + ) + span.set_output({ + "is_patch_file_available": report.is_patch_file_available, + "is_patch_in_spec_file": report.is_patch_in_spec_file, + "spec_file_log_change": report.spec_file_log_change, + "is_patch_applied_in_build": report.is_patch_applied_in_build, + "build_log_patch_applied": report.build_log_patch_applied, + "parsed_patch": report.parsed_patch.patch_filename if report.parsed_patch else None, + }) + + return { + "downstream_report": report, + "messages": [AIMessage(content="Downstream flow preprocess completed")], + } + + + async def gather_more_info(state: CodeAgentState) -> dict: + logger.info("gather_more_info: starting") + # Extract commit URL candidates from intel references for patch fetching + candidates = extract_commit_url_candidates(aIntel) + with tracer.push_active_function("gather_more_info", input_data={}) as span: + report: UpstreamSearchReport = await upstream_search_preprocess( + vuln_id=vuln_id, + fix_info=fix_info, + brew_downloader=brew_downloader, + patch_dir=Path(patch_dir), + source_path=Path(source_dir), + target_package=target_package, + tracer=tracer, + intel=intel, + commit_url_candidates=candidates, + cve_description=cve_description, + llm=llm, + ) + + + + span.set_output({ + "is_fixed_srpm_is_needed": report.is_fixed_srpm_is_needed, + "is_rebase_fix": report.is_code_fixed_by_rebase == "yes", + }) + return { + "messages": [AIMessage(content="Gathering more information...")], + "upstream_report": report, + } + + async def should_continue_after_gather(state: CodeAgentState) -> str: + """Route after gather_more_info: to reference mining if no upstream patch found.""" + if not config.reference_mining_enabled: + return L1_AGENT_NODE + + upstream_report = state.get("upstream_report") + + has_upstream_patch = ( + upstream_report and + (upstream_report.fixed_parsed_patch or upstream_report.osv_result) + ) + + if has_upstream_patch: + logger.info("should_continue_after_gather: upstream patch found, proceeding to L1_agent") + return L1_AGENT_NODE + else: + logger.info("should_continue_after_gather: no upstream patch, trying reference mining") + return REFERENCE_MINING_NODE + + async def reference_mining_node(state: CodeAgentState) -> dict: + """Extract hints from advisory references when no patch is found. + + Uses LLM with reflection pattern to: + 1. Extract hints from advisory content + 2. Decide if more sources need to be fetched + + Returns: + State update with reference_hints + """ + logger.info("reference_mining_node: starting for %s", vuln_id) + + with tracer.push_active_function("reference_mining", input_data={}) as span: + # Extract advisory URLs from intel + advisory_urls = extract_advisory_urls(aIntel) + if not advisory_urls: + logger.info("reference_mining_node: no advisory URLs found") + span.set_output({"status": "no_advisory_urls"}) + return { + "messages": [AIMessage(content="No advisory URLs available for reference mining")], + "reference_hints": None, + "advisory_urls_remaining": [], + "advisory_urls_processed": [], + } + + logger.info("reference_mining_node: found %d advisory URLs", len(advisory_urls)) + + # Initialize parser and fetcher + max_chunk_tokens = config.context_window_token_limit - config.reference_mining_prompt_overhead + parser_config = ParserConfig( + max_chunk_tokens=max_chunk_tokens, + chunk_overlap_tokens=config.reference_mining_chunk_overlap, + ) + parser = ReflectiveReferenceParser(llm=llm, config=parser_config) + fetcher = ReferenceFetcher( + timeout_seconds=config.reference_mining_fetch_timeout, + max_content_length=config.reference_mining_max_content_bytes, + ) + + # Initialize state + hints = ReferenceHints() + processed: list[str] = [] + remaining = list(advisory_urls) + max_refs = config.reference_mining_max_urls + + async with aiohttp.ClientSession() as session: + while remaining and len(processed) < max_refs: + # Pop highest priority URL (sorted by priority) + url, source_type, priority = remaining.pop(0) + processed.append(url) + + logger.info( + "reference_mining_node: processing URL %d/%d: %s", + len(processed), max_refs, url + ) + + # Fetch advisory content + content: AdvisoryContent = await fetcher.fetch_advisory( + session=session, + url=url, + source_type=source_type, + ) + + if not content.fetch_success: + logger.warning( + "reference_mining_node: failed to fetch %s: %s", + url, content.error_message + ) + continue + + if not content.raw_text.strip(): + logger.warning("reference_mining_node: empty content from %s", url) + continue + + # Extract hints from content + with tracer.push_active_function("extract_advisory_hints", input_data={ + "url": url, + "source_type": source_type, + "content_length": len(content.raw_text), + }) as extract_span: + extraction = await parser.extract_hints( + advisory_content=content.raw_text, + vuln_id=vuln_id, + cve_description=cve_description, + package_name=target_package.name, + advisory_url=url, + source_type=source_type, + ) + + extract_span.set_output({ + "revision_hint": extraction.revision_hint, + "function_hints": extraction.function_hints[:5], + "file_hints": extraction.file_hints[:5], + }) + + # Merge hints BEFORE reflection so reflection sees accumulated state + hints = _merge_hints(hints, extraction) + + # Reflect on ACCUMULATED hints to assess overall strength + reflection = await parser.reflect_on_extraction( + extraction=hints, + vuln_id=vuln_id, + cve_description=cve_description, + package_name=target_package.name, + current_url=url, + remaining_urls=remaining, + ) + + hints.hint_strength = reflection.hint_strength + hints.confidence = max(hints.confidence, reflection.confidence) + + # Control flow decision + if not reflection.needs_more_sources: + logger.info( + "reference_mining_node: hints sufficient (strength=%s, confidence=%.2f)", + reflection.hint_strength, reflection.confidence + ) + break + + # Update sources processed + hints.sources_processed = processed + + span.set_output({ + "urls_processed": len(processed), + "hint_strength": hints.hint_strength, + "confidence": hints.confidence, + "revision_hint": hints.revision_hint, + "function_hints": len(hints.function_hints), + "file_hints": len(hints.file_hints), + }) + + logger.info( + "reference_mining_node: completed - revision=%s, %d functions, %d files", + hints.revision_hint, + len(hints.function_hints), + len(hints.file_hints), + ) + + return { + "messages": [AIMessage(content="Reference mining completed")], + "reference_hints": hints if (hints.revision_hint or hints.function_hints or hints.file_hints) else None, + "advisory_urls_remaining": [u[0] for u in remaining], + "advisory_urls_processed": processed, + } + + def should_run_repo_resolution(state: CodeAgentState) -> str: + """Decide if repo resolution should run after reference mining. + + Run repo resolution when reference mining produced usable hints. + + Note: By the time we reach here, we already know there's no upstream + patch (that's why we went to reference_mining in the first place). + + Returns: + Node name to route to. + """ + reference_hints = state.get("reference_hints") + + # Skip if no hints to search for + if not reference_hints: + logger.debug("should_run_repo_resolution: no reference hints") + return L1_AGENT_NODE + + # Check hint strength + hint_strength = getattr(reference_hints, "hint_strength", "none") + if hint_strength == "none": + logger.debug("should_run_repo_resolution: hint strength is none") + return L1_AGENT_NODE + + logger.info("should_run_repo_resolution: routing to repo_resolution") + return REPO_RESOLUTION_NODE + + async def repo_resolution_node(state: CodeAgentState) -> dict: + """Resolve package name to upstream git repository URL. + + Uses multiple strategies in priority order: + 1. OSIDB upstream_purls (confidence 0.95) + 2. Curated package → repo mapping (confidence 0.90, PRIMARY) + 3. PURL parsing from any intel source (confidence 0.80) + + Returns: + State update with repo_resolution + """ + logger.info("repo_resolution_node: starting for %s", target_package.name) + + with tracer.push_active_function("repo_resolution", input_data={}) as span: + from vuln_analysis.utils.repo_resolver import RepoResolver + + resolver = RepoResolver() + resolution = resolver.resolve( + package_name=target_package.name, + intel=aIntel, + ) + + span.set_output({ + "repo_url": resolution.repo_url, + "resolution_method": resolution.resolution_method, + "confidence": resolution.confidence, + "is_alias": resolution.is_alias, + "strategies_tried": resolution.strategies_tried, + }) + + if resolution.repo_url: + logger.info( + "repo_resolution_node: resolved %s → %s (method=%s, confidence=%.2f)", + target_package.name, + resolution.repo_url, + resolution.resolution_method, + resolution.confidence, + ) + msg_content = f"Resolved repository: {resolution.repo_url}" + else: + logger.info( + "repo_resolution_node: no repo found for %s", + target_package.name, + ) + msg_content = f"Could not resolve repository for {target_package.name}" + + return { + "messages": [AIMessage(content=msg_content)], + "repo_resolution": resolution, + } + + def should_run_git_search(state: CodeAgentState) -> str: + """Decide if git search should run after repo resolution. + + Run git search when: + 1. Git search is enabled in config + 2. Repo URL was resolved successfully + 3. Reference hints exist (something to search for) + """ + if not config.git_search_enabled: + logger.debug("should_run_git_search: disabled in config") + return L1_AGENT_NODE + + repo_resolution = state.get("repo_resolution") + reference_hints = state.get("reference_hints") + + if not repo_resolution or not repo_resolution.repo_url: + logger.info("should_run_git_search: no repo_url, skipping to L1") + return L1_AGENT_NODE + + if not reference_hints: + logger.info("should_run_git_search: no hints, skipping to L1") + return L1_AGENT_NODE + + has_useful_hints = ( + reference_hints.revision_hint or + reference_hints.function_hints + ) + + if not has_useful_hints: + logger.info("should_run_git_search: no useful hints, skipping to L1") + return L1_AGENT_NODE + + logger.info("should_run_git_search: routing to git_search") + return GIT_SEARCH_NODE + + async def git_search_node(state: CodeAgentState) -> dict: + """Search git repository for CVE fix commit. + + Requires: + - state.repo_resolution.repo_url (from REPO_RESOLUTION_NODE) + - state.reference_hints (from REFERENCE_MINING_NODE) + + Returns: + State update with git_search_report + """ + import time + from vuln_analysis.utils.git_repo_manager import GitRepoManager + from vuln_analysis.utils.git_commit_searcher import GitCommitSearcher + + start_time = time.time() + + repo_resolution = state.get("repo_resolution") + reference_hints = state.get("reference_hints") + + if not repo_resolution or not repo_resolution.repo_url: + logger.warning("git_search_node: No repo_url available, skipping") + return {"git_search_report": None} + + logger.info("git_search_node: searching %s", repo_resolution.repo_url) + + with tracer.push_active_function("git_search", input_data={ + "repo_url": repo_resolution.repo_url, + "revision_hint": reference_hints.revision_hint if reference_hints else None, + "has_function_hints": bool(reference_hints and reference_hints.function_hints), + }) as span: + try: + repo_manager = GitRepoManager( + cache_dir=config.git_search_cache_dir, + default_depth=config.git_search_clone_depth, + fetch_timeout_seconds=config.git_search_fetch_timeout_seconds, + ) + searcher = GitCommitSearcher(repo_manager, tracer=tracer) + + repo_path, is_fresh = await repo_manager.clone_or_update( + repo_resolution.repo_url + ) + clone_time = int((time.time() - start_time) * 1000) + + search_start = time.time() + report = await searcher.search( + repo_path=repo_path, + repo_url=repo_resolution.repo_url, + hints=reference_hints, + llm=llm, + package_name=target_package.name, + cve_description=cve_description, + ) + search_time = int((time.time() - search_start) * 1000) + + if report.best_result and report.best_result.confidence >= config.git_search_min_confidence: + parsed_patch = await searcher.commit_to_parsed_patch( + repo_path=repo_path, + commit_hash=report.best_result.commit_hash, + max_diff_size=config.git_search_max_diff_size, + repo_url=repo_resolution.repo_url, + ) + report.parsed_patch = parsed_patch + report.commit_diff = await searcher.get_commit_diff( + repo_path=repo_path, + commit_hash=report.best_result.commit_hash, + max_diff_size=config.git_search_max_diff_size, + ) + + report.clone_time_ms = clone_time + report.search_time_ms = search_time + report.total_time_ms = int((time.time() - start_time) * 1000) + + span.set_output({ + "search_success": report.search_success, + "strategies_tried": report.strategies_tried, + "strategies_succeeded": report.strategies_succeeded, + "best_result_method": report.best_result.search_method if report.best_result else None, + "best_result_confidence": report.best_result.confidence if report.best_result else 0, + "total_results": len(report.all_results), + "clone_time_ms": report.clone_time_ms, + "search_time_ms": report.search_time_ms, + "cache_hit": not is_fresh, + }) + + if report.best_result: + msg_content = ( + f"Found fix commit: {report.best_result.commit_hash_short} " + f"(confidence={report.best_result.confidence:.2f}, method={report.best_result.search_method})" + ) + logger.info( + "git_search_node: Found commit %s with confidence %.2f", + report.best_result.commit_hash_short, + report.best_result.confidence, + ) + else: + msg_content = "Git search did not find a fix commit" + logger.info("git_search_node: No commit found") + + return { + "messages": [AIMessage(content=msg_content)], + "git_search_report": report, + } + + except Exception as e: + logger.exception("git_search_node: Error during search") + error_report = GitSearchReport( + repo_url=repo_resolution.repo_url, + search_success=False, + error_messages=[str(e)], + ) + span.set_output({"error": str(e)}) + return { + "messages": [AIMessage(content=f"Git search failed: {e}")], + "git_search_report": error_report, + } + + async def thought_node(state: CodeAgentState) -> dict: + """Generate next thought/action using the LLM.""" + step_num = state.get("step", 0) + logger.info("thought_node: starting step %d", step_num) + runtime_prompt = state.get("runtime_prompt") or "You are a security analyst investigating a CVE." + messages = [SystemMessage(content=runtime_prompt)] + state["messages"] + with tracer.push_active_function("thought_node", input_data={}) as span: + obs = state.get("observation", None) + if obs is not None: + memory_list = obs.memory if obs.memory else ["No prior knowledge."] + recent_findings = obs.results if obs.results else ["No recent findings."] + memory_context = "\n".join(f"- {m}" for m in memory_list) + findings_context = "\n".join(f"- {f}" for f in recent_findings) + context_block = f"KNOWLEDGE:\n{memory_context}\nLATEST FINDINGS:\n{findings_context}" + messages.append(SystemMessage(content=context_block)) + response: CheckerThought = await thought_llm.ainvoke(messages) + if response.mode == "finish": + ai_message = AIMessage(content=response.final_answer) + else: + tool_name = response.actions.tool + arguments = response.actions.query + tool_call_id = str(uuid.uuid4()) + ai_message = AIMessage( + content=response.thought, + tool_calls=[{"name": tool_name, "args": {"query": arguments}, "id": tool_call_id}] + ) + span.set_output({ + "thought": response.thought, + "mode": response.mode, + "actions": response.actions, + "final_answer": response.final_answer, + }) + return { + "messages": [ai_message], + "thought": response, + "step": step_num + 1, + "max_steps": config.max_iterations, + } + + async def forced_finish_node(state: CodeAgentState) -> dict: + """Force finish when max iterations reached. + + Invokes the LLM with FORCED_FINISH_PROMPT to generate a final answer + based on evidence gathered so far. + """ + step_num = state.get("step", 0) + with tracer.push_active_function("forced_finish_node", input_data=f"step:{step_num}") as span: + try: + active_prompt = state.get("runtime_prompt") + messages = [SystemMessage(content=active_prompt)] + state["messages"] + messages.append(HumanMessage(content=FORCED_FINISH_PROMPT)) + + obs = state.get("observation") + if obs is not None and obs.memory: + memory_context = "\n".join(f"- {m}" for m in obs.memory) + messages.append(SystemMessage(content=f"KNOWLEDGE:\n{memory_context}")) + + response: CheckerThought = await thought_llm.ainvoke(messages) + + if response.mode == "finish" and response.final_answer: + ai_message = AIMessage(content=response.final_answer) + final_answer = response.final_answer + else: + final_answer = "Failed to generate a final answer within the maximum allowed steps." + ai_message = AIMessage(content=final_answer) + response = CheckerThought( + thought=response.thought or "Max steps exceeded", + mode="finish", + actions=None, + final_answer=final_answer, + ) + + span.set_output({"final_answer_length": len(final_answer), "step": step_num}) + return { + "messages": [ai_message], + "thought": response, + "step": step_num, + "max_steps": state.get("max_steps", config.max_iterations), + "observation": state.get("observation"), + "output": final_answer, + } + except Exception as e: + logger.exception("forced_finish_node failed at step %d", step_num) + span.set_output({"error": str(e), "exception_type": type(e).__name__, "step": step_num}) + raise + + async def observation_node(state: CodeAgentState) -> dict: + """Process tool output: comprehension -> memory update with VulnerabilityIntel context.""" + logger.info("observation_node: starting") + tool_message = state["messages"][-1] + last_thought = state.get("thought") + if not last_thought: + return { + "messages": [AIMessage(content="No thought found")], + } + if not last_thought.actions: + logger.warning("observation_node: thought has no actions (mode=%s), skipping", last_thought.mode) + return { + "messages": [AIMessage(content="No tool action to process")], + } + last_thought_text = last_thought.thought + tool_used = last_thought.actions.tool + tool_input_detail = last_thought.actions.query + previous_memory = state.get("observation").memory if state.get("observation") else ["No data gathered yet."] + + vulnerability_intel = state.get("vulnerability_intel") + intel_formatted = vulnerability_intel.format_for_prompt() if vulnerability_intel else "No intel available" + target_package_name = target_package.name if target_package else "unknown" + + with tracer.push_active_function("observation node", input_data=f"tool used:{tool_used} + {tool_input_detail}") as span: + tool_output_for_llm = tool_message.content + + # Check for empty/error outputs + # For empty Source Grep on source code, let LLM classify against RAW_PATCH_DIFF + # to distinguish VULNERABLE_CODE_ABSENT vs FIX_CODE_ABSENT + empty_findings, needs_llm_classification = check_empty_output( + tool_output_for_llm, tool_used, tool_input_detail, + allow_llm_classification=True + ) + + # Get parsed_patch from state for raw diff context (needed for classification and comprehension) + downstream_report = state.get("downstream_report") + upstream_report = state.get("upstream_report") + parsed_patch = None + + if downstream_report: + if isinstance(downstream_report, dict): + parsed_patch = downstream_report.get('parsed_patch') + else: + parsed_patch = getattr(downstream_report, 'parsed_patch', None) + + if not parsed_patch and upstream_report: + if isinstance(upstream_report, dict): + parsed_patch = upstream_report.get('fixed_parsed_patch') + else: + parsed_patch = getattr(upstream_report, 'fixed_parsed_patch', None) + + # If parsed_patch is a dict, convert it to ParsedPatch model + if parsed_patch and isinstance(parsed_patch, dict): + try: + parsed_patch = ParsedPatch(**parsed_patch) + except Exception as e: + logger.warning("Failed to parse parsed_patch dict: %s", e) + parsed_patch = None + + logger.debug("observation_node: parsed_patch=%s, downstream=%s, upstream=%s", + parsed_patch is not None, + downstream_report is not None, + upstream_report is not None) + + # Extract relevant hunks based on grep target file + raw_patch_diff = "" + if tool_used == "Source Grep" and parsed_patch: + raw_patch_diff = get_relevant_hunks(parsed_patch, tool_input_detail) + + if empty_findings: + code_findings = empty_findings + elif needs_llm_classification: + # Empty source grep - use classification prompt to determine meaning + classification_prompt = L1_EMPTY_RESULT_CLASSIFICATION_PROMPT.format( + tool_used=tool_used, + last_thought=last_thought_text, + tool_input=tool_input_detail, + raw_patch_diff=raw_patch_diff if raw_patch_diff else "No patch diff available", + ) + code_findings = await structured_comprehension_llm.ainvoke( + [SystemMessage(content=classification_prompt)] + ) + logger.debug("Empty source grep classified: %s", code_findings.findings) + else: + # Has actual content - split into chunks and process each + chunks = truncate_tool_output_list(tool_output_for_llm, tool_used, max_tokens=1000) + all_findings = [] + best_tool_outcome = "" + + for chunk in chunks: + comp_prompt = L1_COMPREHENSION_PROMPT.format( + vuln_id=vuln_id, + target_package=target_package_name, + vulnerability_intel=intel_formatted, + raw_patch_diff=raw_patch_diff, + tool_used=tool_used, + tool_input=tool_input_detail, + last_thought=last_thought_text, + tool_output=chunk, + ) + chunk_findings = await invoke_comprehension( + structured_comprehension_llm, + comp_prompt, + tool_used, + tool_input_detail, + chunk, + agent_label="L1", + ) + all_findings.extend(chunk_findings.findings) + # Keep tool_outcome from chunk with actual findings (not FAILED) + if not best_tool_outcome or ( + chunk_findings.findings and + not any("FAILED" in f for f in chunk_findings.findings) + ): + best_tool_outcome = chunk_findings.tool_outcome + + code_findings = CodeFindings( + findings=all_findings, + tool_outcome=best_tool_outcome or "No matches found" + ) + findings_text = "\n".join(f"- {f}" for f in code_findings.findings) + + # Step 2: Memory update - merge findings into cumulative memory + mem_prompt = L1_MEMORY_UPDATE_PROMPT.format( + vuln_id=vuln_id, + target_package=target_package_name, + previous_memory="\n".join(f"- {m}" for m in previous_memory) if isinstance(previous_memory, list) else previous_memory, + findings=findings_text, + tool_outcome=code_findings.tool_outcome, + ) + new_observation: Observation = await observation_llm.ainvoke([SystemMessage(content=mem_prompt)]) + + messages = state["messages"] + active_prompt = state.get("runtime_prompt") + estimated = _estimate_tokens(active_prompt, messages, new_observation) + prune_messages = [] + orig_estimated = estimated + + if estimated > config.context_window_token_limit: + l_tool_count = _count_tokens(tool_output_for_llm) + for msg in messages: + prune_messages.append(RemoveMessage(id=msg.id)) + estimated -= _count_tokens(msg.content) if hasattr(msg, "content") and isinstance(msg.content, str) else 0 + if estimated <= config.context_window_token_limit: + break + logger.info( + "Context pruning: removed %d messages, estimated tokens now ~%d (limit %d)", + len(prune_messages), estimated, config.context_window_token_limit, + ) + + span.set_output({ + "last_thought_text": last_thought_text, + "tool_output_for_llm": tool_output_for_llm[:500], + "findings": code_findings.findings, + "tool_outcome": code_findings.tool_outcome, + "new_memory": new_observation.memory, + "amount_of_orig_tokens": orig_estimated, + "amount_of_estimated_tokens": estimated, + }) + return { + "messages": prune_messages, + "observation": new_observation, + } + + async def should_continue(state: CodeAgentState) -> str: + thought = state.get("thought", None) + if thought is not None and thought.mode == "finish": + return END + if state.get("step", 0) >= state.get("max_steps", config.max_iterations): + return FORCED_FINISH_NODE + return TOOL_NODE + + def should_continue_after_intel(state: CodeAgentState) -> str: + """Route after L1_agent: skip ReAct loop if architecture mismatch detected.""" + if state.get("arch_mismatch_reason"): + return END + return THOUGHT_NODE + + flow = StateGraph(CodeAgentState) + + flow.add_node(DOWNSTREAM_SEARCH_NODE, downstream_search) + flow.add_node(GATHER_MORE_INFO_NODE, gather_more_info) + flow.add_node(REFERENCE_MINING_NODE, reference_mining_node) + flow.add_node(REPO_RESOLUTION_NODE, repo_resolution_node) + flow.add_node(GIT_SEARCH_NODE, git_search_node) + flow.add_node(L1_AGENT_NODE, L1_agent) + flow.add_node(THOUGHT_NODE, thought_node) + flow.add_node(TOOL_NODE, tools_node) + flow.add_node(FORCED_FINISH_NODE, forced_finish_node) + flow.add_node(OBSERVATION_NODE, observation_node) + + flow.add_edge(START, DOWNSTREAM_SEARCH_NODE) + flow.add_conditional_edges(DOWNSTREAM_SEARCH_NODE, should_continue_downstream, { + L1_AGENT_NODE: L1_AGENT_NODE, + GATHER_MORE_INFO_NODE: GATHER_MORE_INFO_NODE, + }) + # Route from gather_more_info: to reference mining if no upstream patch, else directly to L1 + flow.add_conditional_edges(GATHER_MORE_INFO_NODE, should_continue_after_gather, { + L1_AGENT_NODE: L1_AGENT_NODE, + REFERENCE_MINING_NODE: REFERENCE_MINING_NODE, + }) + # Route from reference mining: to repo resolution if hints found, else directly to L1 + flow.add_conditional_edges(REFERENCE_MINING_NODE, should_run_repo_resolution, { + L1_AGENT_NODE: L1_AGENT_NODE, + REPO_RESOLUTION_NODE: REPO_RESOLUTION_NODE, + }) + # Route from repo resolution: to git search if enabled and hints available, else directly to L1 + flow.add_conditional_edges(REPO_RESOLUTION_NODE, should_run_git_search, { + L1_AGENT_NODE: L1_AGENT_NODE, + GIT_SEARCH_NODE: GIT_SEARCH_NODE, + }) + # Git search always leads to L1 agent + flow.add_edge(GIT_SEARCH_NODE, L1_AGENT_NODE) + flow.add_conditional_edges(L1_AGENT_NODE, should_continue_after_intel, { + END: END, + THOUGHT_NODE: THOUGHT_NODE, + }) + flow.add_conditional_edges( + THOUGHT_NODE, + should_continue, + {END: END, TOOL_NODE: TOOL_NODE, FORCED_FINISH_NODE: FORCED_FINISH_NODE} + ) + flow.add_edge(TOOL_NODE, OBSERVATION_NODE) + flow.add_edge(OBSERVATION_NODE, THOUGHT_NODE) + flow.add_edge(FORCED_FINISH_NODE, END) + + + app = flow.compile() + return app + + +@register_function(config_type=CVEPackageCodeAgentConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def cve_package_code_agent(config: CVEPackageCodeAgentConfig, builder: Builder): + + async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Run L1 investigation and return intermediate result for routing to L2 or report generation.""" + trace_id.set(message.input.scan.id) + tracer = Context.get() + + # Set ctx_state so tools (e.g., Source Grep, Lexical Search) can access checker_context + from types import SimpleNamespace + workflow_state = SimpleNamespace(original_input=message, info=message.info) + ctx_state.set(workflow_state) + + logger.info("package_code_agent: starting L1 investigation") + + l1_agent_graph = await create_graph_code_agent(config, builder, message, tracer) + initial_state: CodeAgentState = { + "messages": [HumanMessage(content="Begin L1 CVE investigation")], + "step": 0, + "max_steps": config.max_iterations, + } + + with tracer.push_active_function("l1_agent_graph", input_data=initial_state["messages"][0].content): + result = await l1_agent_graph.ainvoke( + initial_state, + config={"recursion_limit": config.max_iterations * 4}, + ) + + logger.info("package_code_agent: L1 investigation finished") + + # Check for early architecture mismatch exit + arch_mismatch_reason = result.get("arch_mismatch_reason") + if arch_mismatch_reason: + logger.info( + "package_code_agent: Architecture mismatch detected - %s", + arch_mismatch_reason + ) + l1_result = L1InvestigationResult( + downstream_report=None, + upstream_report=None, + git_search_report=None, + l1_agent_answer=arch_mismatch_reason, + vulnerability_intel=result.get("vulnerability_intel"), + preliminary_verdict="not_present", + confidence=1.0, + ) + if message.info.checker_context is not None: + message.info.checker_context.l1_result = l1_result + return message + + final_answer = None + thought = result.get("thought") + if thought and thought.mode == "finish": + final_answer = thought.final_answer + + vuln_id = message.input.scan.vulns[0].vuln_id + target_package = message.input.image.target_package + target_package_name = target_package.name if target_package else "unknown" + + llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + verdict_extraction = await extract_l1_verdict( + llm=llm, + vuln_id=vuln_id, + target_package=target_package_name, + final_answer=final_answer or "No final answer produced.", + tracer=tracer, + ) + preliminary_verdict = verdict_extraction.preliminary_verdict + confidence = verdict_extraction.confidence + + downstream_report: DownstreamSearchReport | None = result.get("downstream_report") + upstream_report: UpstreamSearchReport | None = result.get("upstream_report") + git_search_report: GitSearchReport | None = result.get("git_search_report") + vulnerability_intel: VulnerabilityIntel | None = result.get("vulnerability_intel") + + observation = result.get("observation") + if observation and vulnerability_intel: + discovered_files = _extract_discovered_files_from_memory(observation.memory or []) + if discovered_files: + existing_files = set(vulnerability_intel.affected_files or []) + vulnerability_intel.affected_files = list(existing_files | discovered_files) + logger.debug( + "package_code_agent: Updated affected_files with L1 discoveries: %s", + discovered_files, + ) + + l1_result = L1InvestigationResult( + downstream_report=downstream_report.model_dump() if downstream_report else None, + upstream_report=upstream_report.model_dump() if upstream_report else None, + git_search_report=git_search_report.model_dump() if git_search_report else None, + l1_agent_answer=final_answer, + vulnerability_intel=vulnerability_intel, + preliminary_verdict=preliminary_verdict, + confidence=confidence, + ) + + with tracer.push_active_function( + "l1_agent_finish", + input_data={"preliminary_verdict": preliminary_verdict}, + ) as span: + span.set_output({ + "l1_agent_answer": final_answer[:500] if final_answer else None, + "vulnerability_intel": vulnerability_intel, + "confidence": l1_result.confidence, + }) + + if message.info.checker_context is not None: + message.info.checker_context.l1_result = l1_result + else: + logger.warning("package_code_agent: checker_context is None, cannot store l1_result") + logger.info( + "package_code_agent: L1 result - verdict=%s, confidence=%.2f", + preliminary_verdict, + l1_result.confidence, + ) + return message + + yield FunctionInfo.from_fn( + _arun, + description="Level 1 Package Code Agent: investigates CVEs using extracted source and Tantivy code index", + ) diff --git a/src/vuln_analysis/functions/cve_process_sbom.py b/src/vuln_analysis/functions/cve_process_sbom.py index 736673add..39a11788b 100644 --- a/src/vuln_analysis/functions/cve_process_sbom.py +++ b/src/vuln_analysis/functions/cve_process_sbom.py @@ -13,7 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +import ipaddress +import os +import socket +from urllib.parse import urlparse from aiq.builder.builder import Builder from aiq.builder.framework_enum import LLMFrameworkEnum @@ -23,7 +26,38 @@ from pydantic import Field from pydantic import NonNegativeInt -logger = logging.getLogger(__name__) +from exploit_iq_commons.data_models.common import AnalysisType +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) + +_ALLOWED_SBOM_DIR = os.getenv("SBOM_ALLOWED_DIR", "/tmp/sbom") + + +def _validate_file_path(file_path: str) -> str: + resolved = os.path.realpath(file_path) + allowed = os.path.realpath(_ALLOWED_SBOM_DIR) + if not resolved.startswith(allowed + os.sep) and resolved != allowed: + raise ValueError(f"SBOM file path not within allowed directory ({_ALLOWED_SBOM_DIR}): {file_path}") + return resolved + + +def _validate_url(url: str) -> str: + parsed = urlparse(url) + if parsed.scheme not in ("https", "http"): + raise ValueError(f"URL scheme must be http or https, got: {parsed.scheme!r}") + hostname = parsed.hostname + if not hostname: + raise ValueError(f"URL has no hostname: {url}") + try: + resolved_ips = socket.getaddrinfo(hostname, None) + except socket.gaierror: + raise ValueError(f"Cannot resolve hostname: {hostname}") + for _family, _type, _proto, _canonname, sockaddr in resolved_ips: + ip = ipaddress.ip_address(sockaddr[0]) + # Allow only loopback for testing(localhost) or public http/https url locations for sbom. + if ip.is_private or ip.is_link_local or ip.is_reserved: + raise ValueError(f"URL resolves to non-public address ({ip}): {url}") + return url class CVEProcessSBOMConfig(FunctionBaseConfig, name="cve_process_sbom"): @@ -36,12 +70,12 @@ class CVEProcessSBOMConfig(FunctionBaseConfig, name="cve_process_sbom"): @register_function(config_type=CVEProcessSBOMConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def cve_process_sbom(config: CVEProcessSBOMConfig, builder: Builder): # pylint: disable=unused-argument - from vuln_analysis.data_models.info import AgentMorpheusInfo - from vuln_analysis.data_models.input import AgentMorpheusEngineInput - from vuln_analysis.data_models.input import FileSBOMInfoInput - from vuln_analysis.data_models.input import HTTPSBOMInfoInput - from vuln_analysis.data_models.input import ManualSBOMInfoInput - from vuln_analysis.data_models.input import SBOMPackage + from exploit_iq_commons.data_models.info import AgentMorpheusInfo + from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput + from exploit_iq_commons.data_models.input import FileSBOMInfoInput + from exploit_iq_commons.data_models.input import HTTPSBOMInfoInput + from exploit_iq_commons.data_models.input import ManualSBOMInfoInput + from exploit_iq_commons.data_models.input import SBOMPackage from vuln_analysis.utils import http_utils from vuln_analysis.utils.http_utils import HTTPMethod @@ -69,7 +103,12 @@ def _parse_sbom_packages(sbom_lines: list[str]) -> list[SBOMPackage]: return packages - if (message.input.image.sbom_info.type == ManualSBOMInfoInput.static_type()): + if message.input.image.analysis_type == AnalysisType.SOURCE: + logger.info("Source code analysis requested. No SBOM processing needed.") + return message + + elif (message.input.image.sbom_info.type == ManualSBOMInfoInput.static_type()): + assert isinstance(message.input.image.sbom_info, ManualSBOMInfoInput) # Create the SBOM object @@ -78,8 +117,8 @@ def _parse_sbom_packages(sbom_lines: list[str]) -> list[SBOMPackage]: elif (message.input.image.sbom_info.type == FileSBOMInfoInput.static_type()): assert isinstance(message.input.image.sbom_info, FileSBOMInfoInput) - # Read the file to an object - with open(message.input.image.sbom_info.file_path, "r", encoding="utf-8") as f: + safe_path = _validate_file_path(message.input.image.sbom_info.file_path) + with open(safe_path, "r", encoding="utf-8") as f: sbom_lines = f.readlines() # Create the SBOM object @@ -89,9 +128,10 @@ def _parse_sbom_packages(sbom_lines: list[str]) -> list[SBOMPackage]: assert isinstance(message.input.image.sbom_info, HTTPSBOMInfoInput) try: + validated_url = _validate_url(message.input.image.sbom_info.url) _, response = http_utils.request_with_retry( request_kwargs={ - "url": message.input.image.sbom_info.url, "method": HTTPMethod.GET.value + "url": validated_url, "method": HTTPMethod.GET.value }, max_retries=config.max_retries ) diff --git a/src/vuln_analysis/functions/cve_segmentation.py b/src/vuln_analysis/functions/cve_segmentation.py new file mode 100644 index 000000000..7a1135d8b --- /dev/null +++ b/src/vuln_analysis/functions/cve_segmentation.py @@ -0,0 +1,280 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Late-phase node: document parsing, VDB creation, and code indexing. + +This node runs late in the pipeline (after verify_vuln_package) so it can +be skipped if no CVE is vulnerable. The repo should already be cloned by +cve_clone_and_deps. +""" + +import json +import os +import time +from pathlib import Path + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.component_ref import FunctionRef +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id +from exploit_iq_commons.utils.credential_client import credential_context +from vuln_analysis.tools.tool_names import ToolNames + +logger = LoggingFactory.get_agent_logger(__name__) + + +class CVESegmentationConfig(FunctionBaseConfig, name="cve_segmentation"): + """ + Configuration for the segmentation node. + + This node builds VDBs and code indexes from already-cloned repositories. + It runs late in the pipeline after verify_vuln_package. + """ + + agent_name: FunctionRef = Field( + description="Name of the agent executor. Used to determine which tools are enabled.", + ) + embedder_name: str = Field(description="Name of embedder to use for VDB creation.") + base_git_dir: str = Field( + default=".cache/am_cache/git", + description="Directory for storing cloned git repositories.", + ) + base_vdb_dir: str = Field( + default=".cache/am_cache/vdb", + description="Directory for storing vector database files.", + ) + base_code_index_dir: str = Field( + default=".cache/am_cache/code_index", + description="Directory for storing code index files.", + ) + base_pickle_dir: str = Field( + default=".cache/am_cache/pickle", + description="Directory for storing pickle cache files.", + ) + ignore_errors: bool = Field( + default=False, + description="Whether to ignore errors during segmentation.", + ) + ignore_code_embedding: bool = Field( + default=False, + description="Whether to skip building code vector database.", + ) + ignore_code_index: bool = Field( + default=False, + description="Whether to skip building code index.", + ) + + +class DocumentCollectionError(Exception): + """Raised when document parsing/collection fails in segmentation phase.""" + + +@register_function(config_type=CVESegmentationConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def segmentation(config: CVESegmentationConfig, builder: Builder): + from exploit_iq_commons.data_models.info import AgentMorpheusInfo + from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput + from exploit_iq_commons.data_models.input import SourceDocumentsInfo + from exploit_iq_commons.utils.document_embedding import DocumentEmbedding + from exploit_iq_commons.utils.standard_library_cache import StandardLibraryCache + from vuln_analysis.functions.cve_agent import CVEAgentExecutorToolConfig + from vuln_analysis.utils.full_text_search import FullTextSearch + + agent_config = builder.get_function_config(config.agent_name) + assert isinstance(agent_config, CVEAgentExecutorToolConfig) + + if ToolNames.CODE_SEMANTIC_SEARCH not in agent_config.tool_names: + logger.info("CODE_SEMANTIC_SEARCH not enabled, setting ignore_code_embedding=True") + config.ignore_code_embedding = True + + if ToolNames.CODE_KEYWORD_SEARCH not in agent_config.tool_names: + logger.info("CODE_KEYWORD_SEARCH not enabled, setting ignore_code_index=True") + config.ignore_code_index = True + + embedding = await builder.get_embedder( + embedder_name=config.embedder_name, + wrapper_type=LLMFrameworkEnum.LANGCHAIN, + ) + + cache_std = StandardLibraryCache.get_instance() + cache_std.set_cache_directory(config.base_pickle_dir) + + embedder = DocumentEmbedding( + embedding=embedding, + vdb_directory=config.base_vdb_dir, + git_directory=config.base_git_dir, + pickle_cache_directory=config.base_pickle_dir, + ) + + def _create_code_index( + source_infos: list[SourceDocumentsInfo], + embedder: DocumentEmbedding, + output_path: Path, + ) -> bool: + logger.info( + "Collecting documents for code index. Source Infos: %s", + json.dumps([x.model_dump(mode="json") for x in source_infos]), + ) + + output_path = Path(output_path) + if output_path.exists(): + logger.warning("Code index already exists and will be overwritten: %s", output_path) + + documents = [] + for si in source_infos: + try: + documents.extend(embedder.collect_documents_from_cloned(si)) + except Exception as e: + logger.warning("Error collecting documents for %s: %s", si, e, exc_info=True) + raise DocumentCollectionError(e) from e + + if len(documents) == 0: + logger.warning("No documents collected, skipping code index for: %s", source_infos) + return False + + chunked_documents = embedder._chunk_documents(documents) + + logger.info( + "Creating code index. Doc count: %d, Chunks: %d, Location: %s", + len(documents), + len(chunked_documents), + output_path, + ) + + full_text_search = FullTextSearch(cache_path=str(output_path)) + indexing_start_time = time.time() + full_text_search.add_documents_from_langchain_chunks(chunked_documents) + logger.info( + "Completed code indexing in %.2f seconds for '%s'", + time.time() - indexing_start_time, + output_path, + ) + return True + + def _build_code_index(source_infos: list[SourceDocumentsInfo]) -> Path | None: + code_sources = [si for si in source_infos if si.type == "code"] + if not code_sources: + return None + + index_embedder = DocumentEmbedding( + embedding=None, + vdb_directory=config.base_vdb_dir, + git_directory=config.base_git_dir, + pickle_cache_directory=config.base_pickle_dir, + ) + + code_index_path = FullTextSearch.get_index_directory( + base_path=config.base_code_index_dir, + hash_value=index_embedder.hash_source_documents_info(code_sources), + ) + + if not code_index_path.exists() or os.environ.get("MORPHEUS_ALWAYS_REBUILD_VDB", "0") == "1": + documents_exist = _create_code_index(code_sources, index_embedder, code_index_path) + if not documents_exist: + return None + else: + logger.info("Cache hit on code index: %s", code_index_path) + + return code_index_path + + async def _arun(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """ + Build VDBs and code indexes from already-cloned repositories. + + This is the second phase of the split pipeline. Repos should already + be cloned by clone_and_deps. + """ + trace_id.set(state.input.scan.id) + message = state.input + source_infos = message.image.source_info + base_image = message.image.name + + vdb_code_path = None + vdb_doc_path = None + code_index_path = None + failure_reason = "" + + try: + logger.debug( + "segmentation: credential_id=%r scan_id=%s", + message.credential_id, + message.scan.id, + ) + + with credential_context(message.credential_id): + vdb_code_path, vdb_doc_path = embedder.build_vdbs( + source_infos, + config.ignore_code_embedding, + ) + + if vdb_code_path is None and not config.ignore_code_embedding: + logger.warning( + "Failed to generate code VDB for image '%s'", + base_image, + ) + + if vdb_doc_path is None: + logger.warning( + "Failed to generate documentation VDB for image '%s'", + base_image, + ) + + if not config.ignore_code_index: + code_index_path = _build_code_index(source_infos) + if code_index_path is None: + logger.warning( + "Failed to generate code index for image '%s'", + base_image, + ) + + except Exception as e: + failure_reason = f"Segmentation failed: {e}" + logger.error( + "segmentation failed for image '%s': %s", + base_image, + e, + exc_info=True, + ) + if not config.ignore_errors: + raise + + vdb_code_path = str(vdb_code_path) if vdb_code_path else None + vdb_doc_path = str(vdb_doc_path) if vdb_doc_path else None + code_index_path = str(code_index_path) if code_index_path else None + code_index_success = code_index_path is not None + + if not code_index_success: + message.failure_reason = failure_reason + + message.code_index_success = code_index_success + + state.info.vdb = AgentMorpheusInfo.VdbPaths( + code_vdb_path=vdb_code_path, + doc_vdb_path=vdb_doc_path, + code_index_path=code_index_path, + ) + + return state + + yield FunctionInfo.from_fn( + _arun, + input_schema=AgentMorpheusEngineInput, + description="Build VDBs and code indexes from cloned repositories (late phase).", + ) diff --git a/src/vuln_analysis/functions/cve_source_acquisition.py b/src/vuln_analysis/functions/cve_source_acquisition.py new file mode 100644 index 000000000..eabb2ded4 --- /dev/null +++ b/src/vuln_analysis/functions/cve_source_acquisition.py @@ -0,0 +1,242 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import json +from datetime import datetime, timezone +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +import shutil +from pathlib import Path +from pathlib import PurePath + +from exploit_iq_commons.data_models.checker_status import PackageCheckerContext, PackageCheckerStatus, PackageIdentifyResult +from exploit_iq_commons.data_models.checker_status import AcquiredArtifacts +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + +from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager, SourceRPMDownloader +from vuln_analysis.utils.package_identifier import PackageIdentifier +from vuln_analysis.tools.brew_downloader import BrewDownloader, BrewDownloaderError, resolve_brew_profile +from vuln_analysis.functions.cve_calculate_intel_score import CVECalculateIntelScoreConfig + +logger = LoggingFactory.get_agent_logger(__name__) + + +def _artifacts_from_cache( + target_dir: Path, + source_dir: Path, + log_file: Path, + metadata_file: Path, +) -> AcquiredArtifacts: + """Populate AcquiredArtifacts from an on-disk checker cache directory.""" + artifacts = AcquiredArtifacts() + artifacts.srpm_path = source_dir + if log_file.exists(): + artifacts.build_log_path = log_file + binaries_dir = target_dir / "binaries" + if binaries_dir.exists(): + artifacts.binary_rpm_path = binaries_dir + if metadata_file.exists(): + try: + metadata = json.loads(metadata_file.read_text(encoding="utf-8")) + artifacts.source_url = metadata.get("source_url") + except (json.JSONDecodeError, OSError) as e: + logger.warning("Failed to read metadata.json: %s", e) + return artifacts + + +def _update_metadata_arch(metadata_file: Path, arch: str) -> None: + """Update metadata.json to add a new arch entry to arch_artifacts.""" + if not metadata_file.exists(): + return + + try: + metadata = json.loads(metadata_file.read_text(encoding="utf-8")) + if "arch_artifacts" not in metadata: + metadata["arch_artifacts"] = {} + + metadata["arch_artifacts"][arch] = { + "build_log_downloaded_at": datetime.now(timezone.utc).isoformat(), + } + + metadata_file.write_text(json.dumps(metadata, indent=2), encoding="utf-8") + logger.info("Updated metadata with arch=%s", arch) + except (json.JSONDecodeError, OSError) as e: + logger.warning("Failed to update metadata.json with arch=%s: %s", arch, e) + + +class CVESourceAcquisitionConfig(FunctionBaseConfig, name="cve_source_acquisition"): + """ + Downloads source containers, extracts layers, and locates package sources + by purl and ecosystem. Populates the pipeline state with source paths for + downstream checker segmentation and investigation nodes. + """ + base_git_dir: str = Field( + default=".cache/am_cache/git", + description="The directory for storing pulled git repositories used for code analysis.", + ) + base_pickle_dir: str = Field( + default=".cache/am_cache/pickle", + description="The directory used for storing pickled document cache files.", + ) + base_rpm_dir: str = Field( + default=".cache/am_cache/rpms", + description="The directory used for storing rpm files.", + ) + base_checker_dir: str = Field( + default=".cache/am_cache/checker", + description="Root directory for checker-specific artifacts (extracted sources, diffs, results).", + ) + rpm_user_type: str = Field( + default="internal", + description=( + "Brew profile: internal (Red Hat VPN) or external (Fedora public Koji). " + "Overridden by RPM_USER_TYPE environment variable when set." + ), + ) + + +@register_function(config_type=CVESourceAcquisitionConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def cve_source_acquisition(config: CVESourceAcquisitionConfig, builder: Builder): + from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput + + async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + logger.info("source_acquisition: starting source code acquisition") + + rpm_manager = RPMDependencyManager.get_instance() + rpm_manager.set_rpm_cache_dir(config.base_rpm_dir) + message.info.checker_context = PackageCheckerContext() + intel_list = message.info.intel or [] + vulns = message.input.scan.vulns + + intel_by_vuln = {i.vuln_id: i for i in intel_list} + target_package = message.input.image.target_package + + identifier = PackageIdentifier( + target_package=target_package, + ) + + status = PackageCheckerStatus.OK + per_vuln_results: dict[str, PackageIdentifyResult] = {} + + intel_score_config = builder.get_function_config("cve_calculate_intel_score") + assert isinstance(intel_score_config, CVECalculateIntelScoreConfig) + + for vuln_info in vulns: + intel = intel_by_vuln.get(vuln_info.vuln_id) + + if intel_score_config.generate_intel_score and intel: + score = intel.get_intel_score() + if score < intel_score_config.intel_low_score and not intel_score_config.insist_analysis: + logger.info("Intel score %d below threshold %d for %s - skipping", + score, intel_score_config.intel_low_score, vuln_info.vuln_id) + status = PackageCheckerStatus.PKG_INTEL_LOW_SCORE + break + + status, result = identifier.identify(intel) + message.info.checker_context.identify_result = result + break + + + message.info.checker_context.status = status + if status != PackageCheckerStatus.OK: + return message + + # create identifier key + str_identifier_key = f"{target_package.name}-{target_package.version}-{target_package.release}" + identifier_key = hashlib.sha256(str_identifier_key.encode()).hexdigest()[:16] + message.info.checker_context.source_key = identifier_key + + target_dir = Path(config.base_checker_dir) / identifier_key + arch = target_package.arch + + source_dir = target_dir / "source" + log_file = target_dir / "logs" / arch / "build.log" + metadata_file = target_dir / "metadata.json" + + source_exists = source_dir.exists() and any(source_dir.iterdir()) + log_exists = log_file.exists() + + brew_profile = resolve_brew_profile(config.rpm_user_type) + brew_downloader = BrewDownloader(brew_profile, config.base_rpm_dir, str(target_dir)) + + if source_exists and (log_exists or not brew_downloader.auto_fetch_build_log): + logger.info("Full cache hit for %s (arch=%s): %s", identifier_key, arch, target_dir) + message.info.checker_context.artifacts = _artifacts_from_cache( + target_dir, source_dir, log_file, metadata_file, + ) + return message + + if source_exists and not log_exists and brew_downloader.auto_fetch_build_log: + logger.info("Partial cache hit for %s - downloading log for arch=%s", identifier_key, arch) + artifacts = _artifacts_from_cache(target_dir, source_dir, log_file, metadata_file) + + try: + brew_downloader.connect() + build = brew_downloader.search_build( + target_package.name, target_package.version, target_package.release, + ) + if build: + artifacts.build_log_path = brew_downloader.try_download_build_log(build, arch) + if artifacts.build_log_path: + _update_metadata_arch(metadata_file, arch) + logger.info("Downloaded build log for arch=%s", arch) + except BrewDownloaderError as e: + logger.warning("Failed to fetch build for build log arch=%s: %s", arch, e) + + message.info.checker_context.artifacts = artifacts + return message + + # Full cache miss - download everything + logger.info("Full cache miss for %s (arch=%s)", identifier_key, arch) + target_dir.mkdir(parents=True, exist_ok=True) + try: + brew_downloader.connect() + artifacts = brew_downloader.download_target_artifacts( + target_package.name, target_package.version, target_package.release, arch, + ) + message.info.checker_context.artifacts = artifacts + + nvr = f"{target_package.name}-{target_package.version}-{target_package.release}" + arch_entry: dict[str, str] = {} + if artifacts.build_log_path: + arch_entry["build_log_downloaded_at"] = datetime.now(timezone.utc).isoformat() + metadata = { + "source_url": artifacts.source_url, + "nvr": nvr, + "downloaded_at": datetime.now(timezone.utc).isoformat(), + "arch_artifacts": {arch: arch_entry} if arch_entry else {}, + } + metadata_file.write_text(json.dumps(metadata, indent=2), encoding="utf-8") + logger.info("Wrote metadata to %s", metadata_file) + except BrewDownloaderError as e: + logger.error("Failed to download patched SRPM: %s", e) + message.info.checker_context.status = PackageCheckerStatus.ERROR_FAILED_TO_DOWNLOAD_SRPM + return message + + + + return message + + yield FunctionInfo.from_fn( + _arun, + input_schema=AgentMorpheusEngineInput, + description="Downloads source containers and locates package sources by purl and ecosystem.", + ) diff --git a/src/vuln_analysis/functions/cve_summarize.py b/src/vuln_analysis/functions/cve_summarize.py index e5dc6f077..6c0644ac6 100644 --- a/src/vuln_analysis/functions/cve_summarize.py +++ b/src/vuln_analysis/functions/cve_summarize.py @@ -14,7 +14,6 @@ # limitations under the License. import asyncio -import logging from aiq.builder.builder import Builder from aiq.builder.framework_enum import LLMFrameworkEnum @@ -23,9 +22,11 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -from vuln_analysis.utils.string_utils import get_checklist_item_string +from exploit_iq_commons.utils.string_utils import get_checklist_item_string -logger = logging.getLogger(__name__) +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id + +logger = LoggingFactory.get_agent_logger(__name__) class CVESummarizeToolConfig(FunctionBaseConfig, name="cve_summarize"): @@ -35,6 +36,42 @@ class CVESummarizeToolConfig(FunctionBaseConfig, name="cve_summarize"): llm_name: str = Field(description="The LLM model to use") +def _all_cca_not_reachable(checklist_items: list[dict]) -> bool: + """Return True if all Call Chain Analyzer results are False (not reachable). + Returns False if no CCA calls were made (no signal to gate on). + """ + all_cca = [] + for item in checklist_items: + all_cca.extend(item.get("cca_results", [])) + return len(all_cca) > 0 and not any(all_cca) + + +def _reachability_without_cca(checklist_items: list[dict]) -> bool: + """Return True if a reachability question was asked but CCA was never called. + This happens when the agent exhausts iterations due to package name + mismatches or tool errors — absence of CCA evidence means reachability + was never verified. + """ + for item in checklist_items: + if item.get("is_reachability") == "yes" and not item.get("cca_results"): + return True + return False + + +def _package_not_found(checklist_items: list[dict]) -> bool: + """Return True if Function Locator confirmed the target package is absent. + Fires when any thread has package_validated=False and no thread has True. + """ + has_false = False + for item in checklist_items: + val = item.get("package_validated") + if val is True: + return False + if val is False: + has_false = True + return has_false + + @register_function(config_type=CVESummarizeToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def cve_summarize(config: CVESummarizeToolConfig, builder: Builder): @@ -44,19 +81,63 @@ async def cve_summarize(config: CVESummarizeToolConfig, builder: Builder): from vuln_analysis.utils.prompting import SUMMARY_PROMPT llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) - prompt = PromptTemplate(input_variables=["response"], template=SUMMARY_PROMPT) - chain = prompt | llm - async def summarize_cve(results): + async def summarize_cve(results, ecosystem: str = ""): + checklist_items = results[1] response = '\n'.join( - [get_checklist_item_string(idx + 1, checklist_item) for idx, checklist_item in enumerate(results[1])]) + [get_checklist_item_string(idx + 1, checklist_item) for idx, checklist_item in enumerate(checklist_items)]) + + if _package_not_found(checklist_items): + response = ( + "PACKAGE PRESENCE GATE: Function Locator confirmed the vulnerable package " + "is NOT present in this container. A CVE cannot be exploitable if the " + "vulnerable package does not exist. Code matches from other packages are " + "irrelevant. The verdict MUST be 'not exploitable'.\n\n" + + response + ) + logger.info("Package presence gate activated: target package not found") + elif _all_cca_not_reachable(checklist_items): + response = ( + "REACHABILITY GATE: Call Chain Analyzer confirmed the vulnerable function " + "is NOT reachable from application code in ALL reachability checks performed. " + "An unreachable function cannot be exploited regardless of other findings " + "(missing mitigations, absent protections, etc. are irrelevant if the code " + "path is never executed). The verdict MUST be 'not exploitable'.\n\n" + + response + ) + logger.info("Reachability gate activated: all CCA results are negative") + elif _reachability_without_cca(checklist_items): + response = ( + "REACHABILITY UNVERIFIED: A reachability question was investigated but " + "Call Chain Analyzer was NEVER called — reachability could not be verified. " + "Library presence in dependencies alone does NOT prove exploitability. " + "Without Call Chain Analyzer confirmation, there is NO evidence that the " + "vulnerable function is reachable from application code. Weigh this lack " + "of reachability evidence heavily when determining the verdict.\n\n" + + response + ) + logger.info("Reachability unverified gate activated: reachability question without CCA") + + summary_prompt = SUMMARY_PROMPT + if ecosystem.lower() == "java": + summary_prompt = summary_prompt.replace( + "3. FOCUS: Use only definitive checklist results; ignore inconclusive items", + "3. FOCUS: Use only definitive checklist results; ignore inconclusive items\n\n" + "4. CRITICAL: If ANY checklist item reports that Call Chain Analyzer confirmed a vulnerable\n" + " function is REACHABLE (True) from application code, the verdict MUST be \"exploitable\"\n" + " unless another item provides definitive contrary evidence (e.g., version check confirmed\n" + " the installed version is fixed). Negative results from other items using different function\n" + " names or wrong inputs do NOT override a confirmed positive reachability finding." + ) + prompt = PromptTemplate(input_variables=["response"], template=summary_prompt) + chain = prompt | llm final_summary = await chain.ainvoke({"response": response}) - return final_summary.content async def _arun(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: - - results = await asyncio.gather(*(summarize_cve(results) for results in state.checklist_results.items())) + trace_id.set(state.original_input.input.scan.id) + ecosystem = state.original_input.input.image.ecosystem.value if state.original_input and state.original_input.input.image.ecosystem else "" + results = await asyncio.gather(*(summarize_cve(results, ecosystem=ecosystem) for results in state.checklist_results.items())) state.final_summaries = dict(zip(state.checklist_results.keys(), results)) return state diff --git a/src/vuln_analysis/functions/cve_verify_vuln_package.py b/src/vuln_analysis/functions/cve_verify_vuln_package.py new file mode 100644 index 000000000..87a4c5dd5 --- /dev/null +++ b/src/vuln_analysis/functions/cve_verify_vuln_package.py @@ -0,0 +1,942 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Verify vulnerable package presence in source code dependencies before LLM analysis. + +This node runs for SOURCE analysis in FULL_PIPELINE mode. It checks if the vulnerable +package (from CVE intel) actually exists in the project's dependency lock files, +and uses LLM to verify if the installed version falls within the vulnerable range. +""" + +from pathlib import Path +from typing import Any + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import BaseModel, Field + +from exploit_iq_commons.data_models.common import AnalysisType +from exploit_iq_commons.data_models.cve_intel import CveIntel +from exploit_iq_commons.data_models.dependencies import ( + CheckedNotVulnerablePackage, + DependencyPackage, + VulnerableDependencies, + VulnerableSBOMPackage, +) +from exploit_iq_commons.utils.git_utils import get_repo_path_with_ref +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id +from nat.builder.context import Context +from exploit_iq_commons.utils.dep_tree import Ecosystem +from vuln_analysis.utils.intel_utils import extract_vuln_packages_from_intel + +logger = LoggingFactory.get_agent_logger(__name__) + +INTEL_SOURCE_RHSA = "rhsa" + +# Canonical mapping between Ecosystem enum and system_name strings. +# Single source of truth - derive inverse mapping programmatically. +ECOSYSTEM_TO_SYSTEM_NAME: dict[Ecosystem, str] = { + Ecosystem.GO: "go", + Ecosystem.PYTHON: "pypi", + Ecosystem.JAVASCRIPT: "npm", + Ecosystem.JAVA: "maven", + Ecosystem.C_CPP: "c", +} +SYSTEM_NAME_TO_ECOSYSTEM: dict[str, Ecosystem] = {v: k for k, v in ECOSYSTEM_TO_SYSTEM_NAME.items()} + +# Language ecosystems use module paths (e.g., "go.etcd.io/etcd", "requests"). +# RHSA package_state entries contain RPM package names (e.g., "etcd", "python3-requests") +# which don't map to language module paths and cause false matches. +LANGUAGE_ECOSYSTEM_SYSTEM_NAMES: frozenset[str] = frozenset({"go", "pypi", "npm", "maven"}) + +# Python version extraction patterns for different file formats. +# pyproject.toml: requires-python = ">=3.8" or "~=3.8" or "^3.8" +# Version specifiers (>=, ~=, ^, etc.) appear AFTER the opening quote. +PYPROJECT_PYTHON_VERSION_RE = r'requires-python\s*=\s*["\'][><=~^!]*\s*(\d+\.\d+(?:\.\d+)?)' +# .python-version: bare version like "3.12.0" or "3.12" (no prefix required) +PYTHON_VERSION_FILE_RE = r'^(\d+\.\d+(?:\.\d+)?)' + + +def _is_language_ecosystem(system_name: str) -> bool: + """Check if ecosystem uses language module paths rather than RPM names. + + Language ecosystems (go, python, java, javascript) should not use RHSA + package_state entries, which contain RPM package names that don't map + to language module paths. + + Args: + system_name: The ecosystem identifier (e.g., 'go', 'pypi', 'maven', 'npm') + + Returns: + True if this is a language ecosystem + """ + return system_name in LANGUAGE_ECOSYSTEM_SYSTEM_NAMES + + +def _extract_toolchain_version(repo_path: Path, ecosystem: Ecosystem) -> str | None: + """Extract the language toolchain version from project configuration files. + + Args: + repo_path: Path to the repository root + ecosystem: The ecosystem enum (GO, PYTHON, JAVA, JAVASCRIPT) + + Returns: + The toolchain version string, or None if not found + """ + import re + + if ecosystem == Ecosystem.GO: + go_mod = repo_path / "go.mod" + if go_mod.exists(): + content = go_mod.read_text(encoding="utf-8", errors="ignore") + match = re.search(r"^go\s+(\d+\.\d+(?:\.\d+)?)", content, re.MULTILINE) + if match: + return match.group(1) + + elif ecosystem == Ecosystem.PYTHON: + # Try pyproject.toml first (requires-python = ">=3.8" format) + pyproject = repo_path / "pyproject.toml" + if pyproject.exists(): + content = pyproject.read_text(encoding="utf-8", errors="ignore") + match = re.search(PYPROJECT_PYTHON_VERSION_RE, content, re.IGNORECASE) + if match: + return match.group(1) + logger.debug("pyproject.toml exists but no requires-python found") + + # Fall back to .python-version (bare version like "3.12.0") + pyver_file = repo_path / ".python-version" + if pyver_file.exists(): + content = pyver_file.read_text(encoding="utf-8", errors="ignore").strip() + match = re.match(PYTHON_VERSION_FILE_RE, content) + if match: + return match.group(1) + logger.debug(".python-version exists but no valid version found: %s", content[:50]) + + elif ecosystem == Ecosystem.JAVASCRIPT: + package_json = repo_path / "package.json" + if package_json.exists(): + import json + try: + data = json.loads(package_json.read_text(encoding="utf-8")) + engines = data.get("engines", {}) + node_ver = engines.get("node", "") + match = re.search(r"(\d+\.\d+(?:\.\d+)?)", node_ver) + if match: + return match.group(1) + except json.JSONDecodeError: + pass + + elif ecosystem == Ecosystem.JAVA: + pom_xml = repo_path / "pom.xml" + if pom_xml.exists(): + content = pom_xml.read_text(encoding="utf-8", errors="ignore") + match = re.search(r"(\d+(?:\.\d+)?)", content) + if match: + return match.group(1) + match = re.search(r"(\d+(?:\.\d+)?)", content) + if match: + return match.group(1) + + return None + + +def _get_tracer(): + try: + return Context.get() + except Exception: + return None + + +def _record_version_check_trace( + *, + cve_id: str, + package: str, + installed_version: str, + ecosystem: str, + intel_source: str | None, + compare_tech: str, + is_vulnerable: bool, + reason: str, + classification_path: str | None = None, + hard_reason: str | None = None, + gate_reason: str | None = None, + error: str | None = None, +) -> None: + """Emit a trace span for a single version comparison decision.""" + tracer = _get_tracer() + if not tracer: + return + + input_data = { + "cve_id": cve_id, + "package": package, + "installed_version": installed_version, + "ecosystem": ecosystem, + "intel_source": intel_source, + } + output: dict[str, Any] = { + "compare_tech": compare_tech, + "is_vulnerable": is_vulnerable, + "reason": reason, + } + if classification_path is not None: + output["classification_path"] = classification_path + if hard_reason is not None: + output["hard_reason"] = hard_reason + if gate_reason is not None: + output["gate_reason"] = gate_reason + if error is not None: + output["error"] = error + + with tracer.push_active_function( + "verify_vuln_package_version_check", + input_data=input_data, + ) as span: + span.set_output(output) + + +def _record_stdlib_check_trace( + *, + cve_id: str, + ecosystem: str, + toolchain_version: str, + is_vulnerable: bool, + affected_component: str, + reason: str, + error: str | None = None, +) -> None: + """Emit a trace span for a stdlib/toolchain vulnerability check.""" + tracer = _get_tracer() + if not tracer: + return + + input_data = { + "cve_id": cve_id, + "ecosystem": ecosystem, + "toolchain_version": toolchain_version, + "check_type": "stdlib_fallback", + } + output: dict[str, Any] = { + "is_vulnerable": is_vulnerable, + "affected_component": affected_component, + "reason": reason, + } + if error is not None: + output["error"] = error + + with tracer.push_active_function( + "verify_vuln_package_stdlib_check", + input_data=input_data, + ) as span: + span.set_output(output) + + +# ============================================================ +# LLM Structured Output Models +# ============================================================ + +class VersionComparisonResult(BaseModel): + """Structured output for LLM version comparison.""" + is_vulnerable: bool = Field( + description="True if the installed version falls within the vulnerable range" + ) + reason: str = Field( + description="Brief explanation of the version comparison result" + ) + + +class PackageMatchResult(BaseModel): + """Structured output for LLM package name matching.""" + is_match: bool = Field( + description="True if the SBOM package is the same or a variant of the CVE package" + ) + confidence: str = Field( + description="Confidence level: high, medium, or low" + ) + reason: str = Field( + description="Brief explanation of why the packages match or don't match" + ) + + +# ============================================================ +# CVE Processing Configuration +# ============================================================ + +from dataclasses import dataclass +from typing import Callable, Awaitable + + +@dataclass +class CveProcessingConfig: + """Configuration for CVE processing loop behavior. + + This dataclass parameterizes the differences between IMAGE and SOURCE + analysis paths, allowing a single shared processing loop. + """ + system_name: str + filter_fix_state: bool = False + use_llm_fallback: bool = False + package_matcher: Callable[[str, str, str], Awaitable[tuple[bool, str, str]]] | None = None + stdlib_fallback_checker: Callable[[CveIntel, str, str], Awaitable[tuple[bool, str, str]]] | None = None + toolchain_version: str | None = None + + +# ============================================================ +# LLM Prompts +# ============================================================ + +PACKAGE_MATCH_PROMPT = """You are a Linux/RPM package naming expert. Determine if an SBOM package matches a CVE-affected package name. + +CVE/Intel package name: {intel_package} +SBOM package name: {sbom_package} +SBOM package version: {sbom_version} + +Rules for package matching: +1. RPM packages often have suffixes like -libs, -devel, -common, -utils that indicate variants of the same base package +2. The CVE typically affects the core library, which means all variants (e.g., openssl, openssl-libs, openssl-devel) are potentially affected +3. Consider that "openssl" CVE affects "openssl-libs" (YES match) +4. Consider that "libxml2" CVE affects "libxml2-devel" (YES match) +5. Be careful: "curl" is different from "libcurl" (usually same project but may need to check) +6. Watch for false positives: "python3" is not the same as "python3-pip" + +Determine if the SBOM package is the same software (or a direct variant) as the CVE package.""" + + +# ============================================================ +# Version Comparison Prompt +# ============================================================ + +HARD_VERSION_COMPARISON_PROMPT = """You are a version comparison expert. Determine if an installed package version is vulnerable. + +CVE ID: {cve_id} +NVD/CVE description (primary ground truth): +{description} + +Package: {package} +Installed version: {installed_version} +Gate reason (why deterministic comparison was skipped): {gate_reason} + +Structured intel (reference only — may be wrong stream or incomplete): +- Start (inclusive): {version_start_incl} +- Start (exclusive): {version_start_excl} +- End (inclusive): {version_end_incl} +- End (exclusive): {version_end_excl} +- GHSA vulnerable range: {vulnerable_range} +- First patched version: {first_patched} +- RHSA fix_state: {fix_state} + +Rules: +1. Prefer the CVE description for multi-branch fixes (e.g. PostgreSQL 13.x → 13.19). +2. Do not compare across distro streams (e.g. el7 fix vs el8 install). +3. Letter-suffix upstream versions (e.g. OpenSSL 1.1.1j vs 1.1.1k) require description context. +4. RPM NEVRA/module strings may embed upstream version separately from distro tags. +5. If description clearly shows the installed version is at or past the fix, answer not vulnerable. + +Respond with whether the installed version is vulnerable and explain your reasoning.""" + + +class CVEVerifyVulnPackageConfig(FunctionBaseConfig, name="cve_verify_vuln_package"): + """ + Defines a function that verifies vulnerable package presence in source dependencies. + + For SOURCE analysis, parses lock files (go.sum, etc.) and checks if the CVE's + vulnerable package exists and if the installed version is vulnerable using LLM + comparison. + """ + skip: bool = Field( + default=False, + description="Whether to skip vulnerable package verification" + ) + base_git_dir: str = Field( + default=".cache/am_cache/git", + description="Base directory for git repos" + ) + llm_name: str = Field( + default="checklist_llm", + description="Name of the LLM to use for version comparison" + ) + + +@register_function(config_type=CVEVerifyVulnPackageConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def cve_verify_vuln_package(config: CVEVerifyVulnPackageConfig, builder: Builder): + # pylint: disable=unused-argument + from functools import partial + + from langchain_core.messages import HumanMessage + + from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput + from vuln_analysis.utils.package_matchers import get_package_matcher, PackageMatch + from vuln_analysis.utils.version_check import ( + StdlibVulnerabilityResult, + check_stdlib_vulnerability as _check_stdlib_vulnerability, + ) + + # Get LLM for version comparison, package matching, and stdlib analysis + llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + version_comparison_llm = llm.with_structured_output(VersionComparisonResult) + package_match_llm = llm.with_structured_output(PackageMatchResult) + stdlib_vulnerability_llm = llm.with_structured_output(StdlibVulnerabilityResult) + + async def check_version_vulnerable( + package: str, + installed_version: str, + version_info: dict[str, Any], + cve_intel: CveIntel, + ecosystem: str, + ) -> tuple[bool, str]: + """ + Route version check through gate: vendor shortcut, univers, or LLM. + + Returns: + Tuple of (is_vulnerable, reason) + """ + from vuln_analysis.utils.version_check import ( + VersionCheckPath, + classify_version_check, + deterministic_version_check, + get_cve_description, + ) + + description = get_cve_description(cve_intel) + classification = classify_version_check( + installed_version=installed_version, + version_info=version_info, + description=description or None, + ecosystem=ecosystem, + package_name=package, + intel=cve_intel, + ) + trace_kwargs = { + "cve_id": cve_intel.vuln_id, + "package": package, + "installed_version": installed_version, + "ecosystem": ecosystem, + "intel_source": version_info.get("source"), + "classification_path": classification.path.value, + "hard_reason": ( + classification.hard_reason.value + if classification.hard_reason is not None + else None + ), + } + + if classification.path == VersionCheckPath.VENDOR_NOT_AFFECTED: + reason = "RHSA fix_state: package not affected on this distro" + _record_version_check_trace( + compare_tech="vendor_not_affected", + is_vulnerable=False, + reason=reason, + **trace_kwargs, + ) + return False, reason + + if classification.path == VersionCheckPath.EASY: + try: + is_vulnerable, reason = deterministic_version_check( + installed_version=installed_version, + version_info=version_info, + ecosystem=ecosystem, + ) + logger.info( + " Easy path verdict: vulnerable=%s, reason=%s", + is_vulnerable, + reason, + ) + _record_version_check_trace( + compare_tech="univers", + is_vulnerable=is_vulnerable, + reason=reason, + **trace_kwargs, + ) + return is_vulnerable, reason + except Exception as e: + logger.warning( + "Deterministic version comparison failed for %s: %s", + package, + e, + exc_info=True, + ) + + gate_reason = ( + classification.hard_reason.value + if classification.hard_reason is not None + else "unknown" + ) + prompt = HARD_VERSION_COMPARISON_PROMPT.format( + cve_id=cve_intel.vuln_id, + description=description or "N/A", + package=package, + installed_version=installed_version, + gate_reason=gate_reason, + version_start_incl=version_info.get("version_start_incl") or "N/A", + version_start_excl=version_info.get("version_start_excl") or "N/A", + version_end_incl=version_info.get("version_end_incl") or "N/A", + version_end_excl=version_info.get("version_end_excl") or "N/A", + vulnerable_range=version_info.get("vulnerable_range") or "N/A", + first_patched=version_info.get("first_patched") or "N/A", + fix_state=version_info.get("fix_state") or "N/A", + ) + + try: + result: VersionComparisonResult = await version_comparison_llm.ainvoke( + [HumanMessage(content=prompt)] + ) + logger.info( + " Hard path verdict: vulnerable=%s, reason=%s", + result.is_vulnerable, + result.reason, + ) + _record_version_check_trace( + compare_tech="llm", + is_vulnerable=result.is_vulnerable, + reason=result.reason, + gate_reason=gate_reason, + **trace_kwargs, + ) + return result.is_vulnerable, result.reason + except Exception as e: + logger.warning( + "LLM version comparison failed for %s: %s", + package, + e, + exc_info=True, + ) + reason = f"LLM comparison failed, assuming vulnerable: {e}" + _record_version_check_trace( + compare_tech="llm", + is_vulnerable=True, + reason=reason, + gate_reason=gate_reason, + error=str(e), + **trace_kwargs, + ) + return True, reason + + async def check_package_match( + intel_package: str, + sbom_package: str, + sbom_version: str + ) -> tuple[bool, str, str]: + """ + Use LLM to check if an SBOM package matches a CVE-affected package name. + + This is used as a fallback when static matching rules don't find a match. + + Returns: + Tuple of (is_match, confidence, reason) + """ + prompt = PACKAGE_MATCH_PROMPT.format( + intel_package=intel_package, + sbom_package=sbom_package, + sbom_version=sbom_version, + ) + + try: + result: PackageMatchResult = await package_match_llm.ainvoke( + [HumanMessage(content=prompt)] + ) + return result.is_match, result.confidence, result.reason + except Exception as e: + logger.warning("LLM package match failed for %s vs %s: %s", + intel_package, sbom_package, e) + return False, "error", f"LLM match failed: {e}" + + async def _invoke_stdlib_llm(prompt: str) -> StdlibVulnerabilityResult: + """Adapter bridging LangChain LLM to StdlibLLMChecker signature.""" + return await stdlib_vulnerability_llm.ainvoke([HumanMessage(content=prompt)]) + + async def _process_cve_intel_loop( + intel_list: list[CveIntel], + matcher, + dependencies: dict[str, str], + version_checker, + proc_config: CveProcessingConfig, + ) -> list[VulnerableDependencies]: + """ + Shared CVE processing loop for both IMAGE and SOURCE analysis paths. + + This function consolidates the duplicated logic for: + 1. Extracting vulnerable packages from intel + 2. Matching against dependencies (with optional LLM fallback) + 3. Checking version vulnerability (with has_version_info guard) + 4. Building VulnerableDependencies result + + Args: + intel_list: List of CveIntel objects to process + matcher: Ecosystem-specific package matcher + dependencies: Dict mapping package names to versions + version_checker: Async function to check version vulnerability + proc_config: Configuration for processing behavior + + Returns: + List of VulnerableDependencies for each CVE + """ + vulnerable_dependencies: list[VulnerableDependencies] = [] + + # Check once if this is a language ecosystem. + # Language ecosystems should skip RHSA package_state entries (RPM names). + is_language_ecosystem = _is_language_ecosystem(proc_config.system_name) + + for intel in intel_list: + vuln_packages = extract_vuln_packages_from_intel(intel) + logger.info("CVE %s: %d vulnerable package(s) in intel", + intel.vuln_id, len(vuln_packages)) + + vuln_sbom_packages: list[VulnerableSBOMPackage] = [] + checked_not_vulnerable: list[CheckedNotVulnerablePackage] = [] + seen_not_vulnerable: set[tuple[str, str]] = set() + intel_sources: list[str] = [vp["source"] for vp in vuln_packages] + + found_vulnerable = False + for vp in vuln_packages: + if found_vulnerable: + break + + pkg_name = vp["package"] + + # Skip ALL RHSA entries for language ecosystems. + # RHSA contains RPM package names (e.g., "etcd", "golang") which don't + # map to language module paths (e.g., "go.etcd.io/etcd"). Language + # ecosystems should rely on GHSA (proper module paths) and NVD instead. + if vp["source"] == INTEL_SOURCE_RHSA and is_language_ecosystem: + logger.info( + " SKIP [%s]: %s - RHSA not applicable for %s ecosystem", + vp["source"], pkg_name, proc_config.system_name + ) + continue + + # Optionally filter by fix_state (IMAGE path uses this) + if proc_config.filter_fix_state: + fix_state = vp.get("fix_state") + if fix_state and fix_state.lower() in ("not affected", "will not fix", "out of support scope"): + logger.info(" SKIP [%s]: %s fix_state=%s", + vp["source"], pkg_name, fix_state) + continue + + # Use ecosystem-specific matcher to find packages + matches: list[PackageMatch] = matcher.find_package(pkg_name, dependencies) + + # Optionally use LLM fallback when static matching fails (IMAGE path uses this) + if not matches and proc_config.use_llm_fallback and proc_config.package_matcher: + logger.info(" Static match failed for %s, trying LLM fallback...", pkg_name) + candidates = [] + pkg_lower = pkg_name.lower() + for dep_name, version in dependencies.items(): + dep_lower = dep_name.lower() + if (pkg_lower in dep_lower or dep_lower in pkg_lower or + any(part in dep_lower for part in pkg_lower.split("-") if len(part) > 3)): + candidates.append((dep_name, version)) + if len(candidates) >= 5: + break + + for dep_name, version in candidates: + is_match, confidence, reason = await proc_config.package_matcher( + intel_package=pkg_name, + sbom_package=dep_name, + sbom_version=version + ) + if is_match and confidence in ("high", "medium"): + logger.info(" LLM MATCH [%s]: %s @ %s (confidence=%s)", + vp["source"], dep_name, version, confidence) + matches.append(PackageMatch( + found=True, + dep_name=dep_name, + installed_version=version, + intel_package=pkg_name, + )) + + if not matches: + logger.info(" NO MATCH [%s]: %s not found in dependencies", + vp["source"], pkg_name) + continue + + # Process each matched package + for match in matches: + logger.info(" MATCH [%s]: %s @ %s", + vp["source"], match.dep_name, match.installed_version) + + is_vulnerable, reason = await version_checker( + package=match.dep_name or pkg_name, + installed_version=match.installed_version or "unknown", + version_info=vp, + cve_intel=intel, + ecosystem=proc_config.system_name, + ) + + if is_vulnerable: + vuln_sbom_packages.append(VulnerableSBOMPackage( + name=match.dep_name or pkg_name, + version=match.installed_version or "unknown", + vulnerable_dependency_package=DependencyPackage( + system=proc_config.system_name, + name=match.dep_name or pkg_name, + version=match.installed_version, + relation="SELF" + ) + )) + found_vulnerable = True + break + else: + pkg_key = (match.dep_name or pkg_name, match.installed_version or "unknown") + if pkg_key not in seen_not_vulnerable: + seen_not_vulnerable.add(pkg_key) + checked_not_vulnerable.append(CheckedNotVulnerablePackage( + name=pkg_key[0], + version=pkg_key[1], + reason=reason + )) + + # Stdlib/toolchain fallback: when no packages matched and it's a language ecosystem + no_packages_found = not vuln_sbom_packages and not checked_not_vulnerable + if ( + no_packages_found + and is_language_ecosystem + and proc_config.stdlib_fallback_checker is not None + ): + logger.info( + "CVE %s: No packages matched, checking stdlib/toolchain vulnerability", + intel.vuln_id, + ) + toolchain_ver = proc_config.toolchain_version or "unknown" + is_vulnerable, component, reason = await proc_config.stdlib_fallback_checker( + intel, proc_config.system_name, toolchain_ver + ) + + # Record trace for stdlib vulnerability check + _record_stdlib_check_trace( + cve_id=intel.vuln_id, + ecosystem=proc_config.system_name, + toolchain_version=toolchain_ver, + is_vulnerable=is_vulnerable, + affected_component=component, + reason=reason, + ) + + if is_vulnerable: + vuln_sbom_packages.append(VulnerableSBOMPackage( + name=f"stdlib:{component}", + version=toolchain_ver, + vulnerable_dependency_package=DependencyPackage( + system=proc_config.system_name, + name=f"stdlib:{component}", + version=toolchain_ver, + relation="SELF" + ) + )) + intel_sources.append("llm_stdlib_check") + else: + checked_not_vulnerable.append(CheckedNotVulnerablePackage( + name=f"stdlib:{component}", + version=toolchain_ver, + reason=reason + )) + intel_sources.append("llm_stdlib_check") + + # Create VulnerableDependencies entry for this CVE + vulnerable_dependencies.append(VulnerableDependencies( + vuln_id=intel.vuln_id, + vuln_package_intel_sources=list(set(intel_sources)), + vulnerable_sbom_packages=vuln_sbom_packages, + checked_not_vulnerable=checked_not_vulnerable + )) + + # Log summary for this CVE + if vuln_sbom_packages: + logger.info("CVE %s: %d vulnerable package(s) confirmed", + intel.vuln_id, len(vuln_sbom_packages)) + else: + logger.info("CVE %s: No vulnerable packages confirmed", + intel.vuln_id) + + return vulnerable_dependencies + + async def _handle_cpp_image_analysis( + message: AgentMorpheusEngineInput, + version_checker, + package_matcher, + ) -> AgentMorpheusEngineInput: + """ + Handle C/C++ container IMAGE analysis using SBOM matching and LLM version checking. + + This function: + 1. Gets SBOM packages from RPMDependencyManager + 2. Uses shared _process_cve_intel_loop with IMAGE-specific config + 3. Builds VulnerableDependencies result + """ + from vuln_analysis.utils.package_matchers import CppPackageMatcher + + logger.info("C/C++ IMAGE analysis: using SBOM + LLM version checking") + + # Get SBOM dependencies + matcher = CppPackageMatcher() + dependencies = matcher.get_sbom_dependencies() + logger.info("SBOM contains %d packages", len(dependencies)) + + if not dependencies: + logger.warning("SBOM is empty. Cannot verify vulnerable packages.") + logger.info("=== verify_vuln_package END (empty SBOM) ===") + return message + + # Check if we have intel to process + if not message.info.intel: + logger.warning("No intel available for CVE matching.") + logger.info("=== verify_vuln_package END (no intel) ===") + return message + + # Use shared processing loop with IMAGE-specific config + proc_config = CveProcessingConfig( + system_name="rpm", + filter_fix_state=True, + use_llm_fallback=True, + package_matcher=package_matcher, + ) + vulnerable_dependencies = await _process_cve_intel_loop( + message.info.intel, matcher, dependencies, + version_checker, proc_config + ) + + # Update message with vulnerable_dependencies + message.info.vulnerable_dependencies = vulnerable_dependencies + + logger.info("=== verify_vuln_package END (C/C++ IMAGE - %d CVEs processed) ===", + len(vulnerable_dependencies)) + return message + + async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + from vuln_analysis.utils.package_matchers import CppPackageMatcher + from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager + + trace_id.set(message.input.scan.id) + + logger.info("=== verify_vuln_package START ===") + logger.info("analysis_type: %s", message.input.image.analysis_type) + logger.info("ecosystem: %s", message.input.image.ecosystem) + logger.info("vulns count: %d", len(message.input.scan.vulns)) + logger.info("intel count: %d", len(message.info.intel) if message.info.intel else 0) + + if config.skip: + logger.info("config.skip is True. Skipping verify_vuln_package.") + logger.info("=== verify_vuln_package END (passthrough) ===") + return message + + # Get ecosystem + ecosystem = message.input.image.ecosystem + + # Route based on analysis type and ecosystem + if message.input.image.analysis_type == AnalysisType.IMAGE and ecosystem == Ecosystem.C_CPP: + # C/C++ container analysis: use SBOM + LLM version checking + return await _handle_cpp_image_analysis( + message, check_version_vulnerable, check_package_match + ) + elif message.input.image.analysis_type == AnalysisType.SOURCE: + # SOURCE analysis: use lock files (Go/Python/Java) + pass # Continue to existing lock file logic below + else: + # Other analysis types: passthrough + logger.info("Non-SOURCE/non-C++ IMAGE analysis. Skipping.") + logger.info("=== verify_vuln_package END (passthrough) ===") + return message + + # --- SOURCE analysis logic (Go/Python/Java with lock files) --- + + if ecosystem is None: + logger.warning("Ecosystem not set. Cannot parse lock files.") + logger.info("=== verify_vuln_package END (no ecosystem) ===") + return message + + # Get package matcher for ecosystem + try: + matcher = get_package_matcher(ecosystem) + logger.info("Using %s for ecosystem %s", type(matcher).__name__, ecosystem) + except NotImplementedError as e: + logger.info("Ecosystem %s not yet supported: %s", ecosystem, e) + logger.info("=== verify_vuln_package END (unsupported ecosystem) ===") + return message + + # Get repo path from source_info + source_info = message.input.image.source_info + if not source_info: + logger.warning("No source_info available. Cannot locate repo.") + logger.info("=== verify_vuln_package END (no source_info) ===") + return message + + # Use first source_info entry for repo path + first_source = source_info[0] + repo_path = get_repo_path_with_ref( + Path(config.base_git_dir), first_source.git_repo, first_source.ref + ) + + logger.info("Repo path: %s", repo_path) + + # Parse dependencies using ecosystem-specific matcher + try: + dependencies = matcher.parse_dependencies( + repo_path, git_repo=first_source.git_repo, ref=first_source.ref + ) + logger.info("Parsed %d packages from lock file", len(dependencies)) + except FileNotFoundError as e: + logger.warning("Lock file not found: %s", e) + logger.info("=== verify_vuln_package END (no lock file) ===") + return message + except NotImplementedError as e: + logger.warning("Parser not implemented: %s", e) + logger.info("=== verify_vuln_package END (parser not implemented) ===") + return message + + # For each CVE, extract vulnerable packages and check against dependencies + if not message.info.intel: + logger.warning("No intel available for CVE matching.") + logger.info("=== verify_vuln_package END (no intel) ===") + return message + + # Map ecosystem to system name for DependencyPackage + system_name = ECOSYSTEM_TO_SYSTEM_NAME.get(ecosystem, ecosystem.value) + + # Extract toolchain version for stdlib fallback + toolchain_version = _extract_toolchain_version(repo_path, ecosystem) + if toolchain_version: + logger.info("Detected toolchain version: %s", toolchain_version) + + # Use shared processing loop with SOURCE-specific config + # Bind LLM adapter to stdlib checker (dependency injection) + stdlib_checker_with_llm = partial(_check_stdlib_vulnerability, llm_checker=_invoke_stdlib_llm) + proc_config = CveProcessingConfig( + system_name=system_name, + filter_fix_state=False, + use_llm_fallback=False, + stdlib_fallback_checker=stdlib_checker_with_llm, + toolchain_version=toolchain_version, + ) + vulnerable_dependencies = await _process_cve_intel_loop( + message.info.intel, matcher, dependencies, + check_version_vulnerable, proc_config + ) + + # Update message with vulnerable_dependencies + message.info.vulnerable_dependencies = vulnerable_dependencies + + logger.info("=== verify_vuln_package END (SOURCE - %d CVEs processed) ===", + len(vulnerable_dependencies)) + return message + + yield FunctionInfo.from_fn( + _arun, + input_schema=AgentMorpheusEngineInput, + description="Verifies vulnerable package presence in source dependencies before LLM analysis." + ) diff --git a/src/vuln_analysis/functions/dispatcher.py b/src/vuln_analysis/functions/dispatcher.py new file mode 100644 index 000000000..0ab1240b1 --- /dev/null +++ b/src/vuln_analysis/functions/dispatcher.py @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Literal + +from langchain_core.messages import HumanMessage +from pydantic import BaseModel, Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from nat.builder.context import Context + +logger = LoggingFactory.get_agent_logger(__name__) +AGENT_TRACER = Context.get() + + +class QuestionRouting(BaseModel): + """Structured output for LLM-based question routing classification. + + The dispatcher classifies each checklist question into one of two agent types: + - reachability: call chain tracing (is vulnerable code reachable from app code?) + - code_understanding: configuration/version/presence/usage investigation + """ + agent_type: Literal["reachability", "code_understanding"] = Field( + description="Route to 'reachability' if the question asks about call chains, " + "code paths, whether vulnerable code is used/called/reachable, " + "or whether untrusted data can reach a function. " + "Route to 'code_understanding' for configuration, version, " + "or application-level settings questions." + ) + reason: str = Field( + description="One-sentence justification for the routing decision." + ) + + +ROUTING_PROMPT_TEMPLATE = """You are classifying a CVE investigation question to route it to the correct sub-agent. + +Two sub-agents are available: +- **reachability**: Traces call chains and code paths. Use when the question asks whether vulnerable code is CALLED, USED, or REACHABLE from application code, or whether untrusted data can reach a specific function. IMPORTANT: If the question asks whether a class/function/module from the VULNERABLE PACKAGE (listed in the context below) is used, imported, or called — route to reachability. The reachability agent has Call Chain Analyzer and Function Locator to verify actual code-level usage and call paths, which is needed to distinguish "imported but never called on vulnerable path" from "actively used." +- **code_understanding**: Investigates configuration, version, presence, and general application behavior. Use when the question asks about application-level configuration (e.g., XML parsing settings, TLS options), environment setup, or properties — things that do NOT require tracing whether a specific vulnerable function is called. + +Context (CVE / vulnerable packages): +{context_block} + +Question: {question} + +Examples: +- "Is the vulnerable function XStream.fromXML() called from application code?" → reachability +- "Is the application configured to use the affected XML parser?" → code_understanding +- "Can untrusted data reach BeanUtils.populate() through the call chain?" → reachability +- "Is the vulnerable version of commons-beanutils installed?" → code_understanding +- "Is the function parseXML() reachable from any HTTP handler?" → reachability +- "Does the application enable external entity processing in its XML configuration?" → code_understanding +- "Is SslHandler used in the application?" → reachability (SslHandler is from the vulnerable package) +- "Is HttpPostStandardRequestDecoder.offer() called by application code?" → reachability +- "Is the vulnerable newTransformer() method invoked anywhere in the application's XSLT processing?" → reachability +- "Does application code call deserialize() or readObject() on the affected library?" → reachability +- "Does the code sanitize input against path traversal characters and command injection metacharacters?" → code_understanding +- "Are there input validation mechanisms to prevent malicious Markdown input from reaching the paragraph function?" → code_understanding +- "Can a single request force the application server to load unbounded data into memory?" → code_understanding +- "Can malformed input cause an unhandled exception that crashes or restarts the process?" → code_understanding + +Classify the question above.""" + + +def build_routing_prompt(context_block: str, question: str) -> str: + """Format the routing prompt template with CVE context and question text.""" + return ROUTING_PROMPT_TEMPLATE.format( + context_block=context_block, + question=question, + ) + + +async def dispatch_question( + routing_llm, + question: str, + context_block: str, +) -> QuestionRouting: + """Classify a checklist question and return routing decision. + + Uses the routing LLM (with structured output) to decide whether the question + should go to the reachability agent (call chain tracing) or the code understanding + agent (configuration/presence/usage investigation). + """ + prompt = build_routing_prompt(context_block, question) + with AGENT_TRACER.push_active_function("dispatch_question", input_data=f"question:{question[:80]}") as span: + logger.debug("Dispatching question to routing LLM: %.80s", question) + result: QuestionRouting = await routing_llm.ainvoke([HumanMessage(content=prompt)]) + logger.info( + "Question routed to '%s' (reason: %s): %.80s", + result.agent_type, + result.reason, + question, + ) + span.set_output({"agent_type": result.agent_type, "reason": result.reason}) + return result \ No newline at end of file diff --git a/src/vuln_analysis/functions/health_endpoint.py b/src/vuln_analysis/functions/health_endpoint.py new file mode 100644 index 000000000..6697354bd --- /dev/null +++ b/src/vuln_analysis/functions/health_endpoint.py @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pydantic import BaseModel + +from aiq.builder.builder import Builder +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig + + +class HealthCheckTool(FunctionBaseConfig, name="health_check"): + pass + + +class HealthStatusResponse(BaseModel): + status: str + + +@register_function(config_type=HealthCheckTool) +async def health_check(config: HealthCheckTool, builder: Builder): + """ + Registers a health-check function used to verify that the agent runtime + is operational and capable of handling requests. + """ + + async def _health(request: None = None) -> HealthStatusResponse: + """ + Returns operational status data for the agent runtime. + + Returns: + HealthStatusResponse: Structured health-status payload. + """ + return HealthStatusResponse(status="ok") + + yield FunctionInfo.from_fn( + _health, description="Health-status endpoint returning operational state." + ) diff --git a/src/vuln_analysis/functions/reachability_agent.py b/src/vuln_analysis/functions/reachability_agent.py new file mode 100644 index 000000000..74aa0bc41 --- /dev/null +++ b/src/vuln_analysis/functions/reachability_agent.py @@ -0,0 +1,312 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from langchain_core.messages import HumanMessage, SystemMessage, AIMessage + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from nat.builder.context import Context +from vuln_analysis.functions.agent_registry import register_agent +from vuln_analysis.functions.base_graph_agent import BaseGraphAgent, _is_tool_available +from vuln_analysis.functions.react_internals import ( + AgentState, + Classification, + Observation, + Thought, + ReachabilityRulesTracker, + build_reachability_system_prompt, + build_classification_prompt, + FORCED_FINISH_PROMPT, + REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS_GO, + REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS_PYTHON, + REACHABILITY_AGENT_NON_REACH_SYS_PROMPT, + REACHABILITY_AGENT_NON_REACH_THOUGHT_INSTRUCTIONS, +) +from vuln_analysis.runtime_context import ctx_state +from vuln_analysis.tools.tool_names import ToolNames +from vuln_analysis.tools.transitive_code_search import package_name_from_locator_query +from vuln_analysis.utils.intel_utils import build_critical_context, enrich_go_candidates +from vuln_analysis.utils.prompting import build_tool_descriptions +from vuln_analysis.utils.prompt_factory import ( + TOOL_SELECTION_STRATEGY, + TOOL_SELECTION_STRATEGY_NON_REACHABILITY, + FEW_SHOT_EXAMPLES, + FL_INPUT_FORMATS, +) +logger = LoggingFactory.get_agent_logger(__name__) +AGENT_TRACER = Context.get() + + +@register_agent("reachability") +class ReachabilityAgent(BaseGraphAgent): + + def __init__(self, tools, llm, config): + super().__init__(tools, llm, config) + self._classification_llm = llm.with_structured_output(Classification) + + @property + def agent_type(self) -> str: + return "reachability" + + _REACHABILITY_TOOLS = frozenset({ + ToolNames.FUNCTION_LOCATOR, + ToolNames.CALL_CHAIN_ANALYZER, + ToolNames.FUNCTION_LIBRARY_VERSION_FINDER, + ToolNames.FUNCTION_CALLER_FINDER, + ToolNames.CODE_KEYWORD_SEARCH, + ToolNames.CVE_WEB_SEARCH, + ToolNames.CODE_SEMANTIC_SEARCH, + ToolNames.DOCS_SEMANTIC_SEARCH, + }) + + @staticmethod + def get_tools(builder, config, state) -> list: + all_tools = BaseGraphAgent._load_all_tools(builder, config) + return [ + t for t in all_tools + if t.name in ReachabilityAgent._REACHABILITY_TOOLS + and _is_tool_available(t.name, config, state) + ] + + @staticmethod + def create_rules_tracker() -> ReachabilityRulesTracker: + return ReachabilityRulesTracker() + + def should_truncate_tool_output(self, state: AgentState, tool_used: str) -> bool: + return state.get("ecosystem", "").lower() == "java" + + def _build_tool_guidance_for_ecosystem(self, ecosystem: str, available_tools: list, + is_reachability: str = "yes") -> tuple[str, str]: + filtered_tools = [ + t for t in available_tools + if (t.name != ToolNames.FUNCTION_CALLER_FINDER or ecosystem == "go") and + (t.name != ToolNames.FUNCTION_LIBRARY_VERSION_FINDER or ecosystem == "java") + ] + list_of_tool_names = [t.name for t in filtered_tools] + lang = ecosystem.lower() if ecosystem else "" + fl_format = FL_INPUT_FORMATS.get(lang, "Input: 'package_name,function_name'.") + list_of_tool_descriptions = [ + t.name + ": " + t.description.replace("{fl_input_format}", fl_format) + for t in filtered_tools + ] + + strategy = TOOL_SELECTION_STRATEGY if is_reachability == "yes" else TOOL_SELECTION_STRATEGY_NON_REACHABILITY + if lang in strategy: + tool_guidance_local = strategy[lang] + if lang == "java": + tool_guidance_local += ( + " Use Function Library Version Finder to verify the installed version of a library " + "before concluding exploitability (e.g., input 'commons-beanutils')." + ) + if is_reachability == "yes": + hint = FEW_SHOT_EXAMPLES.get(lang, "") + if hint: + tool_guidance_local += f"\nHint: {hint}" + else: + tool_guidance_list_local = build_tool_descriptions(list_of_tool_names) + tool_guidance_local = "\n".join(tool_guidance_list_local) + + descriptions_local = "\n".join(list_of_tool_descriptions) + return tool_guidance_local, descriptions_local + + async def pre_process_node(self, state: AgentState) -> AgentState: + workflow_state = ctx_state.get() + ecosystem = workflow_state.original_input.input.image.ecosystem.value if workflow_state.original_input.input.image.ecosystem else "" + with AGENT_TRACER.push_active_function("pre_process node", input_data=f"ecosystem:{ecosystem}") as span: + try: + precomputed = state.get("precomputed_intel") + if precomputed is not None: + critical_context = list(precomputed[0]) + candidate_packages = [dict(p) for p in precomputed[1]] + vulnerable_functions = list(precomputed[2]) + else: + critical_context, candidate_packages, vulnerable_functions = build_critical_context(workflow_state.cve_intel) + vulnerable_functions_set = set(vulnerable_functions) + + already_enriched = any("Go vuln DB hint" in c for c in critical_context) + if ecosystem == "go" and not already_enriched: + candidate_packages, vulnerable_functions = await enrich_go_candidates( + workflow_state.cve_intel, + workflow_state.original_input.input.image.source_info, + critical_context, + candidate_packages, + vulnerable_functions_set, + ) + + critical_context, selected_package = await self._select_package( + ecosystem, candidate_packages, critical_context, workflow_state, + ) + app_package = selected_package + + critical_context.append( + "TASK: Investigate usage and reachability of the vulnerable function/module in the container. " + "Use the vulnerable module name from GHSA as primary investigation target." + ) + + question = state.get("input") or "" + context_block = "\n".join(critical_context) + classification_prompt = build_classification_prompt(context_block, question) + classification_result: Classification = await self._classification_llm.ainvoke([HumanMessage(content=classification_prompt)]) + span.set_output({ + "critical_context": critical_context, + "candidate_packages": candidate_packages, + "selected_package": selected_package, + "app_package": app_package if selected_package else None, + "reachability_question": classification_result.is_reachability, + }) + + is_reachability = classification_result.is_reachability + + if is_reachability == "yes": + tool_guidance_local, descriptions_local = self._build_tool_guidance_for_ecosystem(ecosystem, self.tools) + ecosystem_instructions = {} + if ecosystem == "go": + ecosystem_instructions = {"instructions": REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS_GO} + elif ecosystem == "python": + ecosystem_instructions = {"instructions": REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS_PYTHON} + runtime_prompt = build_reachability_system_prompt(descriptions_local, tool_guidance_local, **ecosystem_instructions) + active_tool_names = [t.name for t in self.tools] + else: + reachability_tool_names = {ToolNames.CALL_CHAIN_ANALYZER, ToolNames.FUNCTION_CALLER_FINDER} + non_reach_tools = [t for t in self.tools if t.name not in reachability_tool_names] + tool_guidance_local, descriptions_local = self._build_tool_guidance_for_ecosystem(ecosystem, non_reach_tools, is_reachability="no") + runtime_prompt = build_reachability_system_prompt( + descriptions_local, tool_guidance_local, + instructions=REACHABILITY_AGENT_NON_REACH_THOUGHT_INSTRUCTIONS, + sys_prompt=REACHABILITY_AGENT_NON_REACH_SYS_PROMPT, + ) + active_tool_names = [t.name for t in non_reach_tools] + logger.info("Non-reachability question detected; removed reachability tools from prompt") + + rules_tracker = state.get("rules_tracker") + app_package = app_package if selected_package else None + rules_tracker.set_ecosystem(ecosystem) + rules_tracker.set_target_package(app_package) + rules_tracker.set_allowed_tools(active_tool_names) + if is_reachability == "yes": + rules_tracker.set_target_functions(vulnerable_functions) + return { + "ecosystem": ecosystem, + "runtime_prompt": runtime_prompt, + "is_reachability": is_reachability, + "observation": Observation(memory=[], results=[]), + "critical_context": critical_context, + "app_package": app_package, + } + except Exception as e: + logger.exception("pre_process_node failed") + span.set_output({"error": str(e), "exception_type": type(e).__name__}) + raise + + def check_finish_allowed(self, state: AgentState) -> tuple[bool, str]: + if state.get("is_reachability") != "yes": + return True, "" + rules_tracker = state.get("rules_tracker") + cca_results = state.get("cca_results", []) + return rules_tracker.check_finish_allowed(cca_results) + + async def forced_finish_node(self, state: AgentState) -> AgentState: + """Override forced_finish to inject a no-CCA warning for reachability questions. + + When the reachability agent exhausts all iterations without ever calling CCA, + the LLM has no reachability evidence. Without this override the LLM can + hallucinate exploitability based on library presence alone. The injected + prompt steers it toward 'insufficient evidence / not exploitable'. + """ + cca_results = state.get("cca_results", []) + is_reachability = state.get("is_reachability") + if is_reachability == "yes" and not cca_results: + step_num = state.get("step", 0) + with AGENT_TRACER.push_active_function( + f"{self.agent_type}_forced_finish", input_data=f"step:{step_num}" + ) as span: + try: + active_prompt = state.get("runtime_prompt") + messages = [SystemMessage(content=active_prompt)] + state["messages"] + crit_context = state.get("critical_context",None) + context_block = self._build_observation_context(state.get("observation", None), crit_context) + if context_block: + messages.append(SystemMessage(content=context_block)) + question = state.get("input", "") + no_cca_prompt = ( + (f"QUESTION: {question}\n\n" if question else "") + + FORCED_FINISH_PROMPT + "\n\n" + "CRITICAL: Call Chain Analyzer was NEVER called during this investigation. " + "Without Call Chain Analyzer verification, there is NO evidence that the " + "vulnerable function is reachable from application code. Library presence " + "in dependencies alone does NOT constitute exploitability. " + "You MUST conclude that there is insufficient evidence to confirm " + "exploitability — the function was NOT confirmed reachable." + ) + messages.append(HumanMessage(content=no_cca_prompt)) + keep_tail = 2 if context_block else 1 + self._prune_messages_to_fit(messages, keep_tail=keep_tail, step_num=step_num, caller="reachability_forced_finish") + response: Thought = await self.thought_llm.ainvoke(messages) + if response.mode == "finish" and response.final_answer: + final_answer = response.final_answer + else: + final_answer = ( + "Insufficient evidence: Call Chain Analyzer was never invoked. " + "Cannot confirm the vulnerable function is reachable from application code." + ) + response = Thought( + thought="Max steps exceeded without CCA verification", + mode="finish", + actions=None, + final_answer=final_answer, + ) + ai_message = AIMessage(content=final_answer) + logger.info("Reachability forced_finish: no CCA calls, injected no-CCA warning") + span.set_output({"final_answer_length": len(final_answer), "step": step_num, "no_cca_warning": True}) + return { + "messages": [ai_message], + "thought": response, + "step": step_num, + "max_steps": state.get("max_steps", self.config.max_iterations), + "observation": state.get("observation", None), + "output": final_answer, + } + except Exception as e: + logger.exception("%s forced_finish_node failed at step %d", self.agent_type, step_num) + span.set_output({"error": str(e), "exception_type": type(e).__name__, "step": step_num}) + raise + return await super().forced_finish_node(state) + + def post_observation(self, state: AgentState, tool_used: str, + tool_output: str, tool_input_detail: str) -> dict: + extra = {} + cca_results = list(state.get("cca_results", [])) + if tool_used == ToolNames.CALL_CHAIN_ANALYZER: + stripped = tool_output.strip().lstrip("([") + first_token = stripped.split(",", 1)[0].strip().lower() + if first_token == "true": + cca_results.append(True) + elif first_token == "false": + cca_results.append(False) + extra["cca_results"] = cca_results + + package_validated = state.get("package_validated") + rules_tracker = state.get("rules_tracker") + if tool_used == ToolNames.FUNCTION_LOCATOR and state.get("is_reachability") == "yes": + input_pkg = package_name_from_locator_query(tool_input_detail) + target_pkg = (state.get("app_package") or "").strip().lower() + if input_pkg and "Package is valid" in tool_output: + rules_tracker.add_validated_package(input_pkg) + if target_pkg and input_pkg == target_pkg: + package_validated = True + elif input_pkg and "Package is not valid" in tool_output: + if target_pkg and input_pkg == target_pkg and package_validated is None: + package_validated = False + extra["package_validated"] = package_validated + return extra diff --git a/src/vuln_analysis/functions/react_internals.py b/src/vuln_analysis/functions/react_internals.py new file mode 100644 index 000000000..4a7c74913 --- /dev/null +++ b/src/vuln_analysis/functions/react_internals.py @@ -0,0 +1,830 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from langchain_core.messages import SystemMessage +from openai import LengthFinishReasonError +from pydantic import BaseModel, Field +from typing import Any +from typing import Literal +from langgraph.graph import MessagesState + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.tools.tool_names import ToolNames + +logger = LoggingFactory.get_agent_logger(__name__) + +# ---- Pydantic Schemas ---- # + +class ToolCall(BaseModel): + tool: str = Field(description="Exact tool name from AVAILABLE_TOOLS") + package_name: str | None = Field( + default=None, + description="Package/module name. REQUIRED when using Function Locator, Function Caller Finder, or Call Chain Analyzer. E.g. libpq, urllib, github.com/org/pkg" + ) + function_name: str | None = Field( + default=None, + description="Function or method name with optional args. REQUIRED with package_name for code path tools. E.g. PQescapeLiteral(), parse(), errors.New(\"x\")" + ) + # For search tools: Code Keyword Search, Code Semantic Search, Docs Semantic Search, CVE Web Search + query: str | None = Field( + default=None, + description="Search query. Use for search tools when package_name/function_name don't apply" + ) + # Fallback for when LLM uses tool_input for simple query-only tools + tool_input: str | None = Field( + default=None, + description="Legacy/fallback input. Prefer package_name+function_name or query." + ) + reason: str = Field(description="Briefly explain why this specific tool/input helps the investigation") + +class Thought(BaseModel): + thought: str = Field( + description="Brief reasoning about next step (max 3-4 sentences)", + max_length=3000, + ) + mode: Literal["act", "finish"] = Field(description="'act' to call tools (when more information is needed), 'finish' to return the final answer (when you have sufficient evidence)") + + actions: ToolCall | None = Field(default=None, description="When mode is 'act', the tool to execute") + + final_answer: str | None = Field( + default=None, + description="When mode is 'finish', concise answer (3-5 sentences) with key evidence", + max_length=3000, + ) + + +class CheckerToolCall(BaseModel): + """Tool call for RPM checker flow - simpler schema with just query.""" + tool: str = Field(description="Exact tool name from AVAILABLE_TOOLS") + query: str = Field(description="Search pattern for Source Grep or Code Keyword Search") + reason: str = Field(description="Briefly explain why this tool helps the investigation") + + +class CheckerThought(BaseModel): + """Thought model for RPM checker flow with simplified tool call schema.""" + thought: str = Field( + description="Brief reasoning about next step (max 3-4 sentences)", + max_length=3000, + ) + mode: Literal["act", "finish"] = Field( + description="'act' to call tools, 'finish' to return final answer" + ) + actions: CheckerToolCall | None = Field( + default=None, + description="When mode is 'act', the tool to execute" + ) + final_answer: str | None = Field( + default=None, + description="When mode is 'finish', concise answer with evidence", + max_length=3000, + ) + + +class CodeFindings(BaseModel): + """Compressed code comprehension output from raw tool results.""" + findings: list[str] = Field( + description="3-5 key technical facts from tool output. Each describes what the code DOES " + "and how it relates to the investigation goal, not just that it was found." + ) + tool_outcome: str = Field( + description="One-line summary: CALLED: [tool] with [input] -> [brief outcome]" + ) + + +def check_empty_output( + tool_output: str | list, + tool_used: str, + tool_input: str, + allow_llm_classification: bool = False, +) -> tuple[CodeFindings | None, bool]: + """Check if tool output is empty or an error, returning factual CodeFindings if so. + + For empty Source Grep on source code, we may want the LLM to classify the result + against the RAW_PATCH_DIFF (to distinguish VULNERABLE_CODE_ABSENT vs FIX_CODE_ABSENT). + + Args: + tool_output: The raw output from the tool. + tool_used: Name of the tool (e.g., "Source Grep"). + tool_input: The input/query passed to the tool. + allow_llm_classification: If True, signal that empty Source Grep on source + code needs LLM classification against the diff. + + Returns: + Tuple of (CodeFindings | None, needs_llm_classification: bool). + - If CodeFindings is returned, use it directly. + - If CodeFindings is None and needs_llm_classification is True, call classification LLM. + - If CodeFindings is None and needs_llm_classification is False, output has content. + """ + is_empty = ( + not tool_output + or (isinstance(tool_output, str) and tool_output.strip() in ("[]", "")) + or (isinstance(tool_output, list) and len(tool_output) == 0) + ) + + is_error = ( + isinstance(tool_output, str) + and any(m in tool_output for m in ["Error:", "error:", "Failed:", "Exception:", "Traceback"]) + ) + + if is_error: + return ( + CodeFindings( + findings=[ + f"FAILED: {tool_used} [{tool_input}] - tool error", + f"Details: {str(tool_output)[:150]}" + ], + tool_outcome=f"FAILED: {tool_used} with {tool_input} -> ERROR" + ), + False, + ) + + if is_empty: + is_source_grep_on_source = ( + tool_used == "Source Grep" + and not tool_input.startswith("logs:") + and not tool_input.startswith("patch:") + ) + if allow_llm_classification and is_source_grep_on_source: + return (None, True) + + return ( + CodeFindings( + findings=[f"{tool_used} for '{tool_input}' returned empty - no matches found"], + tool_outcome=f"CALLED: {tool_used} with {tool_input} -> EMPTY (no results)" + ), + False, + ) + + return (None, False) + + +async def invoke_comprehension( + structured_comprehension_llm, + prompt: str, + tool_used: str, + tool_input: str, + tool_output: str, + agent_label: str = "", +) -> CodeFindings: + """Invoke comprehension LLM with fallback on token limit overflow.""" + try: + return await structured_comprehension_llm.ainvoke([SystemMessage(content=prompt)]) + except LengthFinishReasonError: + logger.warning("%s comprehension LLM hit token limit (tool=%s), using fallback", agent_label, tool_used) + summary = tool_output[:500] if isinstance(tool_output, str) else str(tool_output)[:500] + return CodeFindings( + findings=[f"Tool output received but comprehension exceeded token limit. Raw excerpt: {summary}"], + tool_outcome=f"CALLED: {tool_used} with {tool_input} -> output received (comprehension truncated)", + ) + + +class Observation(BaseModel): + results: list[str] = Field( + description="3-5 key technical facts from this tool output. Each fact must describe what the code DOES and how it relates to the investigation goal, not just that it was found." + ) + + memory: list[str] = Field( + description="Cumulative factual findings. Each item is a single, discrete technical fact that records functional behavior and context (e.g. what a function does, which package it belongs to, whether it is semantically relevant to the goal)." + ) + + +class Classification(BaseModel): + """Structured output for reachability-question classification.""" + is_reachability: Literal["yes", "no"] = Field( + description="Answer 'yes' if the question is a reachability question, 'no' otherwise." + ) + + +class L1VerdictExtraction(BaseModel): + """Lightweight structured output for extracting verdict from L1 final answer.""" + preliminary_verdict: str = Field( + description=( + "Classify the L1 agent's conclusion: " + "'protected' if fix/patch applied or code mitigated, " + "'not_present' if vulnerable code not found in this version, " + "'vulnerable' if vulnerable code confirmed present, " + "'uncertain' if evidence is insufficient or conflicting" + ) + ) + confidence: float = Field( + description="Confidence in the verdict (0.0 to 1.0) based on evidence strength in the answer" + ) + reasoning: str = Field( + description="Brief explanation of why this verdict was chosen" + ) + + +class PackageSelection(BaseModel): + """Structured output for selecting the most relevant package from multiple candidates.""" + selected_package: str = Field( + description="The exact name of the single most relevant package to investigate, copied verbatim from the candidates list." + ) + reason: str = Field( + description="One-sentence justification for why this package is the best investigation target." + ) + +class BaseRulesTracker: + """Shared behavioral rule infrastructure for all sub-agents.""" + def __init__(self): + self.action_history = {} + self.target_package = None + self.allowed_tools = [] + + def set_allowed_tools(self, allowed_tools: list[str]): + self.allowed_tools = allowed_tools + + def set_target_package(self, target_package: str): + self.target_package = target_package + + @staticmethod + def _is_empty_result(output) -> bool: + """Check if a tool output represents an empty/no-results response. + + Handles both string ('[]', '') and list ([]) formats since + ToolMessage.content can be either type. + """ + if isinstance(output, list): + return len(output) == 0 + if isinstance(output, str): + return output.strip() in ("[]", "") + return False + + def add_action(self, action: str, action_input: str, output): + entry = {"input": action_input, "output": output} + if action not in self.action_history: + self.action_history[action] = [entry] + else: + self.action_history[action].append(entry) + + def _rule_duplicate_call(self, action: str, action_input: str) -> bool: + if action not in self.action_history: + return False + return any(prev["input"] == action_input for prev in self.action_history[action]) + + def _rule_number_7(self, action: str, action_input: str, output) -> bool: + if action != "Code Keyword Search": + return False + if "." not in action_input: + return False + if not self._is_empty_result(output): + return False + if action not in self.action_history: + return False + prev = self.action_history[action][-1] + if "." in prev["input"] and self._is_empty_result(prev["output"]): + return True + return False + + def _rule_use_allowed_tools(self, action: str) -> bool: + if action not in self.allowed_tools: + return True + return False + + def check_thought_behavior(self, action: str, action_input: str, output) -> tuple[bool, str]: + if self._rule_duplicate_call(action, action_input): + return True, ( + f"You already called {action} with this exact input. " + "You MUST use a DIFFERENT tool or a DIFFERENT input query. " + "Check KNOWLEDGE for what was already tried." + ) + if self._rule_number_7(action, action_input, output): + return True, ("You are NOT following Rule 5. Your query contains dots and returned " + "no results. You MUST retry with just the final component. Follow the rules.") + if self._rule_use_allowed_tools(action): + return True, (f"You are NOT following AVAILABLE_TOOLS. You MUST use the allowed tools {self.allowed_tools}. Follow the rules.") + self.add_action(action, action_input, output) + return False, "" + + +class CheckerSearchTracker(BaseRulesTracker): + """Lightweight search tracker for L2 Build Agent. + + Prevents duplicate tool calls by checking BEFORE execution. Inherits + duplicate detection logic from BaseRulesTracker but provides a simpler + interface for the "check-then-execute" pattern needed when context + pruning causes the LLM to forget previous searches. + """ + + def check_before_execute(self, tool: str, query: str) -> str | None: + """Check if this search was already performed BEFORE executing. + + Args: + tool: Tool name (e.g., 'Source Grep') + query: Search query (e.g., 'logs:mod_proxy_ftp.c') + + Returns: + None if this is a new search (automatically recorded). + Warning message string if this is a duplicate search. + """ + if self._rule_duplicate_call(tool, query): + return ( + f"DUPLICATE SEARCH DETECTED: You already called '{tool}' with '{query}'. " + f"Check KNOWLEDGE for the previous result. " + f"Use a DIFFERENT query or tool, or finish with available evidence." + ) + self.add_action(tool, query, output=None) + return None + + def reset(self): + """Clear search history. Call when switching investigation phases.""" + self.action_history.clear() + + +class ReachabilityRulesTracker(BaseRulesTracker): + """Behavioral rules for the reachability sub-agent.""" + def __init__(self): + super().__init__() + self.target_functions: dict[str, bool] = {} + self.ecosystem: str = "" + self.validated_packages: set[str] = set() + + def set_ecosystem(self, ecosystem: str): + self.ecosystem = ecosystem.lower() if ecosystem else "" + + def check_finish_allowed(self, cca_results: list[bool]) -> tuple[bool, str]: + """Block finish when mandatory reachability tools were not used. + + Two rules enforced: + 1. CCA must be called before finishing any reachability question. + 2. (Java only) FLVF must be called when CCA found reachability. + """ + if not cca_results and ToolNames.CALL_CHAIN_ANALYZER not in self.action_history: + return False, ( + "You MUST use Function Locator and Call Chain Analyzer before concluding. " + "Code Keyword Search alone is NOT sufficient to determine reachability. " + "Call Function Locator to validate the package, then Call Chain Analyzer " + "to check if the vulnerable function is reachable from application code." + ) + if self.ecosystem == "java" and any(cca_results): + if ToolNames.FUNCTION_LIBRARY_VERSION_FINDER not in self.action_history: + return False, ( + "MANDATORY VERSION CHECK: Call Chain Analyzer found the vulnerable function is reachable, " + "but you have NOT verified the installed library version. " + "You MUST call Function Library Version Finder (e.g., input 'commons-beanutils') " + "to check whether the installed version is within the vulnerable range before concluding." + ) + return True, "" + + def set_target_functions(self, functions: list[str]): + self.target_functions = {f: False for f in functions} + + @staticmethod + def _normalize_package_name(name: str) -> str: + return name.strip().lower().replace("-", "_") + + def add_validated_package(self, package_name: str): + """Register a package that FL confirmed as valid (e.g., uber-jar alternative).""" + self.validated_packages.add(self._normalize_package_name(package_name)) + + def _rule_number_8(self, action: str, action_input: str, output) -> bool: + if self.target_package is None: + return False + if action not in ("Function Locator", "Call Chain Analyzer", "Function Caller Finder"): + return False + if action not in self.action_history: + input_pkg = self._normalize_package_name(action_input.split(",")[0]) + target_pkg = self._normalize_package_name(self.target_package) + # Allow Java GAV with version suffix: input "group:artifact:version" + # should match target "group:artifact" + if input_pkg == target_pkg or input_pkg.startswith(target_pkg + ":"): + return False + # Allow packages that FL already validated (handles uber-jars like + # netty-all containing netty-codec-http classes) + if any(input_pkg == vp or input_pkg.startswith(vp + ":") for vp in self.validated_packages): + return False + return True + return False + + def _rule_number_9(self, action: str, action_input: str) -> tuple[bool, str]: + if not self.target_functions: + return False, "" + if action not in ("Call Chain Analyzer", "Function Caller Finder"): + return False, "" + input_function = action_input.split(",", 1)[-1].strip() if "," in action_input else "" + if not input_function: + return False, "" + input_short = input_function.rsplit(".", 1)[-1].lower() + for fn in self.target_functions: + if fn.lower() == input_short: + self.target_functions[fn] = True + return False, "" + pending = [fn for fn, checked in self.target_functions.items() if not checked] + if pending: + return True, ( + f"You are NOT following Rule 7. The CVE lists specific vulnerable functions " + f"that you MUST investigate first: {', '.join(pending)}. " + f"Check these functions before investigating other functions." + ) + return False, "" + + def check_thought_behavior(self, action: str, action_input: str, output) -> tuple[bool, str]: + if self._rule_duplicate_call(action, action_input): + logger.debug("Duplicate call rule triggered: '%s' with same input", action) + return True, ( + f"You already called {action} with this exact input. " + "You MUST use a DIFFERENT tool or a DIFFERENT input query. " + "Check KNOWLEDGE for what was already tried." + ) + if self._rule_number_7(action, action_input, output): + logger.debug("Reachability Rule 5 triggered: dotted query with empty results for tool '%s'", action) + return True, ("You are NOT following Rule 5. Your query contains dots and returned " + "no results. You MUST retry with just the final component. Follow the rules.") + if self._rule_number_8(action, action_input, output): + logger.debug("Reachability Rule 6 triggered: wrong package for tool '%s', expected '%s'", action, self.target_package) + return True, (f"You are NOT following Rule 6. You are using the wrong package name. You MUST use the target package name {self.target_package} as the package_name before trying alternative packages. Follow the rules.") + if self._rule_use_allowed_tools(action): + logger.debug("Reachability allowed-tools rule triggered: '%s' not in %s", action, self.allowed_tools) + return True, (f"You are NOT following AVAILABLE_TOOLS. You MUST use the allowed tools {self.allowed_tools}. Follow the rules.") + rule9, msg9 = self._rule_number_9(action, action_input) + if rule9: + logger.debug("Reachability Rule 7 triggered: must investigate target functions first for tool '%s'", action) + return True, msg9 + self.add_action(action, action_input, output) + return False, "" + + +class AgentState(MessagesState): + input: str = "" + step: int = 0 + max_steps: int = 10 + thought: Thought | None = None + observation: Observation | None = None + output: str = "" + ecosystem: str | None = None + runtime_prompt: str | None = None + app_package: str | None = None + is_reachability: str = "yes" + rules_tracker: BaseRulesTracker = ReachabilityRulesTracker() + critical_context: list[str] = [] + cca_results: list[bool] = [] + package_validated: bool | None = None + precomputed_intel: tuple | None = None + + +# ---- Reachability Sub-Agent Prompts ---- # +# The following prompts (REACHABILITY_AGENT_SYS_PROMPT, REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS, +# REACHABILITY_REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS_GO) are used exclusively by the reachability sub-agent, +# which traces call chains to determine if vulnerable code is reachable from application code. + +REACHABILITY_AGENT_SYS_PROMPT = ( + "You are a security analyst investigating CVE exploitability in container images.\n" + "MANDATORY STEPS (follow in order, do NOT skip any):\n" + "1. IDENTIFY the vulnerable component/function from the CVE description.\n" + "2. SEARCH for its presence using Code Keyword Search.\n" + "3. TRACE reachability using Call Chain Analyzer. " + " - Use the Function Locator to verify the package name and find the function name." + " - For Go: use Function Caller Finder to identify which application functions call the vulnerable library function, BEFORE running Call Chain Analyzer." + " - Keyword search alone is NOT sufficient -- you must trace the call chain.\n" + "4. ASSESS: only after completing reachability checks, determine exploitability.\n" + "GENERAL RULES:\n" + "- Base conclusions ONLY on tool results, not assumptions.\n" + "- If a search returns no results, that is evidence the code is absent.\n" + "- Do NOT claim a function is used unless a tool confirmed it.\n" + "- Code Keyword Search proves code PRESENCE in the container, NOT reachability.\n" + "- Function Locator validates package/function NAMES, NOT reachability. It confirms the name exists, not that it is called.\n" + "- Only Call Chain Analyzer can confirm reachability. The application may contain code it never calls.\n" + "- When the question asks whether a function is called or reachable, do NOT conclude based on Code Keyword Search or Function Locator alone -- you MUST use Call Chain Analyzer.\n" + "STOPPING RULES:\n" + "- POSITIVE reachability (Call Chain Analyzer returns True): you MAY conclude exploitable and finish.\n" + "- NEGATIVE reachability (Call Chain Analyzer returns False): record the result. " + "You may only conclude 'not exploitable' after Call Chain Analyzer has confirmed the function is not reachable.\n" + "ANSWER QUALITY:\n" + "- Answer the SPECIFIC question asked with evidence. Do NOT just report what tools found.\n" + "- Never give bare assertions (e.g. 'not exploitable'). Always state: WHAT you checked, WHAT you found, and WHY it leads to your conclusion.\n" + "- Distinguish between code being PRESENT (exists in container), REACHABLE (called from application code), and EXPLOITABLE (attacker-controlled input can trigger it). Do not conflate these.\n" + "- If tool results conflict with each other, state the conflict explicitly rather than silently picking one side.\n" + "- Finding that a security check is ABSENT is potential evidence of vulnerability, not evidence of safety.\n" + "- When citing evidence, explain HOW it relates to the question -- do not just state that something was found." +) + +LANGGRAPH_SYSTEM_PROMPT_TEMPLATE = """{sys_prompt} + + +{tools} + + + +{tool_selection_strategy} + + +{tool_instructions} + +RESPONSE: +{{""" + +REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS = """ +1. Output valid JSON only. thought < 100 words, final_answer < 150 words. mode="act" REQUIRES actions; mode="finish" REQUIRES final_answer. +2. Function Locator, Function Caller Finder, Call Chain Analyzer: MUST set package_name AND function_name. Code Keyword Search, Code Semantic Search, Docs Semantic Search, CVE Web Search: use query field. +3. Code Keyword Search and Function Locator are NOT reachability proof -- only Call Chain Analyzer is. +4. Do NOT call the same tool with the same input twice. Check KNOWLEDGE for "CALLED:" entries. If already tried, use a DIFFERENT tool or different input. +5. If Code Keyword Search returns no results and the query contains dots (e.g. a.b.ClassName), retry with just the final component (e.g. ClassName). This does NOT apply to simple names without dots. +6. When using Function Locator, Call Chain Analyzer, or Function Caller Finder: always use the TARGET PACKAGE from KNOWLEDGE as the package_name before trying alternative packages. +7. For function names, prioritize in this order: (a) "Vulnerable functions (GHSA)" or "Vulnerable functions (Go vuln DB)" — you MUST investigate these first, (b) "Vulnerable functions (remediation patch hint)" — prefer these over names inferred from the CVE description. If Function Locator returns close matches instead of an exact match, retry with the closest matching function name before calling Call Chain Analyzer. + + +{{"thought": "Search for the vulnerable function in the codebase first", "mode": "act", "actions": {{"tool": "Code Keyword Search", "package_name": null, "function_name": null, "query": "", "tool_input": null, "reason": "Check if vulnerable function is present"}}, "final_answer": null}} + + +{{"thought": "Found the function. Now use Function Locator to verify the package name and function", "mode": "act", "actions": {{"tool": "Function Locator", "package_name": "", "function_name": "", "query": null, "tool_input": null, "reason": "Validate package and function name before Call Chain Analyzer"}}, "final_answer": null}} + + +{{"thought": "Function Locator confirmed the package. Now trace reachability with Call Chain Analyzer", "mode": "act", "actions": {{"tool": "Call Chain Analyzer", "package_name": "", "function_name": "", "query": null, "tool_input": null, "reason": "Check if function is reachable from application code"}}, "final_answer": null}} +""" + +REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS_GO = """ +1. Output valid JSON only. thought < 100 words, final_answer < 150 words. mode="act" REQUIRES actions; mode="finish" REQUIRES final_answer. +2. Function Locator, Function Caller Finder, Call Chain Analyzer: MUST set package_name AND function_name. Code Keyword Search, Code Semantic Search, Docs Semantic Search, CVE Web Search: use query field. +3. Code Keyword Search and Function Locator are NOT reachability proof -- only Call Chain Analyzer is. +4. Do NOT call the same tool with the same input twice. Check KNOWLEDGE for "CALLED:" entries. If already tried, use a DIFFERENT tool or different input. +5. If Code Keyword Search returns no results and the query contains dots (e.g. a.b.ClassName), retry with just the final component (e.g. ClassName). This does NOT apply to simple names without dots. +6. When using Function Locator, Call Chain Analyzer, or Function Caller Finder: always use the TARGET PACKAGE from KNOWLEDGE as the package_name before trying alternative packages. +7. For function names, prioritize in this order: (a) "Vulnerable functions (GHSA)" or "Vulnerable functions (Go vuln DB)" — you MUST investigate these first, (b) "Vulnerable functions (remediation patch hint)" — prefer these over names inferred from the CVE description. If Function Locator returns close matches instead of an exact match, retry with the closest matching function name before calling Call Chain Analyzer. +8. After Function Locator validates the package, use Function Caller Finder to identify callers. If FCF finds callers: the function IS reachable -- you MAY conclude and finish. If FCF returns empty: this does NOT mean unreachable. You MUST proceed to Call Chain Analyzer. Do NOT finish or conclude without calling Call Chain Analyzer when FCF returned empty. + + +{{"thought": "Search for the vulnerable function in the codebase first", "mode": "act", "actions": {{"tool": "Code Keyword Search", "package_name": null, "function_name": null, "query": "", "tool_input": null, "reason": "Check if vulnerable function is present"}}, "final_answer": null}} + + +{{"thought": "Found the function. Now use Function Locator to verify the package name and function", "mode": "act", "actions": {{"tool": "Function Locator", "package_name": "", "function_name": "", "query": null, "tool_input": null, "reason": "Validate package and function name before tracing callers"}}, "final_answer": null}} + + +{{"thought": "Function Locator confirmed the package. Now find which app functions call this library function", "mode": "act", "actions": {{"tool": "Function Caller Finder", "package_name": "", "function_name": ".()", "query": null, "tool_input": null, "reason": "Identify application functions that call the vulnerable library function"}}, "final_answer": null}} + + + +{{"thought": "Function Caller Finder returned no callers, but this does not prove unreachable. MUST call Call Chain Analyzer to confirm reachability", "mode": "act", "actions": {{"tool": "Call Chain Analyzer", "package_name": "", "function_name": "", "query": null, "tool_input": null, "reason": "Check if function is reachable from application code"}}, "final_answer": null}} +""" + +REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS_PYTHON = """ +1. Output valid JSON only. thought < 100 words, final_answer < 150 words. mode="act" REQUIRES actions; mode="finish" REQUIRES final_answer. +2. Function Locator, Function Caller Finder, Call Chain Analyzer: MUST set package_name AND function_name. Code Keyword Search, Code Semantic Search, Docs Semantic Search, CVE Web Search: use query field. +3. Code Keyword Search and Function Locator are NOT reachability proof -- only Call Chain Analyzer is. +4. Do NOT call the same tool with the same input twice. Check KNOWLEDGE for "CALLED:" entries. If already tried, use a DIFFERENT tool or different input. +5. If Code Keyword Search returns no results and the query contains dots (e.g. a.b.ClassName), retry with just the final component (e.g. ClassName). This does NOT apply to simple names without dots. +6. When using Function Locator, Call Chain Analyzer, or Function Caller Finder: always use the TARGET PACKAGE from KNOWLEDGE as the package_name before trying alternative packages. +7. For function names, prioritize in this order: (a) "Vulnerable functions (GHSA)" or "Vulnerable functions (Go vuln DB)" — you MUST investigate these first, (b) "Vulnerable functions (remediation patch hint)" — prefer these over names inferred from the CVE description. If Function Locator returns close matches instead of an exact match, retry with the closest matching function name before calling Call Chain Analyzer. +8. For Call Chain Analyzer, use the class name (e.g. formparser.MultiPartParser) NOT the method name (e.g. formparser.MultiPartParser.parse). Python imports are at the class/module level, so reachability must be traced at that level. If Function Locator returned a module.ClassName.method match, drop the method part for Call Chain Analyzer. + + +{{"thought": "Search for the vulnerable function in the codebase first", "mode": "act", "actions": {{"tool": "Code Keyword Search", "package_name": null, "function_name": null, "query": "", "tool_input": null, "reason": "Check if vulnerable function is present"}}, "final_answer": null}} + + +{{"thought": "Found the function. Now use Function Locator to verify the package name and function", "mode": "act", "actions": {{"tool": "Function Locator", "package_name": "", "function_name": "", "query": null, "tool_input": null, "reason": "Validate package and function name before Call Chain Analyzer"}}, "final_answer": null}} + + +{{"thought": "Function Locator confirmed the package. Now trace reachability with Call Chain Analyzer using the class name", "mode": "act", "actions": {{"tool": "Call Chain Analyzer", "package_name": "", "function_name": ".", "query": null, "tool_input": null, "reason": "Check if class is reachable from application code"}}, "final_answer": null}} +""" + +REACHABILITY_AGENT_NON_REACH_SYS_PROMPT = ( + "You are a security analyst investigating CVE exploitability in container images.\n" + "This is NOT a reachability question -- do NOT trace call chains.\n" + "MANDATORY STEPS (follow in order, do NOT skip any):\n" + "1. IDENTIFY the vulnerable component/function from the CVE description.\n" + "2. SEARCH for its presence using Code Keyword Search.\n" + "3. DISTINGUISH where the code was found: main application code vs. package dependencies. " + " Results from Code Keyword Search are grouped into 'Main application' and 'Application library dependencies'. " + " Pay close attention to which group contains the match.\n" + "4. ASSESS: determine the answer based on code presence, version info, configuration, " + " or any other evidence relevant to the question. Use Code Semantic Search or Docs Semantic Search " + " for deeper understanding when needed.\n" + "CRITICAL RULE:\n" + "- If a vulnerable function is found ONLY in 'Application library dependencies' " + " and NOT in 'Main application', this means the code exists in the dependency " + " tree but the main application does NOT directly use it. " + " Do NOT conclude the application is vulnerable based solely on presence in " + " dependency libraries. State clearly: 'found in dependency libraries but not " + " directly referenced by the main application.'\n" + "GENERAL RULES:\n" + "- Base conclusions ONLY on tool results, not assumptions.\n" + "- If a search returns no results, that is evidence the code is absent.\n" + "- Do NOT claim a function is used unless a tool confirmed it.\n" + "- Use CVE Web Search to gather additional vulnerability context if needed.\n" + "ANSWER QUALITY:\n" + "- Answer the SPECIFIC question asked with evidence. Do NOT just report what tools found.\n" + "- Never give bare assertions (e.g. 'not exploitable'). Always state: WHAT you checked, WHAT you found, and WHY it leads to your conclusion.\n" + "- Distinguish between code being PRESENT (exists in container) and EXPLOITABLE (attacker-controlled input can trigger it). Do not conflate these.\n" + "- If tool results conflict with each other, state the conflict explicitly rather than silently picking one side.\n" + "- Finding that a security check is ABSENT is potential evidence of vulnerability, not evidence of safety.\n" + "- When citing evidence, explain HOW it relates to the question -- do not just state that something was found." +) + +REACHABILITY_AGENT_NON_REACH_THOUGHT_INSTRUCTIONS = """ +1. Output valid JSON only. thought < 100 words. final_answer < 150 words. +2. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. +3. Function Locator: MUST set package_name AND function_name. Do NOT use query. +4. Code Keyword Search, Code Semantic Search, Docs Semantic Search, CVE Web Search: use query field. +5. Do NOT call the same tool with the same input twice. Check KNOWLEDGE for "CALLED:" entries. If already tried, use a DIFFERENT tool or different input. +6. If Code Keyword Search returns no results and the query contains dots (e.g. a.b.ClassName), retry with just the final component (e.g. ClassName). This does NOT apply to simple names without dots. +7. When Code Keyword Search results are grouped, note whether matches are in "Main application" or "Application library dependencies" -- this distinction is important for your analysis. +8. Function Locator validates package/function NAMES only. It confirms the name exists, not that it is called or reachable. +9. When using Function Locator, always start with the TARGET PACKAGE from KNOWLEDGE as the package_name before trying alternative packages. + + +{{"thought": "Search for the vulnerable function in the codebase first", "mode": "act", "actions": {{"tool": "Code Keyword Search", "package_name": null, "function_name": null, "query": "", "tool_input": null, "reason": "Check if vulnerable function is present in the container"}}, "final_answer": null}} + + +{{"thought": "Found the function in dependencies. Use Function Locator to validate the package and function name", "mode": "act", "actions": {{"tool": "Function Locator", "package_name": "", "function_name": "", "query": null, "tool_input": null, "reason": "Validate package and function name"}}, "final_answer": null}} + + +{{"thought": "Have enough evidence about the vulnerable component presence and context", "mode": "finish", "actions": null, "final_answer": "The vulnerable function was found in the container's dependency libraries under . The main application code does not directly reference it. Based on the CVE description and the evidence gathered, ..."}} +""" + +CLASSIFICATION_PROMPT_TEMPLATE = """You are classifying a CVE investigation question. + +Context (CVE / vulnerable packages): +{context_block} + +Question: {question} + +A reachability question asks whether the vulnerable code/symbol is called or reachable in the codebase, or whether untrusted data can reach it. Is this a reachability question? Answer only yes or no.""" + +PACKAGE_FILTER_PROMPT_TEMPLATE = """The container runs "{image_context}". +{image_match_note} +Candidate packages: +{candidates} + +Ecosystem: {ecosystem} +{critical_context_section} +Rules: +1. If a candidate was pre-identified as matching the container image/repo (see note above), you MUST select it. This OVERRIDES all other rules. Do NOT select a sub-library or component package. +2. Otherwise, pick the package whose ecosystem matches "{ecosystem}". +3. Discard ecosystem-repackaging wrappers (e.g. maven webjars wrapping an npm library) in favour of the native package. +4. Return the package name exactly as it appears in the candidates list above.""" + +FORCED_FINISH_PROMPT = """Maximum steps reached. You MUST set mode="finish" and provide final_answer NOW. +Do NOT call any more tools. Summarize your evidence in 3-5 sentences. +RESPONSE: +{{""" + +COMPREHENSION_PROMPT = """Analyze the tool output and extract key technical findings. +GOAL: {goal} +TARGET PACKAGE (vulnerability): {selected_package} +VULNERABILITY CONTEXT: +{critical_context} +TOOL USED: {tool_used} +TOOL INPUT: {tool_input_detail} +THOUGHT: {last_thought_text} +NEW OUTPUT: +{tool_output} + +CRITICAL — CALL CHAIN ANALYZER REACHABILITY: +When TOOL USED is "Call Chain Analyzer": +- If the result is POSITIVE (reachable): your first finding MUST be "REACHABLE via [package] - sufficient evidence." Do NOT hedge, qualify, or say "further investigation required." +- If the result is NEGATIVE (not reachable): your first finding MUST be "NOT reachable via [package]." +These override all other rules for Call Chain Analyzer results. + +CODE COMPREHENSION RULES: +1. READ the actual code snippets in NEW OUTPUT. Do NOT just check whether something was "found" or "not found." +2. For each code snippet or function returned, determine: + - What does this code actually DO? (its functional purpose, not just its name) + - What data does it operate on and in what context? + - Is its purpose semantically related to the GOAL, or is it merely a keyword match? +3. DISTINGUISH same-named entities: a function with the same name in a different package, module, or namespace is a DIFFERENT function. Always record the specific package/module context. +4. CHECK technology fit: if a finding comes from a different programming language, framework, or platform than the target under investigation, note the mismatch explicitly. Do NOT treat it as equivalent. +5. RECORD functional behavior, not just location. Findings must describe what the code DOES (e.g., "rate_limit() in X throttles I/O bandwidth for read/write operations"), not just where it was found (e.g., "rate_limit found in X"). +6. SEPARATE keyword presence from investigation relevance: a security-related function existing in the codebase is NOT automatically a mitigation or evidence for the vulnerability being investigated. Explain HOW the finding relates to the specific GOAL. +7. GROUND TO VENDOR MITIGATIONS: If VULNERABILITY CONTEXT contains "KNOWN MITIGATIONS:", compare the code and behavior in NEW OUTPUT against those mitigations. Record whether the codebase implements, partially implements, or contradicts the vendor's recommended mitigations (e.g. configuration settings, patches, workarounds). Do not treat unrelated security-related code as evidence of mitigation for this CVE. + +TOOL-SPECIFIC RULES: +- If NEW OUTPUT is empty, contains an error, or indicates tool failure, findings must only state: "FAILED: {tool_used} [{tool_input_detail}] - [reason]". Do NOT infer or fabricate positive findings. +- Reachability tags apply ONLY to Call Chain Analyzer results. Code Keyword Search and Function Locator do NOT determine reachability. +- If Call Chain Analyzer returned NEGATIVE (False), include: "NOT reachable via [package]." +- If Call Chain Analyzer returned POSITIVE (True), include: "REACHABLE via [package] - sufficient evidence." +- For Function Locator results, include: "VALIDATED: [package],[function] exists" -- this is NOT reachability. +- When Code Keyword Search results are grouped into "Main application" and "Application library dependencies", prioritize findings from the main application group. +- findings: 3-5 key technical facts from this OUTPUT only. Each fact must reflect code comprehension (what the code does), not just keyword presence. +- tool_outcome: a single line "CALLED: [tool] with [input] -> [brief outcome]". +- Keep only CVE-exploitability-relevant information. +RESPONSE: +{{""" + +MEMORY_UPDATE_PROMPT = """Merge new findings into the investigation memory. +GOAL: {goal} +TARGET PACKAGE (vulnerability): {selected_package} +PREVIOUS MEMORY: {previous_memory} +NEW FINDINGS (from tool analysis): +{findings} +TOOL CALL RECORD: {tool_outcome} + +RULES: +- memory: Start from PREVIOUS MEMORY. Append new facts from NEW FINDINGS. Record absence if nothing was found. No duplicates. Factual only. +- Add TOOL CALL RECORD verbatim to memory so future steps know what was already tried. +- If NEW FINDINGS report a failure (starts with "FAILED:"), add the failure to memory. Do NOT infer or fabricate positive findings. +- Reachability tags apply ONLY to Call Chain Analyzer results. Code Keyword Search and Function Locator do NOT determine reachability. +- If NEW FINDINGS mention "NOT reachable", add to memory: "NOT reachable via [package]." +- If NEW FINDINGS mention "REACHABLE", add: "REACHABLE via [package] - sufficient evidence." +- For Function Locator validation, record: "VALIDATED: [package],[function] exists" -- this is NOT reachability. +- When findings mention "Main application" vs "Application library dependencies", preserve this distinction in memory. +- If PREVIOUS MEMORY contains "KNOWN MITIGATIONS:" and findings are relevant (e.g. config, code path, or version), add to memory whether the codebase aligns with or contradicts those mitigations (e.g. "Mitigation check: follow_symlinks=False not set" or "Vendor mitigation (patch X) applied in version Y"). +- results: copy the NEW FINDINGS as-is. +- Keep only CVE-exploitability-relevant information. +RESPONSE: +{{""" + +def build_reachability_system_prompt( + tool_descriptions: str, + tool_guidance: str, + instructions: str = REACHABILITY_AGENT_THOUGHT_INSTRUCTIONS, + sys_prompt: str | None = None, +) -> str: + sys_prompt = sys_prompt or REACHABILITY_AGENT_SYS_PROMPT + return LANGGRAPH_SYSTEM_PROMPT_TEMPLATE.format( + sys_prompt=sys_prompt, + tools=tool_descriptions, + tool_instructions=instructions, + tool_selection_strategy=tool_guidance, + ) + + +def build_classification_prompt(context_block: str, question: str) -> str: + """Build the reachability-question classification prompt from context and user question.""" + return CLASSIFICATION_PROMPT_TEMPLATE.format( + context_block=context_block, + question=question, + ) + + +def _find_image_matching_candidate( + candidates: list[dict], + image_name: str | None, + image_repo: str | None, +) -> str | None: + """Return the candidate name that appears in the image/repo string, or None.""" + context = " ".join( + part.lower() for part in (image_name, image_repo) if part + ) + if not context: + return None + for c in candidates: + name = c["name"].lower() + if len(name) >= 3 and name in context: + return c["name"] + return None + + +def build_package_filter_prompt( + ecosystem: str, + candidates: list[dict], + image_name: str | None = None, + image_repo: str | None = None, + critical_context: list[str] | None = None, +) -> str: + """Build the package-selection prompt from ecosystem, candidate packages, and container image info.""" + candidate_lines = "\n".join( + f"- {c['name']} (source: {c.get('source', 'unknown')}, ecosystem: {c.get('ecosystem', 'N/A')})" + for c in candidates + ) + parts = [] + if image_name: + parts.append(image_name) + if image_repo: + parts.append(f"repo: {image_repo}") + image_context = ", ".join(parts) if parts else "unknown" + + matched = _find_image_matching_candidate(candidates, image_name, image_repo) + if matched: + image_match_note = f'MATCH DETECTED: candidate "{matched}" matches the container image/repo. Select it (Rule 1).' + critical_context_section = "" + else: + image_match_note = "NO MATCH: no candidate package name was found in the image/repo identifier. Rule 1 does not apply — use Rule 2." + if critical_context: + context_block = "\n".join(critical_context) + critical_context_section = f"\nVulnerability context (use to disambiguate candidates):\n{context_block}\n" + else: + critical_context_section = "" + + return PACKAGE_FILTER_PROMPT_TEMPLATE.format( + ecosystem=ecosystem, + candidates=candidate_lines, + image_context=image_context, + image_match_note=image_match_note, + critical_context_section=critical_context_section, + ) + + +def _build_tool_arguments(actions: ToolCall)->dict[str, Any]: + pkg_tools = {"Function Locator", "Function Caller Finder", "Call Chain Analyzer"} + if actions.tool in pkg_tools and actions.package_name and actions.function_name: + return {"query": f"{actions.package_name},{actions.function_name}"} + if actions.query: + return {"query": actions.query} + if actions.tool_input: + return {"query": actions.tool_input} # fallback + logger.warning("Tool '%s' called without required arguments (package_name=%s, function_name=%s, query=%s)", + actions.tool, actions.package_name, actions.function_name, actions.query) + raise ValueError(f"Tool {actions.tool} requires package_name+function_name or query/tool_input") + + + diff --git a/src/vuln_analysis/functions/tests/test.py b/src/vuln_analysis/functions/tests/test.py new file mode 100644 index 000000000..9e1d223e2 --- /dev/null +++ b/src/vuln_analysis/functions/tests/test.py @@ -0,0 +1 @@ +assert True \ No newline at end of file diff --git a/src/vuln_analysis/functions/tests/test_cve_fetch_patches.py b/src/vuln_analysis/functions/tests/test_cve_fetch_patches.py new file mode 100644 index 000000000..5f9119a2e --- /dev/null +++ b/src/vuln_analysis/functions/tests/test_cve_fetch_patches.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for cve_fetch_patches module.""" + +import asyncio +import inspect + +from vuln_analysis.functions import cve_fetch_patches as mod + + +class TestCveFetchPatchesSemaphore: + """Tests for semaphore-bounded concurrency (fix 4).""" + + def test_semaphore_exists_in_source(self): + """The module should use asyncio.Semaphore to bound concurrent fetches.""" + source = inspect.getsource(mod) + assert "Semaphore" in source + + def test_semaphore_value_is_reasonable(self): + """The default max_concurrency should be a small positive integer (not unbounded).""" + config = mod.CVEFetchPatchesConfig() + assert config.max_concurrency == 5 \ No newline at end of file diff --git a/src/vuln_analysis/register.py b/src/vuln_analysis/register.py index a6d938609..eaeef6039 100644 --- a/src/vuln_analysis/register.py +++ b/src/vuln_analysis/register.py @@ -13,8 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging -from datetime import datetime +from datetime import datetime, timezone from io import TextIOWrapper from aiq.builder.builder import Builder @@ -24,47 +23,104 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -from vuln_analysis.data_models.input import AgentMorpheusEngineInput -from vuln_analysis.data_models.input import AgentMorpheusInput -from vuln_analysis.data_models.output import AgentMorpheusOutput +from exploit_iq_commons.data_models.common import PipelineMode +from exploit_iq_commons.data_models.checker_status import PackageCheckerStatus, PACKAGE_CHECKER_STATUS_DESCRIPTIONS +from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput +from exploit_iq_commons.data_models.input import AgentMorpheusInput +from exploit_iq_commons.data_models.input import DEFAULT_FAILURE_REASON +from exploit_iq_commons.data_models.info import AgentMorpheusInfo +from vuln_analysis.data_models.output import AgentMorpheusEngineOutput, AgentMorpheusOutput, JustificationOutput, OutputPayload from vuln_analysis.data_models.state import AgentMorpheusEngineState # pylint: disable=unused-import from vuln_analysis.functions import cve_agent -from vuln_analysis.functions import cve_check_vuln_deps from vuln_analysis.functions import cve_checklist from vuln_analysis.functions import cve_fetch_intel from vuln_analysis.functions import cve_file_output -from vuln_analysis.functions import cve_generate_vdbs +from vuln_analysis.functions import cve_clone_and_deps +from vuln_analysis.functions import cve_segmentation from vuln_analysis.functions import cve_http_output from vuln_analysis.functions import cve_justify +from vuln_analysis.functions import cve_package_code_agent +from vuln_analysis.functions import cve_checker_segmentation +from vuln_analysis.functions import cve_source_acquisition from vuln_analysis.functions import cve_process_sbom +from vuln_analysis.functions import cve_verify_vuln_package from vuln_analysis.functions import cve_summarize +from vuln_analysis.functions import cve_checker_report +from vuln_analysis.functions import cve_build_agent +from vuln_analysis.functions import cve_fetch_patches +from vuln_analysis.functions import cve_generate_cvss +from vuln_analysis.functions import cve_generate_vex +from vuln_analysis.functions import health_endpoint from vuln_analysis.tools import lexical_full_search +# This is actually registers the tool in the type registry of NAT! +from vuln_analysis.tools import transitive_code_search +from vuln_analysis.tools import container_image_analysis_data from vuln_analysis.tools import local_vdb from vuln_analysis.tools import serp +from vuln_analysis.tools import configuration_scanner +from vuln_analysis.tools import import_usage_analyzer +from vuln_analysis.tools import source_grep +from vuln_analysis.utils.error_handling_decorator import catch_pipeline_errors_async # pylint: enable=unused-import -from vuln_analysis.utils.llm_engine_utils import postprocess_engine_output +from vuln_analysis.utils.llm_engine_utils import postprocess_engine_output, finalize_preprocess_engine_input from vuln_analysis.utils.llm_engine_utils import preprocess_engine_input +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id -logger = logging.getLogger(__name__) +logger = LoggingFactory.get_agent_logger(__name__) class CVEAgentWorkflowConfig(FunctionBaseConfig, name="cve_agent"): """ Defines the workflow function for determining the impact of a documented CVEs on a specific project or container. """ - cve_generate_vdbs_name: str = Field(description="Function name to generate vector databases") + cve_clone_and_deps_name: str = Field( + description="Function name for early clone and dependency installation.", + ) + cve_segmentation_name: str = Field( + description="Function name for late segmentation (VDB and code index building).", + ) cve_fetch_intel_name: str = Field(description="Function name to fetch intel") + cve_calculate_intel_score_name: str = Field(description="Function name to calculate intel score") cve_process_sbom_name: str = Field(description="Function name to process SBOMs") - cve_check_vuln_deps_name: str = Field(description="Function name to check vulnerable dependencies") + cve_verify_vuln_package_name: str | None = Field( + default=None, + description="Function name to verify vulnerable package presence in source dependencies" + ) cve_checklist_name: str = Field(description="Function name to generate checklist") cve_agent_executor_name: str = Field(description="Function name to run CVE agent on checklist") cve_summarize_name: str = Field(description="Function name to generate summary") cve_justify_name: str = Field(description="Function to generate justifications for each CVE") + cve_generate_vex_name: str = Field(description="Function name to generate VEX for vulnerable components") + cve_generate_cvss_name: str = Field(description="Function name to generate CVSS") cve_output_config_name: str | None = Field(default=None, description="Function to output workflow results " - "(e.g. cve_file_output, cve_http_output). " - " If None, only prints to console") + "(e.g. cve_file_output, cve_http_output). " + " If None, only prints to console") + cve_source_acquisition_name: str | None = Field( + default=None, + description="Function name for source acquisition (downloads source containers, locates package sources)", + ) + cve_checker_segmentation_name: str | None = Field( + default=None, + description="Function name for scoped code indexing of extracted checker sources (Tantivy only)", + ) + cve_package_code_agent_name: str | None = Field( + default=None, + description="Function name for the Level 1 Package Code Agent (source-level CVE investigation)", + ) + cve_checker_report_name: str | None = Field( + default=None, + description="Function name for the checker report generation (L1/L2 report synthesis)", + ) + cve_build_agent_name: str | None = Field( + default=None, + description="Function name for the Level 2 Build Agent (build compilation and hardening check)", + ) + cve_fetch_patches_name: str | None = Field( + default=None, + description="Function name for fetching vulnerability fix patches from intel references and OSV", + ) description: str = Field(default="Vulnerability analysis for container security workflow", description="Workflow function description") @@ -77,41 +133,127 @@ async def cve_agent_workflow(config: CVEAgentWorkflowConfig, builder: Builder): from langgraph.graph import StateGraph # Access functions that will be used in this workflow - cve_generate_vdbs_fn = builder.get_function(name=config.cve_generate_vdbs_name) cve_fetch_intel_fn = builder.get_function(name=config.cve_fetch_intel_name) + cve_calculate_intel_score_fn = builder.get_function(name=config.cve_calculate_intel_score_name) cve_process_sbom_fn = builder.get_function(name=config.cve_process_sbom_name) - cve_check_vuln_deps_fn = builder.get_function(name=config.cve_check_vuln_deps_name) + cve_verify_vuln_package_fn = ( + builder.get_function(name=config.cve_verify_vuln_package_name) + if config.cve_verify_vuln_package_name else None + ) cve_checklist_fn = builder.get_function(name=config.cve_checklist_name) cve_agent_executor_fn = builder.get_function(name=config.cve_agent_executor_name) cve_summary_fn = builder.get_function(name=config.cve_summarize_name) cve_justify_fn = builder.get_function(name=config.cve_justify_name) + cve_generate_vex_fn = builder.get_function(name=config.cve_generate_vex_name) + cve_generate_cvss_fn = builder.get_function(name=config.cve_generate_cvss_name) cve_output_fn = builder.get_function(name=config.cve_output_config_name) if config.cve_output_config_name else None + cve_source_acquisition_fn = ( + builder.get_function(name=config.cve_source_acquisition_name) + if config.cve_source_acquisition_name else None + ) + cve_checker_segmentation_fn = ( + builder.get_function(name=config.cve_checker_segmentation_name) + if config.cve_checker_segmentation_name else None + ) + cve_package_code_agent_fn = ( + builder.get_function(name=config.cve_package_code_agent_name) + if config.cve_package_code_agent_name else None + ) + cve_checker_report_fn = ( + builder.get_function(name=config.cve_checker_report_name) + if config.cve_checker_report_name else None + ) + cve_build_agent_fn = ( + builder.get_function(name=config.cve_build_agent_name) + if config.cve_build_agent_name else None + ) + cve_fetch_patches_fn = ( + builder.get_function(name=config.cve_fetch_patches_name) + if config.cve_fetch_patches_name else None + ) + cve_clone_and_deps_fn = builder.get_function(name=config.cve_clone_and_deps_name) + cve_segmentation_fn = builder.get_function(name=config.cve_segmentation_name) # Define langgraph node functions + @catch_pipeline_errors_async async def add_start_time_node(state: AgentMorpheusInput) -> AgentMorpheusInput: """Adds the start time to the input""" - state.scan.started_at = datetime.now().isoformat() + state.scan.started_at = datetime.now(timezone.utc).isoformat() return state - async def generate_vdbs_node(state: AgentMorpheusInput) -> AgentMorpheusEngineInput: - """Generates VDBs based on CVE input""" - - return await cve_generate_vdbs_fn.ainvoke(state.model_dump()) - + @catch_pipeline_errors_async async def fetch_intel_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: """Fetch intel for CVE input""" return await cve_fetch_intel_fn.ainvoke(state.model_dump()) + @catch_pipeline_errors_async + async def calculate_intel_score_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Calculate score for intel source""" + + return await cve_calculate_intel_score_fn.ainvoke(state.model_dump()) + + + @catch_pipeline_errors_async async def process_sbom_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: """Process SBOMs for CVE input""" return await cve_process_sbom_fn.ainvoke(state.model_dump()) - async def check_vuln_deps_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: - """Check for vulnerable dependencies""" + @catch_pipeline_errors_async + async def verify_vuln_package_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Verify vulnerable package presence in source dependencies""" + if cve_verify_vuln_package_fn: + return await cve_verify_vuln_package_fn.ainvoke(state.model_dump()) + return state - return await cve_check_vuln_deps_fn.ainvoke(state.model_dump()) + # --- Split pipeline nodes (clone_and_deps + segmentation) --- + + @catch_pipeline_errors_async + async def clone_and_deps_node(state: AgentMorpheusInput) -> AgentMorpheusEngineInput: + """Clone repositories and install dependencies (early phase).""" + return await cve_clone_and_deps_fn.ainvoke(state.model_dump()) + + @catch_pipeline_errors_async + async def segmentation_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Build VDBs and code indexes from cloned repos (late phase).""" + result = await cve_segmentation_fn.ainvoke(state.model_dump()) + result_dict = result.model_dump() + result_dict["code_index_success"] = result.input.code_index_success + result_dict["failure_reason"] = result.input.failure_reason + return result_dict + + def route_after_verify_vuln_package(state: AgentMorpheusEngineInput) -> str: + """Route to segmentation if any CVE is vulnerable, else skip to llm_engine.""" + vuln_deps = state.info.vulnerable_dependencies + if vuln_deps is None: + return "segmentation" + any_vulnerable = any(len(v.vulnerable_sbom_packages) > 0 for v in vuln_deps) + return "segmentation" if any_vulnerable else "llm_engine" + + def route_after_clone_and_deps(state: AgentMorpheusEngineInput | AgentMorpheusInput) -> str: + """Route to fetch_intel on success, or failure if clone failed. + + Note: state may be AgentMorpheusEngineInput (has .input.failure_reason) or + AgentMorpheusInput (has .failure_reason directly) depending on LangGraph's + state propagation behavior. + """ + if isinstance(state, AgentMorpheusEngineInput): + failure_reason = state.input.failure_reason + else: + failure_reason = state.failure_reason + + if failure_reason and failure_reason != DEFAULT_FAILURE_REASON: + return "failure" + return "fetch_intel" + + async def failure_node(state: AgentMorpheusEngineInput) -> AgentMorpheusOutput: + """Handles pipeline failure (e.g., clone/install failed).""" + return AgentMorpheusOutput( + input=state.input, + info=state.info, + output=OutputPayload(analysis=[], vex=None), + ) async def checklist_node(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: """Generates a checklist based on CVE input""" @@ -133,17 +275,179 @@ async def justify_node(state: AgentMorpheusEngineState) -> AgentMorpheusEngineSt return await cve_justify_fn.ainvoke(state.model_dump()) + async def generate_vex_node(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: + """Generates VEX for vulnerable components""" + + return await cve_generate_vex_fn.ainvoke(state.model_dump()) + + async def generate_cvss_node(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: + """Generates CVSS for the results of the execution""" + + return await cve_generate_cvss_fn.ainvoke(state.model_dump()) + + async def fetch_patches_node(state: AgentMorpheusEngineState) -> AgentMorpheusEngineState: + """Fetches vulnerability fix patches from intel references and OSV.""" + + return await cve_fetch_patches_fn.ainvoke(state.model_dump()) + + @catch_pipeline_errors_async async def add_completed_time_node(state: AgentMorpheusOutput) -> AgentMorpheusOutput: """Adds the completed time to the output""" - state.input.scan.completed_at = datetime.now().isoformat() + state.input.scan.completed_at = datetime.now(timezone.utc).isoformat() return state + @catch_pipeline_errors_async async def output_results_node(state: AgentMorpheusOutput) -> AgentMorpheusOutput: """Outputs results using configured output function""" return await cve_output_fn.ainvoke(state.model_dump()) if cve_output_fn else state - # define langgraph + # --- Package checker path nodes --- + + @catch_pipeline_errors_async + async def checker_init_state_node(state: AgentMorpheusInput) -> AgentMorpheusEngineInput: + """Bridges AgentMorpheusInput -> AgentMorpheusEngineInput with empty info (skips VDB generation).""" + return AgentMorpheusEngineInput(input=state, info=AgentMorpheusInfo()) + + @catch_pipeline_errors_async + async def checker_fetch_intel_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Fetch intel for CVE input (package checker path). Reuses the same fetch_intel function.""" + return await cve_fetch_intel_fn.ainvoke(state.model_dump()) + + @catch_pipeline_errors_async + async def checker_calculate_intel_score_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Calculate intel score for CVE input (package checker path).""" + return await cve_calculate_intel_score_fn.ainvoke(state.model_dump()) + + @catch_pipeline_errors_async + async def source_acquisition_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Acquires source code for the target package (source containers, git fallback).""" + if cve_source_acquisition_fn: + state = await cve_source_acquisition_fn.ainvoke(state.model_dump()) + else: + logger.warning("Source acquisition function not configured, passing state through") + + if state.info.checker_context and state.info.checker_context.status is not None: + logger.info( + "PackageIdentify aggregate status: %s", + state.info.checker_context.status.name, + ) + return state + + @catch_pipeline_errors_async + async def checker_segmentation_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Builds scoped Tantivy code index from extracted checker sources.""" + if cve_checker_segmentation_fn: + state = await cve_checker_segmentation_fn.ainvoke(state.model_dump()) + else: + logger.warning("Checker segmentation not configured, skipping indexing") + return state + + @catch_pipeline_errors_async + async def l1_code_agent_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Level 1 Package Code Agent: investigates CVEs using extracted source and Tantivy code index. + + Returns AgentMorpheusEngineInput with l1_result populated on checker_context. + """ + if cve_package_code_agent_fn: + return await cve_package_code_agent_fn.ainvoke(state.model_dump()) + logger.warning("Package code agent function not configured, passing state through") + return state + + def route_after_l1(state: AgentMorpheusEngineInput) -> str: + """Route to L2 Build Agent if vulnerable or uncertain, else to report generation.""" + ctx = state.info.checker_context + if ctx and ctx.l1_result: + verdict = ctx.l1_result.preliminary_verdict + if verdict in ("vulnerable", "uncertain"): + return "l2_build_agent" + return "generate_report" + + @catch_pipeline_errors_async + async def l2_build_agent_node(state: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Level 2 Build Agent: BuildCompilationCheck + HardeningCheck. + + Returns AgentMorpheusEngineInput with l2_result populated on checker_context. + """ + if cve_build_agent_fn: + return await cve_build_agent_fn.ainvoke(state.model_dump()) + logger.warning("Build agent function not configured, passing state through") + return state + + @catch_pipeline_errors_async + async def generate_report_node(state: AgentMorpheusEngineInput) -> AgentMorpheusOutput: + """Generate the final checker report from L1/L2 investigation results.""" + if cve_checker_report_fn: + return await cve_checker_report_fn.ainvoke(state.model_dump()) + logger.warning("Checker report function not configured, producing empty output") + return AgentMorpheusOutput( + input=state.input, + info=state.info, + output=OutputPayload(analysis=[], vex=None), + ) + + @catch_pipeline_errors_async + async def checker_early_exit_node(state: AgentMorpheusEngineInput) -> AgentMorpheusOutput: + """Produces a proper output when source_acquisition exits with a non-OK status.""" + ctx = state.info.checker_context + status = ctx.status if ctx else None + + # Build detailed reason from identification result when available + detailed_reason = "" + if ctx and ctx.identify_result and ctx.identify_result.conclusion_reason: + detailed_reason = ctx.identify_result.conclusion_reason + + base_reason = ( + PACKAGE_CHECKER_STATUS_DESCRIPTIONS[status] + if status is not None and status in PACKAGE_CHECKER_STATUS_DESCRIPTIONS + else f"Checker exited early with status {status}" + ) + + # Combine base reason with detailed conclusion if available + reason = f"{base_reason}. {detailed_reason}" if detailed_reason else base_reason + logger.info("checker_early_exit: status=%s reason=%s", status, reason) + def _get_justification_label(s: PackageCheckerStatus | None) -> str: + if s in (PackageCheckerStatus.PKG_IDENT_NOT_VUL, PackageCheckerStatus.PKG_IDENT_CVE_MISMATCH): + return "not_vulnerable" + if s == PackageCheckerStatus.PKG_INTEL_LOW_SCORE: + return "poor_quality_intel" + return "error" + + analysis = [ + AgentMorpheusEngineOutput( + vuln_id=v.vuln_id, + checklist=[], + summary=reason, + justification=JustificationOutput( + label=_get_justification_label(status), + reason=reason, + status="FALSE" if status in ( + PackageCheckerStatus.PKG_IDENT_NOT_VUL, + PackageCheckerStatus.PKG_IDENT_CVE_MISMATCH, + ) else "UNKNOWN", + ), + intel_score=0, + cvss=None, + ) + for v in state.input.scan.vulns + ] + return AgentMorpheusOutput( + input=state.input, info=state.info, + output=OutputPayload(analysis=analysis, vex=None), + ) + + def route_after_source_acquisition(state: AgentMorpheusEngineInput): + """Route to checker_segmentation (happy path) or early exit on non-OK status.""" + ctx = state.info.checker_context + if ctx and ctx.status == PackageCheckerStatus.OK: + return "checker_segmentation" + return "checker_early_exit" + + def route_after_add_start_time(state: AgentMorpheusInput): + """Route to full pipeline or package checker based on pipeline_mode.""" + if state.image.pipeline_mode == PipelineMode.PACKAGE_CHECKER: + return "checker_init_state" + return "clone_and_deps" # build llm engine subgraph subgraph_builder = StateGraph(AgentMorpheusEngineState) @@ -151,45 +455,128 @@ async def output_results_node(state: AgentMorpheusOutput) -> AgentMorpheusOutput subgraph_builder.add_node("agent_executor", agent_executor_node) subgraph_builder.add_node("summarize", summarize_node) subgraph_builder.add_node("justify", justify_node) + subgraph_builder.add_node("generate_vex", generate_vex_node) + subgraph_builder.add_node("generate_cvss", generate_cvss_node) subgraph_builder.add_edge(START, "checklist") - subgraph_builder.add_edge("checklist", "agent_executor") + if cve_fetch_patches_fn: + subgraph_builder.add_node("fetch_patches", fetch_patches_node) + subgraph_builder.add_edge("checklist", "fetch_patches") + subgraph_builder.add_edge("fetch_patches", "agent_executor") + else: + subgraph_builder.add_edge("checklist", "agent_executor") subgraph_builder.add_edge("agent_executor", "summarize") subgraph_builder.add_edge("summarize", "justify") + subgraph_builder.add_edge("justify", "generate_vex") + subgraph_builder.add_edge("generate_vex", "generate_cvss") subgraph = subgraph_builder.compile() + @catch_pipeline_errors_async async def call_llm_engine_subgraph_node(message: AgentMorpheusEngineInput): - + trace_id.set(message.input.scan.id) subgraph_input = preprocess_engine_input(message) - results = await subgraph.ainvoke(subgraph_input) - subgraph_output = AgentMorpheusEngineState(**results) + subgraph_input = finalize_preprocess_engine_input(message, subgraph_input, builder) + if len(subgraph_input.cve_intel) > 0: + results = await subgraph.ainvoke(subgraph_input) + subgraph_output = AgentMorpheusEngineState(**results) + else: + subgraph_output = subgraph_input output = postprocess_engine_output(message=message, result=subgraph_output) return output # build parent graph graph_builder = StateGraph(AgentMorpheusOutput, input=AgentMorpheusInput) graph_builder.add_node("add_start_time", add_start_time_node) - graph_builder.add_node("generate_vdbs", generate_vdbs_node) graph_builder.add_node("fetch_intel", fetch_intel_node) + graph_builder.add_node("calculate_intel_score_node", calculate_intel_score_node) graph_builder.add_node("process_sbom", process_sbom_node) - graph_builder.add_node("check_vuln_deps", check_vuln_deps_node) + graph_builder.add_node("verify_vuln_package", verify_vuln_package_node) + graph_builder.add_node("clone_and_deps", clone_and_deps_node) + graph_builder.add_node("segmentation", segmentation_node) graph_builder.add_node("llm_engine", call_llm_engine_subgraph_node) graph_builder.add_node("add_completed_time", add_completed_time_node) graph_builder.add_node("output_results", output_results_node) + graph_builder.add_node("failure", failure_node) +# -- Package checker nodes -- + graph_builder.add_node("checker_init_state", checker_init_state_node) + graph_builder.add_node("checker_fetch_intel", checker_fetch_intel_node) + graph_builder.add_node("checker_calculate_intel_score", checker_calculate_intel_score_node) + graph_builder.add_node("source_acquisition", source_acquisition_node) + graph_builder.add_node("checker_early_exit", checker_early_exit_node) + graph_builder.add_node("checker_segmentation", checker_segmentation_node) + graph_builder.add_node("l1_code_agent", l1_code_agent_node) + graph_builder.add_node("l2_build_agent", l2_build_agent_node) + graph_builder.add_node("generate_report", generate_report_node) graph_builder.add_edge(START, "add_start_time") - graph_builder.add_edge("add_start_time", "generate_vdbs") - graph_builder.add_edge("generate_vdbs", "fetch_intel") - graph_builder.add_edge("fetch_intel", "process_sbom") - graph_builder.add_edge("process_sbom", "check_vuln_deps") - graph_builder.add_edge("check_vuln_deps", "llm_engine") + # Conditional: route to full pipeline or package checker after add_start_time + graph_builder.add_conditional_edges( + "add_start_time", + route_after_add_start_time, + { + "clone_and_deps": "clone_and_deps", + "checker_init_state": "checker_init_state", + }, + ) + + # Split pipeline: clone_and_deps -> [check success] -> fetch_intel -> ... -> verify_vuln_package -> [conditional] -> segmentation -> llm_engine + graph_builder.add_conditional_edges( + "clone_and_deps", + route_after_clone_and_deps, + { + "fetch_intel": "fetch_intel", + "failure": "failure", + }, + ) + graph_builder.add_edge("failure", "add_completed_time") + graph_builder.add_edge("fetch_intel", "calculate_intel_score_node") + graph_builder.add_edge("calculate_intel_score_node", "process_sbom") + graph_builder.add_edge("process_sbom", "verify_vuln_package") + graph_builder.add_conditional_edges( + "verify_vuln_package", + route_after_verify_vuln_package, + { + "segmentation": "segmentation", + "llm_engine": "llm_engine", + }, + ) + graph_builder.add_edge("segmentation", "llm_engine") graph_builder.add_edge("llm_engine", "add_completed_time") + + # Package checker path + graph_builder.add_edge("checker_init_state", "checker_fetch_intel") + graph_builder.add_edge("checker_fetch_intel", "checker_calculate_intel_score") + graph_builder.add_edge("checker_calculate_intel_score", "source_acquisition") + + graph_builder.add_conditional_edges( + "source_acquisition", + route_after_source_acquisition, + { + "checker_segmentation": "checker_segmentation", + "checker_early_exit": "checker_early_exit", + }, + ) + graph_builder.add_edge("checker_early_exit", "add_completed_time") + graph_builder.add_edge("checker_segmentation", "l1_code_agent") + graph_builder.add_conditional_edges( + "l1_code_agent", + route_after_l1, + { + "l2_build_agent": "l2_build_agent", + "generate_report": "generate_report", + }, + ) + graph_builder.add_edge("l2_build_agent", "generate_report") + graph_builder.add_edge("generate_report", "add_completed_time") + + # Shared tail graph_builder.add_edge("add_completed_time", "output_results") graph_builder.add_edge("output_results", END) graph = graph_builder.compile() - + #graph.get_graph().draw_mermaid_png(output_file_path="checker_flow.png") + def convert_str_to_agent_morpheus_input(input: str) -> AgentMorpheusInput: - logger.debug("Converting input to AgentMorpheusInput: %s", input) + logger.debug("Converting JSON string input to AgentMorpheusInput (length: %d)", len(input)) try: return AgentMorpheusInput.model_validate_json(input) except Exception as e: @@ -197,7 +584,7 @@ def convert_str_to_agent_morpheus_input(input: str) -> AgentMorpheusInput: raise e def convert_textio_to_agent_morpheus_input(input: TextIOWrapper) -> AgentMorpheusInput: - logger.debug("Converting input to AgentMorpheusInput: %s", input) + logger.debug("Converting TextIOWrapper input to AgentMorpheusInput") try: data = input.read() return AgentMorpheusInput.model_validate_json(data) @@ -207,7 +594,7 @@ def convert_textio_to_agent_morpheus_input(input: TextIOWrapper) -> AgentMorpheu raise e def convert_agent_morpheus_output_to_str(output: AgentMorpheusOutput) -> str: - logger.debug("Converting AgentMorpheusOutput to str: %s", output) + logger.debug("Converting AgentMorpheusOutput to JSON string") try: return output.model_dump_json() except Exception as e: diff --git a/src/vuln_analysis/runtime_context.py b/src/vuln_analysis/runtime_context.py new file mode 100644 index 000000000..adca2360e --- /dev/null +++ b/src/vuln_analysis/runtime_context.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A shared runtime context for passing workflow state to tools and functions. +# This must be imported by all modules that need to read or set the active state. + +import contextvars + +# Holds the current AgentMorpheusEngineState for the active task +ctx_state = contextvars.ContextVar("ctx_state", default="default_value") + +# Source scope for CU agent tools (Docs Semantic Search, Code Keyword Search). +# List of path substrings; tools keep results whose path matches any entry. +# None means no scoping (search all sources). +cu_source_scope = contextvars.ContextVar("cu_source_scope", default=None) \ No newline at end of file diff --git a/src/vuln_analysis/tools/brew_downloader.py b/src/vuln_analysis/tools/brew_downloader.py new file mode 100644 index 000000000..fb36b5d50 --- /dev/null +++ b/src/vuln_analysis/tools/brew_downloader.py @@ -0,0 +1,353 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Production Brew Downloader -- fetch SRPMs, build logs, and binary RPMs from Brew (Koji). + +Evolved from the PoC at docs/package_analyzer/standalone_checker/brew/brew_downloader.py. +Storage is split: SRPMs go to the shared rpms/ cache, everything else to checker-specific dirs. +""" + +from __future__ import annotations + +import os +import re +import shutil +from enum import Enum +from pathlib import Path + +import koji +import requests +import yaml + +from exploit_iq_commons.data_models.checker_status import AcquiredArtifacts +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.source_rpm_downloader import SourceRPMDownloader + +logger = LoggingFactory.get_agent_logger(__name__) + +_CONFIGS_DIR = Path(__file__).resolve().parent.parent / "configs" / "brew" +# --------------------------------------------------------------------------- +# Exceptions +# --------------------------------------------------------------------------- + +class BrewDownloaderError(Exception): + """Base for all Brew downloader errors.""" + + +class BrewConnectionError(BrewDownloaderError): + """Raised when the Brew hub is unreachable or session creation fails.""" + + +class BrewBuildNotFoundError(BrewDownloaderError): + """Raised when getBuild returns None for the requested NVR.""" + + +class BrewDownloadError(BrewDownloaderError): + """Raised when an HTTP download of an artifact fails.""" + + +class BrewProfileNotImplementedError(BrewDownloaderError): + """Raised when a profile type is not yet implemented.""" + + +# --------------------------------------------------------------------------- +# Profile types +# --------------------------------------------------------------------------- + +class BrewProfileType(Enum): + INTERNAL = "internal" + EXTERNAL = "external" + + +_PROFILE_PATHS: dict[BrewProfileType, Path] = { + BrewProfileType.INTERNAL: _CONFIGS_DIR / "internal-user-profile.yml", + BrewProfileType.EXTERNAL: _CONFIGS_DIR / "external-user-profile.yml", +} + + +_ENV_DEFAULT_RE = re.compile(r"^\$\{([^:}]+):-([^}]+)\}$") + + +def resolve_brew_profile(config_value: str) -> BrewProfileType: + """Resolve brew profile from deployment config. + + Supports ``rpm_user_type: ${RPM_USER_TYPE:-internal}`` in YAML (expanded at + deploy, or parsed here when the literal is still present). ``RPM_USER_TYPE`` + environment variable always wins when set. + """ + env_override = os.environ.get("RPM_USER_TYPE") + if env_override is not None: + name = env_override.strip().lower() + else: + raw = config_value.strip() + match = _ENV_DEFAULT_RE.match(raw) + if match: + var_name, default = match.group(1), match.group(2) + name = os.environ.get(var_name, default).strip().lower() + else: + name = raw.lower() + try: + return BrewProfileType(name) + except ValueError as exc: + raise BrewProfileNotImplementedError( + f"Unknown brew profile '{name}' (expected internal or external)" + ) from exc + + +# --------------------------------------------------------------------------- +# BrewDownloader +# --------------------------------------------------------------------------- + +class BrewDownloader: + """Downloads RPM artifacts and build logs from Brew (Koji) using a profile YAML. + + Storage destinations: + - SRPMs -> ``rpm_cache_dir/{NVR}.src.rpm`` (shared with SourceRPMDownloader) + - Build logs -> ``checker_dir/logs/{arch}/build.log`` + - Binary RPMs -> ``checker_dir/binaries/{NVR}/{NVRA}.rpm`` + """ + + def __init__(self, profile_type: BrewProfileType, rpm_cache_dir: str, checker_dir: str) -> None: + profile_path = _PROFILE_PATHS.get(profile_type) + if profile_path is None: + raise BrewProfileNotImplementedError( + f"Profile type '{profile_type.value}' is not configured" + ) + self._profile = self._load_profile(str(profile_path)) + + hosts = self._profile["hosts"]["rpm"] + self._brew_hub: str = hosts["brew_hub"] + self._brew_download: str = hosts["brew_download"] + self._default_arch: str = self._profile.get("default_arch", "x86_64") + self._download_binary_rpm_enabled: bool = self._profile.get("download_binary_rpm", False) + self._auto_fetch_build_log: bool = self._profile.get("build_log", {}).get("auto_fetch", True) + self._ssl_verify: bool = self._profile.get("ssl_verify", True) + self._ssl_verify_path: str | None = self._profile.get("verify_path") + self._http_verify: bool | str = self._resolve_http_verify() + + self._rpm_cache_dir = Path(rpm_cache_dir) + self._rpm_cache_dir.mkdir(parents=True, exist_ok=True) + + self._checker_dir = Path(checker_dir) + self._checker_dir.mkdir(parents=True, exist_ok=True) + + self._session: koji.ClientSession | None = None + self._pathinfo: koji.PathInfo | None = None + self._http = requests.Session() + + # -- properties -------------------------------------------------------- + + @property + def download_binary_rpm_enabled(self) -> bool: + return self._download_binary_rpm_enabled + + @property + def default_arch(self) -> str: + return self._default_arch + + @property + def auto_fetch_build_log(self) -> bool: + return self._auto_fetch_build_log + + # -- setup ------------------------------------------------------------- + + @staticmethod + def _load_profile(path: str) -> dict: + with open(path, encoding="utf-8") as fh: + return yaml.safe_load(fh) + + def _resolve_http_verify(self) -> bool | str: + if not self._ssl_verify: + return False + if self._ssl_verify_path: + return self._ssl_verify_path + return True + + def connect(self) -> None: + """Create a Koji client session and PathInfo helper from the profile.""" + logger.info("Connecting to Brew hub: %s", self._brew_hub) + try: + opts: dict = {} + if not self._ssl_verify: + opts["no_ssl_verify"] = True + elif self._ssl_verify_path: + opts["serverca"] = self._ssl_verify_path + self._session = koji.ClientSession(self._brew_hub, opts=opts) + self._pathinfo = koji.PathInfo(topdir=self._brew_download) + self._http.verify = self._http_verify + + except Exception as exc: + raise BrewConnectionError( + f"Failed to connect to Brew hub {self._brew_hub}: {exc}" + ) from exc + + # -- query ------------------------------------------------------------- + + def search_build(self, name: str, version: str, release: str) -> dict | None: + """Look up a build by NVR. Returns the build-info dict or ``None``.""" + nvr = f"{name}-{version}-{release}" + logger.info("Searching for build: %s", nvr) + build = self._session.getBuild(nvr) + if build is None: + logger.warning("Build not found: %s", nvr) + return None + logger.info( + "Found build %s (id=%s, volume=%s, task=%s)", + build["nvr"], build["id"], build.get("volume_name"), build.get("task_id"), + ) + return build + + # -- downloads --------------------------------------------------------- + + def _download_file(self, url: str, dest: Path) -> Path: + """Stream-download *url* to *dest*. Returns the destination path.""" + logger.info("Downloading %s -> %s", url, dest) + dest.parent.mkdir(parents=True, exist_ok=True) + try: + resp = self._http.get( + url, stream=True, timeout=120, verify=self._http_verify + ) + resp.raise_for_status() + except requests.RequestException as exc: + raise BrewDownloadError(f"Failed to download {url}: {exc}") from exc + with open(dest, "wb") as fh: + for chunk in resp.iter_content(chunk_size=1 << 18): # 256 KB + fh.write(chunk) + logger.info("Saved %s (%d bytes)", dest.name, dest.stat().st_size) + return dest + + def _get_srpm_url(self, build: dict) -> str: + """Compute the download URL for the source RPM of *build*.""" + rpms = self._session.listRPMs(buildID=build["id"], arches="src") + if not rpms: + raise BrewDownloadError(f"No source RPM found for build {build['nvr']}") + rpm_info = rpms[0] + return f"{self._pathinfo.build(build)}/{self._pathinfo.rpm(rpm_info)}" + + def download_srpm(self, build: dict) -> Path: + """Download the .src.rpm for *build* into the shared RPM cache. + + Skips the download when the destination file already exists and is non-empty. + """ + rpms = self._session.listRPMs(buildID=build["id"], arches="src") + if not rpms: + raise BrewDownloadError(f"No source RPM found for build {build['nvr']}") + + rpm_info = rpms[0] + dest = self._rpm_cache_dir / f"{rpm_info['nvr']}.src.rpm" + + if dest.exists() and dest.stat().st_size > 0: + logger.info("SRPM cache hit: %s", dest) + return dest + + url = f"{self._pathinfo.build(build)}/{self._pathinfo.rpm(rpm_info)}" + return self._download_file(url, dest) + + def download_build_log(self, build: dict, arch: str | None = None) -> Path: + """Download ``build.log`` for the given arch into ``checker_dir/logs/{arch}/``.""" + arch = arch or self._default_arch + url = f"{self._pathinfo.build(build)}/data/logs/{arch}/build.log" + dest = self._checker_dir / "logs" / arch / "build.log" + return self._download_file(url, dest) + + def try_download_build_log(self, build: dict, arch: str | None = None) -> Path | None: + """Download build log when ``build_log.auto_fetch`` is enabled; return None on failure.""" + if not self._auto_fetch_build_log: + return None + arch = arch or self._default_arch + try: + return self.download_build_log(build, arch) + except BrewDownloadError as exc: + logger.warning( + "Build log unavailable for %s arch=%s: %s", + build.get("nvr", "?"), + arch, + exc, + ) + return None + + def download_binary_rpm(self, build: dict, arch: str | None = None) -> Path | None: + """Download all binary RPMs for the given arch (excludes debuginfo/debugsource). + + Saves to ``checker_dir/binaries/{NVR}/``. Returns an empty list when no + matching RPMs are found. + """ + arch = arch or self._default_arch + rpms = self._session.listRPMs(buildID=build["id"], arches=arch) + if not rpms: + logger.warning("No %s RPMs found for build %s", arch, build["nvr"]) + return None + + nvr = build["nvr"] + build_dir = self._checker_dir / "binaries" / nvr + + downloaded: list[Path] = [] + for rpm_info in rpms: + rpm_name: str = rpm_info["name"] + if rpm_name.endswith(("-debuginfo", "-debugsource")): + continue + url = f"{self._pathinfo.build(build)}/{self._pathinfo.rpm(rpm_info)}" + nvra = f"{rpm_info['name']}-{rpm_info['version']}-{rpm_info['release']}.{rpm_info['arch']}" + dest = build_dir / f"{nvra}.rpm" + self._download_file(url, dest) + downloaded.append(dest) + return build_dir + + def download_patched_srpm(self, name: str, version: str, release: str) -> Path | None: + """Download the SRPM for a patched version (from CVE fix info). + + Returns the cached SRPM path, or ``None`` if the patched build is not + found in Brew. + """ + build = self.search_build(name, version, release) + if build is None: + return None + return self.download_srpm(build) + + def download_patched_srpm_by_nevra(self, nevra: str) -> Path | None: + """Download the SRPM for a patched version (from NEVRA). + + Returns the cached SRPM path, or ``None`` if the patched build is not + found in Brew. + """ + build = self._session.getBuild(nevra) + if build is None: + logger.warning("Build not found: %s", nevra) + return None + logger.info( + "Found build %s (id=%s, volume=%s, task=%s)", + build["nvr"], build["id"], build.get("volume_name"), build.get("task_id"), + ) + return self.download_srpm(build) + + def download_target_artifacts(self, name: str, version: str, release: str, arch: str) -> AcquiredArtifacts | None: + artifacts = AcquiredArtifacts() + build = self.search_build(name, version, release) + if build is None: + raise BrewBuildNotFoundError(f"Build not found for {name}-{version}-{release}") + + artifacts.source_url = self._get_srpm_url(build) + cache_srpm_path = self.download_srpm(build) + + srpm_target_path = self._checker_dir / "source" + srpm_target_path.mkdir(parents=True, exist_ok=True) + shutil.copy2(cache_srpm_path, srpm_target_path) + SourceRPMDownloader.extract_src_rpm(cache_srpm_path, srpm_target_path) + artifacts.srpm_path = srpm_target_path + + artifacts.build_log_path = self.try_download_build_log(build, arch) + if self._download_binary_rpm_enabled: + artifacts.binary_rpm_path = self.download_binary_rpm(build, arch) + return artifacts diff --git a/src/vuln_analysis/tools/configuration_scanner.py b/src/vuln_analysis/tools/configuration_scanner.py new file mode 100644 index 000000000..a9801d450 --- /dev/null +++ b/src/vuln_analysis/tools/configuration_scanner.py @@ -0,0 +1,249 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import os +import re +from collections import OrderedDict +from pathlib import Path + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.git_utils import get_repo_path_with_ref +from exploit_iq_commons.utils.data_utils import DEFAULT_GIT_DIRECTORY +from vuln_analysis.utils.error_handling_decorator import catch_tool_errors +from vuln_analysis.utils.source_classification import is_dependency_path, filter_by_source_scope, format_app_dep_output + +CONFIGURATION_SCANNER = "configuration_scanner" + +logger = LoggingFactory.get_agent_logger(__name__) + + +def format_context_snippet(lines: list[str], match_line: int, context_lines: int) -> str: + """Format source lines around a match, marking the match line with '>'.""" + start = max(0, match_line - context_lines) + end = min(len(lines), match_line + context_lines + 1) + return "\n".join( + f"{'>' if i == match_line else ' '} {i+1}: {lines[i]}" + for i in range(start, end) + ) + +# File patterns and directory names considered as configuration files +CONFIG_FILE_PATTERNS = [ + # Named config files — matched anywhere in the repo + "application.yml", "application.yaml", "application.properties", + "config.yaml", "config.yml", "config.xml", + "settings.toml", "settings.yaml", "settings.yml", + "web.xml", "beans.xml", + "Dockerfile", "Dockerfile.*", "docker-compose*.yml", + # Config-specific extensions — safe to match anywhere + "*.properties", "*.env", "*.conf", "*.ini", +] + +# Directory names that typically contain configuration files +CONFIG_DIR_PATTERNS = ["config", "conf", "conf.d", "etc", "resources"] + +# Avoids per-file regex compilation +_CONFIG_EXTENSIONS = [] +_CONFIG_EXACT_NAMES = [] +_CONFIG_WILDCARD_PATTERNS = [] + +for _pat in CONFIG_FILE_PATTERNS: + if _pat.startswith("*."): + _CONFIG_EXTENSIONS.append(_pat[1:]) + elif "*" in _pat: + _CONFIG_WILDCARD_PATTERNS.append( + re.compile(_pat.replace(".", r"\.").replace("*", ".*"), re.IGNORECASE) + ) + else: + _CONFIG_EXACT_NAMES.append(_pat.lower()) + + +class ConfigurationScannerToolConfig(FunctionBaseConfig, name=CONFIGURATION_SCANNER): + """Scans configuration files for vulnerability-relevant patterns.""" + max_results: int = Field(default=15, description="Maximum config entries to return") + context_lines: int = Field(default=5, description="Lines of context around matches") + + +def _is_config_file(file_path: str) -> bool: + """Check if a file path matches any known configuration file pattern.""" + name = os.path.basename(file_path) + lower_name = name.lower() + + # Check in order of speed: extension → exact name → wildcard regex + if any(lower_name.endswith(ext) for ext in _CONFIG_EXTENSIONS): + return True + if lower_name in _CONFIG_EXACT_NAMES: + return True + if any(p.match(lower_name) for p in _CONFIG_WILDCARD_PATTERNS): + return True + + return False + + +def _is_in_config_dir(file_path: str) -> bool: + """Check if a file resides under a known configuration directory.""" + parts = Path(file_path).parts + return any(p.lower() in CONFIG_DIR_PATTERNS for p in parts) + + +def _collect_config_files(repo_path: str) -> list[tuple[str, str]]: + """Walk repo and collect config file paths and contents. + + Skips .git, __pycache__, node_modules, .tox directories. + Skips files larger than 500KB to avoid memory issues with large generated configs. + """ + config_files = [] + for root, dirs, files in os.walk(repo_path): + dirs[:] = [d for d in dirs if d not in (".git", "__pycache__", "node_modules", ".tox")] + for fname in files: + full_path = os.path.join(root, fname) + rel_path = os.path.relpath(full_path, repo_path) + if _is_config_file(rel_path) or _is_in_config_dir(rel_path): + try: + with open(full_path, "r", errors="ignore") as f: + content = f.read() + if len(content) < 500_000: + config_files.append((rel_path, content)) + else: + logger.warning("Skipping config file %s: size %d exceeds 500KB limit", rel_path, len(content)) + except Exception: + continue + logger.debug("Collected %d config files from %s", len(config_files), repo_path) + return config_files + + +def _count_config_matches(config_files: list[tuple[str, str]], keywords: list[str]) -> int: + """Count total keyword-matching lines across config files (no formatting, no cap).""" + count = 0 + for _, content in config_files: + for line in content.split("\n"): + if any(kw in line.lower() for kw in keywords): + count += 1 + return count + + +def search_config_content( + config_files: list[tuple[str, str]], + keywords: list[str], + max_results: int = 15, + context_lines: int = 5, + source_label: str = "unknown", +) -> list[str]: + """Match keywords against config file contents, returning formatted snippets.""" + matches = [] + for rel_path, content in config_files: + lines = content.split("\n") + for line_num, line in enumerate(lines): + line_lower = line.lower() + if any(kw in line_lower for kw in keywords): + snippet = format_context_snippet(lines, line_num, context_lines) + matches.append(f"--- {rel_path} (source: {source_label}) line {line_num+1} ---\n{snippet}") + if len(matches) >= max_results: + break + if len(matches) >= max_results: + break + return matches + + +@register_function(config_type=ConfigurationScannerToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def configuration_scanner(config: ConfigurationScannerToolConfig, builder: Builder): + from vuln_analysis.runtime_context import ctx_state, cu_source_scope + + _CONFIG_CACHE_MAX_SIZE = 20 + _config_files_cache: OrderedDict[tuple, list[tuple[str, str]]] = OrderedDict() + _repo_locks: dict[tuple, asyncio.Lock] = {} + _repo_locks_guard = asyncio.Lock() + + @catch_tool_errors(CONFIGURATION_SCANNER) + async def _arun(query: str) -> str: + workflow_state = ctx_state.get() + source_infos = workflow_state.original_input.input.image.source_info + source_scope = cu_source_scope.get() + logger.debug("Configuration scanner: searching for '%s' across %d source(s), scope=%s", + query, len(source_infos), source_scope) + + keywords = [w.strip().lower() for w in re.split(r"[,\s]+", query) if len(w.strip()) >= 2] + all_app_configs = [] + all_dep_configs = [] + + for si in source_infos: + if not hasattr(si, "git_repo"): + continue + repo_path = get_repo_path_with_ref(Path(DEFAULT_GIT_DIRECTORY), si.git_repo, si.ref) + if not repo_path.is_dir(): + continue + + repo_key = (si.git_repo, si.ref) + if repo_key in _config_files_cache: + async with _repo_locks_guard: + _config_files_cache.move_to_end(repo_key) + else: + async with _repo_locks_guard: + if repo_key not in _repo_locks: + _repo_locks[repo_key] = asyncio.Lock() + repo_lock = _repo_locks[repo_key] + async with repo_lock: + if repo_key not in _config_files_cache: + _config_files_cache[repo_key] = _collect_config_files(str(repo_path)) + if len(_config_files_cache) > _CONFIG_CACHE_MAX_SIZE: + _config_files_cache.popitem(last=False) + + for cfg in _config_files_cache[repo_key]: + if is_dependency_path(cfg[0]): + all_dep_configs.append(cfg) + else: + all_app_configs.append(cfg) + + all_dep_configs = filter_by_source_scope(all_dep_configs, source_scope, lambda x: x[0]) + + source_label = source_infos[0].git_repo if source_infos and hasattr(source_infos[0], "git_repo") else "unknown" + app_matches = search_config_content( + all_app_configs, keywords, + max_results=config.max_results, + context_lines=config.context_lines, + source_label=source_label, + ) + remaining = config.max_results - len(app_matches) + dep_matches = search_config_content( + all_dep_configs, keywords, + max_results=max(remaining, 0), + context_lines=config.context_lines, + source_label=source_label, + ) if remaining > 0 else [] + + no_results_msg = f"No configuration entries found matching: {query}" + total_app = _count_config_matches(all_app_configs, keywords) + total_dep = _count_config_matches(all_dep_configs, keywords) + logger.debug("Configuration scanner: %d app + %d dep match(es) for '%s'", + total_app, total_dep, query) + return format_app_dep_output(app_matches, dep_matches, total_app, total_dep, no_results_msg) + + yield FunctionInfo.from_fn( + _arun, + description=( + "Scans configuration files (YAML, XML, properties, build files, Dockerfiles) " + "for patterns related to the vulnerability. Input: keywords describing what " + "configuration to look for (e.g. 'XML external entity processing', " + "'deserialization enabled', 'xstream allowTypes'). " + "Searches all indexed sources including framework dependencies." + ), + ) \ No newline at end of file diff --git a/src/vuln_analysis/tools/container_image_analysis_data.py b/src/vuln_analysis/tools/container_image_analysis_data.py new file mode 100644 index 000000000..53495dac3 --- /dev/null +++ b/src/vuln_analysis/tools/container_image_analysis_data.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig + +from vuln_analysis.utils.error_handling_decorator import catch_tool_errors + +CONTAINER_IMAGE_ANALYSIS_DATA = "container_image_analysis_data" + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) + + +class ContainerImageAnalysisDataToolConfig(FunctionBaseConfig, name=("%s" % CONTAINER_IMAGE_ANALYSIS_DATA)): + """ + Tool to retrieve pre-analyzed container image data produced earlier in the workflow. + It does not perform new analysis, it returns structured data gathered previously. + """ + + +@register_function(config_type=ContainerImageAnalysisDataToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def container_image_analysis_data(config: ContainerImageAnalysisDataToolConfig, builder: Builder): + from vuln_analysis.runtime_context import ctx_state + + @catch_tool_errors(CONTAINER_IMAGE_ANALYSIS_DATA) + async def _arun(query: str) -> list[dict[str, Any]]: + state = ctx_state.get() + vuln_id = getattr(state, "current_vuln_id", None) + + results: list[dict[str, Any]] = [] + + if vuln_id is None: + return results + + try: + for item in state.checklist_results[vuln_id]: + results.append({ + "response": item.get("output"), + "intermediate_steps": item.get("intermediate_steps") + }) + except Exception as e: + logger.exception("Failed to read analysis data from state: %s", e) + return [] + + return results + + yield FunctionInfo.from_fn( + _arun, + description=( + "Retrieves pre-analyzed container image source code analysis results for the current CVE. " + "Returns a list of findings with 'response' (summary) and 'intermediate_steps' (reasoning) fields. " + "No input required - uses context automatically." + ) + ) + diff --git a/src/vuln_analysis/tools/import_usage_analyzer.py b/src/vuln_analysis/tools/import_usage_analyzer.py new file mode 100644 index 000000000..0b43141d6 --- /dev/null +++ b/src/vuln_analysis/tools/import_usage_analyzer.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +import tantivy +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers_factory import get_language_function_parser +from vuln_analysis.utils.error_handling_decorator import catch_tool_errors +from vuln_analysis.utils.source_classification import is_dependency_path, filter_by_source_scope, format_app_dep_output + +IMPORT_USAGE_ANALYZER = "import_usage_analyzer" + +logger = LoggingFactory.get_agent_logger(__name__) + + +def _find_usage_in_file(content: str, imported_names: list[str], max_usages: int = 5) -> list[str]: + """Find usage sites of imported names, excluding import lines themselves.""" + usages = [] + lines = content.split("\n") + for line_num, line in enumerate(lines): + for name in imported_names: + short_name = name + if "/" in short_name: + short_name = short_name.rsplit("/", 1)[-1] + if "." in short_name: + short_name = short_name.rsplit(".", 1)[-1] + if re.search(rf'\b{re.escape(short_name)}\b', line) and not line.strip().startswith(("import ", "from ", "#include")): + usages.append(f" L{line_num+1}: {line.strip()}") + if len(usages) >= max_usages: + return usages + return usages + + +class ImportUsageAnalyzerToolConfig(FunctionBaseConfig, name=IMPORT_USAGE_ANALYZER): + """Analyzes imports and usage patterns of a package across indexed sources.""" + max_files: int = Field(default=20, description="Maximum files to report") + + +def analyze_imports(searcher, import_patterns: list[re.Pattern], package_name: str, + max_files: int = 20, source_scope: list[str] | None = None, + ecosystem_label: str = "") -> str: + """Scan a Tantivy searcher for import patterns, with app/dep source awareness. + + Results are separated into application code and dependency code sections, + with dependency results filtered by source_scope when provided. + """ + num_docs = searcher.num_docs + logger.debug("analyze_imports: scanning %d docs for '%s' (patterns: %d)", num_docs, package_name, len(import_patterns)) + app_results = [] + dep_results = [] + docs_with_pkg_name = 0 + docs_with_regex_match = 0 + docs_with_fallback_match = 0 + doc_errors = 0 + + all_docs = searcher.search(tantivy.Query.all_query(), limit=num_docs).hits + + logged_samples = 0 + for _score, doc_address in all_docs: + try: + raw = searcher.doc(doc_address) + file_path = raw["file_path"][0] + # Skip .git/ internal directory (but not .github/, .gitignore, etc.) + if file_path == ".git" or file_path.startswith(".git/") or "/.git/" in file_path: + continue + content = raw["content"][0] + except Exception as e: + doc_errors += 1 + if doc_errors <= 3: + logger.debug("analyze_imports: doc_address=%s error: %s %s", doc_address, type(e).__name__, e) + continue + + found_imports = [] + for pattern in import_patterns: + for match in pattern.finditer(content): + found_imports.append(match.group(0)) + + pkg_lower = package_name.lower() + content_has_pkg = pkg_lower in content.lower() + + if found_imports: + docs_with_regex_match += 1 + if logged_samples < 5: + logger.debug("analyze_imports: REGEX MATCH file=%s imports=%s", + file_path, found_imports[:3]) + logged_samples += 1 + elif content_has_pkg: + docs_with_pkg_name += 1 + if logged_samples < 5: + import_lines = [line.strip() for line in content.split('\n') + if pkg_lower in line.lower()] + logger.debug("analyze_imports: PKG IN CONTENT but no regex match " + "file=%s lines_with_pkg=%s", + file_path, import_lines[:3]) + logged_samples += 1 + + if not found_imports: + # Fallback: regex may miss imports in chunked documents where the + # import keyword is in a different chunk than the package path. + if not content_has_pkg: + continue + for line in content.split('\n'): + stripped = line.strip() + if pkg_lower in stripped.lower() and len(stripped) > len(package_name): + found_imports.append(stripped) + if not found_imports: + continue + docs_with_fallback_match += 1 + + imported_names = [] + for imp in found_imports: + parts = imp.split() + for p in parts: + cleaned = p.strip(";'\",<>(){}").strip() + if package_name.lower() in cleaned.lower() and len(cleaned) > 2: + imported_names.append(cleaned) + + usages = _find_usage_in_file(content, imported_names) + + entry = f"--- {file_path} ---\n" + entry += f" Imports: {'; '.join(found_imports[:3])}\n" + if usages: + entry += f" Usages ({len(usages)}):\n" + "\n".join(usages) + else: + entry += " No usage sites found beyond import." + + if is_dependency_path(file_path): + dep_results.append((file_path, entry)) + else: + app_results.append(entry) + + logger.debug("analyze_imports: docs_scanned=%d, regex_matches=%d, fallback_matches=%d, " + "pkg_name_present=%d, errors=%d, app_results=%d, dep_results=%d", + num_docs, docs_with_regex_match, docs_with_fallback_match, + docs_with_pkg_name, doc_errors, len(app_results), len(dep_results)) + + pre_filter_count = len(dep_results) + dep_results = filter_by_source_scope(dep_results, source_scope, lambda x: x[0]) + dep_entries = [entry for _, entry in dep_results] + if pre_filter_count != len(dep_entries): + logger.debug("analyze_imports: source_scope filter reduced dep_results from %d to %d (scope=%s)", + pre_filter_count, len(dep_entries), source_scope) + + no_results_msg = f"No imports of '{package_name}' found in indexed sources (ecosystem: {ecosystem_label})." + total_app = len(app_results) + total_dep = len(dep_entries) + + trimmed_app = app_results[:max_files] + remaining = max_files - len(trimmed_app) + trimmed_dep = dep_entries[:max(remaining, 0)] + + return format_app_dep_output(trimmed_app, trimmed_dep, total_app, total_dep, no_results_msg) + + +@register_function(config_type=ImportUsageAnalyzerToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def import_usage_analyzer(config: ImportUsageAnalyzerToolConfig, builder: Builder): + from vuln_analysis.runtime_context import ctx_state, cu_source_scope + from vuln_analysis.utils.full_text_search import FullTextSearch + + @catch_tool_errors(IMPORT_USAGE_ANALYZER) + async def _arun(query: str) -> str: + workflow_state = ctx_state.get() + code_index_path = workflow_state.code_index_path + ecosystem = workflow_state.original_input.input.image.ecosystem + source_scope = cu_source_scope.get() + + fts = FullTextSearch.get_instance(cache_path=code_index_path) + if fts.is_empty(): + logger.debug("Import usage analyzer: no source code indexed at %s", code_index_path) + return "No source code indexed." + + package_name = query.strip() + parser = get_language_function_parser(ecosystem, tree=None) if ecosystem else None + import_patterns = parser.get_import_search_patterns(package_name) if parser else [re.compile(re.escape(package_name), re.IGNORECASE)] + ecosystem_label = ecosystem.value if ecosystem else "" + logger.debug("Import usage analyzer: searching for '%s' (ecosystem: %s), scope=%s, " + "parser=%s, patterns=%s", + package_name, ecosystem_label, source_scope, + type(parser).__name__ if parser else "None", + [p.pattern for p in import_patterns]) + + fts.index.reload() + searcher = fts.index.searcher() + logger.debug("Import usage analyzer: index has %d docs after reload", searcher.num_docs) + result = analyze_imports( + searcher, import_patterns, package_name, + max_files=config.max_files, source_scope=source_scope, + ecosystem_label=ecosystem_label, + ) + logger.debug("Import usage analyzer: %s", result.split("\n", 1)[0]) + return result + + yield FunctionInfo.from_fn( + _arun, + description=( + "Finds all imports and usage patterns of a specific package/module across indexed sources. " + "Input: package or module name (e.g. 'encoding/xml', 'com.thoughtworks.xstream', " + "'urllib.parse'). Reports which files import it, how many, and how the package is used. " + "Searches all sources including framework dependencies." + ), + ) \ No newline at end of file diff --git a/src/vuln_analysis/tools/lexical_full_search.py b/src/vuln_analysis/tools/lexical_full_search.py index 2dca0253d..a01292b25 100644 --- a/src/vuln_analysis/tools/lexical_full_search.py +++ b/src/vuln_analysis/tools/lexical_full_search.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging - from aiq.builder.builder import Builder from aiq.builder.framework_enum import LLMFrameworkEnum from aiq.builder.function_info import FunctionInfo @@ -22,38 +20,62 @@ from aiq.data_models.function import FunctionBaseConfig from pydantic import Field -logger = logging.getLogger(__name__) +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.data_models.input import PipelineMode +from vuln_analysis.utils.error_handling_decorator import catch_tool_errors + +LEXICAL_CODE_SEARCH = "lexical_code_search" +logger = LoggingFactory.get_agent_logger(__name__) -class LexicalSearchToolConfig(FunctionBaseConfig, name="lexical_code_search"): +class LexicalSearchToolConfig(FunctionBaseConfig, name=LEXICAL_CODE_SEARCH): """ Lexical search tool used to search source code. """ top_k: int = Field(default=5, description="Top K to use for the lexical search") + base_code_index_dir: str = Field( + default=".cache/am_cache/code_index", + description="Base directory for Tantivy code index storage.", + ) @register_function(config_type=LexicalSearchToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) async def lexical_search(config: LexicalSearchToolConfig, builder: Builder): # pylint: disable=unused-argument - from vuln_analysis.functions.cve_agent import ctx_state + from vuln_analysis.runtime_context import ctx_state, cu_source_scope from vuln_analysis.utils.full_text_search import FullTextSearch - async def _arun(query: str) -> list: - + @catch_tool_errors(LEXICAL_CODE_SEARCH) + async def _arun(query: str) -> str: workflow_state = ctx_state.get() - code_index_path = workflow_state.code_index_path + + pipeline_mode = getattr(workflow_state.original_input.input.image, 'pipeline_mode', None) + + if pipeline_mode == PipelineMode.PACKAGE_CHECKER: + source_key = workflow_state.original_input.info.checker_context.source_key + code_index_path = str(FullTextSearch.get_index_directory(config.base_code_index_dir, source_key)) + else: + code_index_path = workflow_state.code_index_path + full_text_search = FullTextSearch(cache_path=code_index_path) if full_text_search.is_empty(): + logger.debug("Lexical search: index is empty at %s", code_index_path) raise ValueError(f"Invalid code index at: {code_index_path}, index is empty") - result = full_text_search.search_index(query, config.top_k) + # Pass source_scope from ContextVar to restrict dependency results + # to the current question's target package (set by CU pre_process_node) + source_scope = cu_source_scope.get() + result = full_text_search.search_index(query, config.top_k, source_scope=source_scope) + logger.debug("Lexical search: query='%.80s', top_k=%d, source_scope=%s", + query, config.top_k, source_scope) return result yield FunctionInfo.from_fn( _arun, - description=("Useful for when you need to check if an application or any dependency " - "within the container image uses a function or a component of a library " - "using keyword search. This tool uses keyword to search code index, and " - "the argument should be a string keyword. You should use this search " - "tool for searching codes before trying other container search related tools.")) + description=( + "Performs keyword search on container source code for exact text matches. " + "Input should be a function name, class name, or code pattern. " + "Use this first before semantic search tools for precise lookups." + ) + ) diff --git a/src/vuln_analysis/tools/local_vdb.py b/src/vuln_analysis/tools/local_vdb.py index 6bff4f448..98b701d64 100644 --- a/src/vuln_analysis/tools/local_vdb.py +++ b/src/vuln_analysis/tools/local_vdb.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging - from aiq.builder.builder import Builder from aiq.builder.framework_enum import LLMFrameworkEnum from aiq.builder.function_info import FunctionInfo @@ -24,10 +22,15 @@ from vuln_analysis.data_models.vdb_type import VdbType -logger = logging.getLogger(__name__) +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.utils.error_handling_decorator import catch_tool_errors + +LOCAL_VDB_RETRIEVER = "local_vdb_retriever" + +logger = LoggingFactory.get_agent_logger(__name__) -class LocalVDBRetrieverToolConfig(FunctionBaseConfig, name="local_vdb_retriever"): +class LocalVDBRetrieverToolConfig(FunctionBaseConfig, name=("%s" % LOCAL_VDB_RETRIEVER)): """ Retriever tool used to query source code and documentation vector databases. """ @@ -44,14 +47,22 @@ async def load_vectordb_asretriever(config: LocalVDBRetrieverToolConfig, builder from langchain_community.vectorstores import FAISS from langchain_core.prompts import PromptTemplate - from vuln_analysis.functions.cve_agent import ctx_state + from vuln_analysis.runtime_context import ctx_state, cu_source_scope embedder = await builder.get_embedder(embedder_name=config.embedder_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) llm = await builder.get_llm(llm_name=config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + qa_prompt = PromptTemplate(template=("Use the following pieces of context to answer the question at the end. " + "If you don't know the answer, just say that you don't know, " + "don't try to make up an answer.\n\n{context}\n\n" + "Question: {question}\nHelpful Answer:"), + input_variables=['context', 'question']) + # FAISS deserialization is expensive — cache per db_source path + _retrieval_cache: dict[str, RetrievalQA] = {} + + @catch_tool_errors(LOCAL_VDB_RETRIEVER) async def _arun(query: str) -> str | dict: - # workaround since the agent executor only accepts strings. workflow_state = ctx_state.get() if config.vdb_type == VdbType.CODE: @@ -61,21 +72,32 @@ async def _arun(query: str) -> str | dict: else: raise ValueError(f"Invalid VDB type: {config.vdb_type}. Must be one of {VdbType.CODE} or {VdbType.DOC}.") - qa_prompt = PromptTemplate(template=("Use the following pieces of context to answer the question at the end. " - "If you don't know the answer, just say that you don't know, " - "don't try to make up an answer.\n\n{context}\n\n" - "Question: {question}\nHelpful Answer:"), - input_variables=['context', 'question']) - - vector_db = FAISS.load_local(db_source, embedder, allow_dangerous_deserialization=True) - retrieval_qa_tool = RetrievalQA.from_chain_type(llm=llm, - chain_type="stuff", - chain_type_kwargs={"prompt": qa_prompt}, - retriever=vector_db.as_retriever(), - return_source_documents=config.return_source_documents) + if db_source not in _retrieval_cache: + logger.info("Loading FAISS index from %s (type: %s)", db_source, config.vdb_type) + vector_db = FAISS.load_local(db_source, embedder, allow_dangerous_deserialization=True) + _retrieval_cache[db_source] = RetrievalQA.from_chain_type( + llm=llm, + chain_type="stuff", + chain_type_kwargs={"prompt": qa_prompt}, + retriever=vector_db.as_retriever(), + return_source_documents=config.return_source_documents, + ) + retrieval_qa_tool = _retrieval_cache[db_source] output_dict = await retrieval_qa_tool.ainvoke(query) + if config.return_source_documents and "source_documents" in output_dict: + source_scope = cu_source_scope.get() + if source_scope: + pre_filter = len(output_dict["source_documents"]) + output_dict["source_documents"] = [ + doc for doc in output_dict["source_documents"] + if any(scope in doc.metadata.get("source", "") for scope in source_scope) + ] + if pre_filter != len(output_dict["source_documents"]): + logger.debug("Source scope %s filtered doc results from %d to %d", + source_scope, pre_filter, len(output_dict["source_documents"])) + # If returning source documents, include the result and source_documents keys in the output if config.return_source_documents: return {k: v for k, v in output_dict.items() if k in ["result", "source_documents"]} @@ -84,11 +106,18 @@ async def _arun(query: str) -> str | dict: return output_dict["result"] if config.vdb_type == VdbType.CODE: - description = ("Useful for when you need to check if an application or any dependency " - "within the container image uses a function or a component of a library.") + description = ( + "Searches container source code using semantic search. " + "Finds how functions, libraries, or components are used in the codebase. " + "Answers questions about code implementation and dependencies." + ) elif config.vdb_type == VdbType.DOC: - description = ("Useful for when you need to ask questions about the purpose and " - "functionality of the container image.") + description = ( + "Searches container documentation using semantic search. " + "Answers questions about application purpose, architecture, and features. " + "Queries must be specific — include a concrete term (library name, config property, protocol). " + "Vague queries like 'configuration' or 'security settings' return unhelpful results." + ) else: raise ValueError(f"Invalid VDB type: {config.vdb_type}. Must be one of {VdbType.CODE} or {VdbType.DOC}.") diff --git a/src/vuln_analysis/tools/serp.py b/src/vuln_analysis/tools/serp.py index 6b1f2d7d3..7fe820b82 100644 --- a/src/vuln_analysis/tools/serp.py +++ b/src/vuln_analysis/tools/serp.py @@ -13,17 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging - from aiq.builder.builder import Builder from aiq.builder.function_info import FunctionInfo from aiq.cli.register_workflow import register_function from aiq.data_models.function import FunctionBaseConfig -logger = logging.getLogger(__name__) +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.utils.error_handling_decorator import catch_tool_errors + +SERP_WRAPPER = "serp_wrapper" + +logger = LoggingFactory.get_agent_logger(__name__) -class SerpWrapperToolConfig(FunctionBaseConfig, name="serp_wrapper"): +class SerpWrapperToolConfig(FunctionBaseConfig, name=("%s" % SERP_WRAPPER)): """ SerpApi Google search tool """ @@ -36,9 +39,15 @@ async def serp_wrapper(config: SerpWrapperToolConfig, builder: Builder): # pyli search = MorpheusSerpAPIWrapper(max_retries=config.max_retries) + @catch_tool_errors(SERP_WRAPPER) async def _arun(query: str) -> str: result = await search.arun(query) return result - yield FunctionInfo.from_fn(_arun, - description=("Useful for when you need to answer questions about external libraries")) + yield FunctionInfo.from_fn( + _arun, + description=( + "Searches the web for information about CVEs, vulnerabilities, libraries, " + "and security advisories not available in the container." + ) + ) diff --git a/src/vuln_analysis/tools/source_grep.py b/src/vuln_analysis/tools/source_grep.py new file mode 100644 index 000000000..9220937e0 --- /dev/null +++ b/src/vuln_analysis/tools/source_grep.py @@ -0,0 +1,262 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native Unix grep tool for fast source code searching. + +Provides an LLM-callable tool that uses native grep subprocess for +faster searching compared to Python-based regex scanning. +""" + +from pathlib import Path + +from aiq.builder.builder import Builder +from aiq.builder.framework_enum import LLMFrameworkEnum +from aiq.builder.function_info import FunctionInfo +from aiq.cli.register_workflow import register_function +from aiq.data_models.function import FunctionBaseConfig +from pydantic import Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.tools.source_inspector import SourceInspector +from vuln_analysis.utils.error_handling_decorator import catch_tool_errors + +SOURCE_GREP = "source_grep" + +logger = LoggingFactory.get_agent_logger(__name__) + + +class SourceGrepToolConfig(FunctionBaseConfig, name=SOURCE_GREP): + """Fast grep search using native Unix grep subprocess.""" + + base_checker_dir: str = Field( + default=".cache/am_cache/checker", + description="Root directory for checker-specific artifacts.", + ) + max_results: int = Field( + default=50, + description="Maximum number of grep results to return.", + ) + context_lines: int = Field( + default=3, + description="Number of context lines around each match.", + ) + + +VALID_TARGETS = ("source", "logs", "patch") +QUERY_FILE_SEPARATOR = "," +QUERY_MULTI_PATTERN_SEPARATOR = ";" +FILE_GLOB_WILDCARDS = ("*", "?", "/") + +TARGET_EXTENSIONS: dict[str, list[str]] = { + "source": ["*.c", "*.h", "*.cpp", "*.hpp", "*.py", "*.go", "*.java", "*.spec", "*.cmake", "Makefile", "*.mk", "*.config"], + "logs": [], # empty = search all files + "patch": ["*.patch", "*.diff"], +} + + +def _looks_like_file_glob(token: str) -> bool: + """Return True when token appears to be a filename or glob.""" + if not token: + return False + filename = Path(token).name + return any(ch in token for ch in FILE_GLOB_WILDCARDS) or "." in filename + + +def _parse_query(query: str) -> tuple[str | list[str], str | None, str, bool]: + """Parse query string into (pattern(s), file_glob, target, word_boundary). + + Supports formats: + - "pattern" -> search source (default) + - "pattern,*.c" -> search source, only .c files + - "target:pattern" -> search specific target + - "target:pattern,file_glob" -> search target with file filter + - "pattern -w" -> search with word boundary (whole words only) + - "target:pattern,file_glob -w" -> full format with word boundary + - "pattern1;pattern2,file.c" -> multiple patterns (only with file_glob) + - "pattern1,pattern2,file.c" -> compatibility syntax (normalized to multi-pattern + file) + + Valid targets: source, logs, patch + + Note: Multiple patterns (separated by ';') are only supported when + a file_glob is provided. This prevents overly broad multi-pattern searches. + """ + query = query.strip().strip('"').strip("'") + + word_boundary = False + if query.endswith(" -w"): + word_boundary = True + query = query[:-3].strip() + + target = "source" + if ":" in query: + prefix, rest = query.split(":", 1) + if prefix in VALID_TARGETS: + target = prefix + query = rest + + if QUERY_FILE_SEPARATOR in query: + parts = query.split(QUERY_FILE_SEPARATOR, 1) + pattern_part = parts[0].strip() + file_glob = parts[1].strip() if len(parts) > 1 else None + + # Multi-pattern support: only when file_glob is provided + if file_glob and QUERY_MULTI_PATTERN_SEPARATOR in pattern_part: + patterns = [p.strip() for p in pattern_part.split(QUERY_MULTI_PATTERN_SEPARATOR) if p.strip()] + return patterns, file_glob, target, word_boundary + + # Compatibility mode: + # LLMs sometimes emit "pattern1,pattern2,file.c" instead of + # the documented "pattern1;pattern2,file.c". Normalize it. + if file_glob and QUERY_FILE_SEPARATOR in file_glob: + trailing_parts = [p.strip() for p in file_glob.split(QUERY_FILE_SEPARATOR) if p.strip()] + candidate_file_glob = trailing_parts[-1] if trailing_parts else "" + extra_patterns = trailing_parts[:-1] + + if _looks_like_file_glob(candidate_file_glob): + patterns = [pattern_part, *extra_patterns] + patterns = [p for p in patterns if p] + if len(patterns) > 1: + return patterns, candidate_file_glob, target, word_boundary + if len(patterns) == 1: + return patterns[0], candidate_file_glob, target, word_boundary + + return pattern_part, file_glob, target, word_boundary + + return query, None, target, word_boundary + + +def _format_results(pattern: str, matches: list, root: Path) -> str: + """Format grep results for LLM consumption.""" + if not matches: + return f"No matches found for '{pattern}'" + + lines = [f"Found {len(matches)} match(es) for '{pattern}':\n"] + for i, match in enumerate(matches, 1): + try: + rel_path = match.file_path.relative_to(root) + except ValueError: + rel_path = match.file_path + lines.append(f"{i}. {rel_path}:{match.match_line_number}") + lines.append(f" {match.full_text.strip()}") + lines.append("") + + return "\n".join(lines) + + +@register_function(config_type=SourceGrepToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN]) +async def source_grep(config: SourceGrepToolConfig, builder: Builder): # pylint: disable=unused-argument + from vuln_analysis.runtime_context import ctx_state + + @catch_tool_errors(SOURCE_GREP) + async def _arun(query: str) -> str: + """Search source code, build logs, or patches using native Unix grep. + + Query format: '[target:]pattern[,file_glob][ -w]' + + Targets: + - source (default): Package source code + - logs: Build compilation logs + - patch: Fixed patches from newer RPM version + + Options: + - -w: Match whole words only (word boundary) + - Multiple patterns: use ';' separator ONLY with a specific file + - Compatibility: "pattern1,pattern2,file.c" is accepted and normalized + + Examples: + - 'archive_read_open' - search source files + - 'archive_read_open,*.c' - search only .c source files + - 'archive_read_open -w' - search for whole word only + - 'unsigned int cursor;unsigned int nodes,archive_read.c' - multiple patterns in one file + - 'sum2,s2length,match.c' - compatibility syntax for multiple patterns in one file + - 'logs:undefined reference' - search build logs for link errors + - 'logs:error:' - search build logs for error messages + - 'patch:CVE-2026-5121' - find patch for specific CVE + - 'patch:archive_read,*.patch' - search in patch files + """ + workflow_state = ctx_state.get() + + checker_context = None + if workflow_state.original_input and workflow_state.original_input.info: + checker_context = workflow_state.original_input.info.checker_context + + if checker_context is None or not checker_context.source_key: + raise ValueError("Checker context or source_key not available in workflow state") + + source_key = checker_context.source_key + pattern, file_glob, target, word_boundary = _parse_query(query) + + # For logs target, normalize path patterns to just filename + # LLMs often pass full paths like "modules/proxy/mod_proxy_ftp.c" from AFFECTED_FILES + # but build logs only reference filenames like "mod_proxy_ftp.c" + if target == "logs" and isinstance(pattern, str) and "/" in pattern: + original_pattern = pattern + pattern = pattern.rsplit("/", 1)[-1] + logger.debug("logs target: normalized pattern '%s' -> '%s'", original_pattern, pattern) + + # For logs target, use arch-specific subdirectory + if target == "logs": + target_package = workflow_state.original_input.input.image.target_package + if target_package is None or not target_package.arch: + raise ValueError("logs target requires target_package.arch in workflow state") + + arch = target_package.arch + target_dir = (Path(config.base_checker_dir) / source_key / "logs" / arch).resolve() + + if not target_dir.is_dir(): + return f"No build log available for architecture '{arch}'. Build log may not have been fetched for this package." + else: + target_dir = (Path(config.base_checker_dir) / source_key / target).resolve() + + if not target_dir.is_dir(): + raise ValueError(f"Target directory does not exist: {target_dir}") + + inspector = SourceInspector(target_dir) + default_extensions = TARGET_EXTENSIONS.get(target, []) + + logger.info("Source grep: searching for '%s' in %s (target: %s, glob: %s, word_boundary: %s)", + pattern, target_dir, target, file_glob or "default extensions", word_boundary) + + matches = await inspector.grep_native( + patterns=pattern, + file_glob=file_glob, + word_boundary=word_boundary, + context_lines=config.context_lines, + max_results=config.max_results, + default_extensions=default_extensions, + ) + + logger.info("Source grep: found matches for '%s' in target '%s'", pattern, target) + return matches + + yield FunctionInfo.from_fn( + _arun, + description=( + "Fast grep search using native Unix grep. " + "Query format: '[target:]pattern[,file_glob][ -w]'. " + "Targets: 'source' (default) for package source code, " + "'logs' for build compilation logs, " + "'patch' for fixed patches from newer RPM. " + "Add ' -w' suffix for whole-word matching. " + "Multiple patterns: use ';' separator ONLY with a specific file, e.g., " + "'pattern1;pattern2,filename.c' searches for both patterns in that file. " + "Examples: 'archive_read_open' searches source, " + "'archive_read_open,*.c' searches only C source files, " + "'archive_read_open -w' searches for whole word only, " + "'unsigned int cursor;unsigned int nodes,archive_read.c' searches multiple patterns in one file, " + "'logs:undefined reference' searches build logs, " + "'patch:CVE-2026-5121' searches patch files." + ), + ) diff --git a/src/vuln_analysis/tools/source_inspector.py b/src/vuln_analysis/tools/source_inspector.py new file mode 100644 index 000000000..34ed3d39d --- /dev/null +++ b/src/vuln_analysis/tools/source_inspector.py @@ -0,0 +1,264 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generic filesystem utility for inspecting extracted RPM source trees. + +Provides low-level primitives (find, grep, read) that can be composed by +pipeline code or called by an LLM agent in the future. +""" + +from __future__ import annotations + +import asyncio +import re +import shlex +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from nat.builder.context import Context + + +@dataclass +class GrepMatch: + file_path: Path + line_number: int + line_content: str + + +class SourceInspector: + """Filesystem inspector scoped to a root directory. + + All returned paths are absolute. The class carries no domain-specific + logic (RPM, spec, changelog); callers compose the primitives for that. + """ + + def __init__(self, source_dir: Path) -> None: + if ".." in source_dir.parts: + raise ValueError(f"Path contains traversal components: {source_dir}") + self._root = source_dir.resolve() + if not self._root.is_dir(): + raise FileNotFoundError(f"source_dir does not exist: {self._root}") + + @property + def root(self) -> Path: + return self._root + + def find_files(self, pattern: str, recursive: bool = True) -> list[Path]: + """Glob over the source tree. + + Parameters + ---------- + pattern: + Shell glob pattern, e.g. ``"*.spec"`` or ``"*.patch"``. + recursive: + If *True* use ``**/`` (deep search). + If *False* use ```` (root-level only). + """ + glob_expr = f"**/{pattern}" if recursive else pattern + return sorted(self._root.glob(glob_expr)) + + def grep_content( + self, + pattern: str, + file_path: Path | None = None, + *, + recursive: bool = False, + ) -> list[GrepMatch]: + """Search file contents for a regex *pattern*. + + Parameters + ---------- + pattern: + Regular expression (case-sensitive by default). + file_path: + If given, search that file only, or (if it is a directory) every file + in that directory (one level, regular files only). + If the path does not exist, return no matches. + If *None*, search every file under *source_dir* + (depth controlled by *recursive*). + recursive: + Only used when *file_path* is ``None``. + ``False`` searches only root-level files; ``True`` walks the tree. + """ + regex = re.compile(pattern) + matches: list[GrepMatch] = [] + + if file_path is not None: + resolved = file_path.resolve() + if resolved.is_file(): + targets = [resolved] + elif resolved.is_dir(): + targets = sorted(p for p in resolved.iterdir() if p.is_file()) + else: + targets = [] + elif recursive: + targets = sorted(p for p in self._root.rglob("*") if p.is_file()) + else: + targets = sorted(p for p in self._root.iterdir() if p.is_file()) + + for fp in targets: + try: + lines = fp.read_text(encoding="utf-8", errors="replace").splitlines() + except (OSError, UnicodeDecodeError): + continue + for idx, line in enumerate(lines, start=1): + if regex.search(line): + matches.append(GrepMatch(file_path=fp, line_number=idx, line_content=line)) + return matches + + async def grep_native( + self, + patterns: str | list[str], + file_glob: str | None = None, + *, + case_insensitive: bool = False, + word_boundary: bool = False, + context_lines: int = 0, + max_results: int = 50, + default_extensions: list[str] | None = None, + ) -> str: + """Fast grep using native Unix grep subprocess. + + Parameters + ---------- + patterns: + Search pattern(s). Can be a single string or list of patterns. + When multiple patterns are provided, matches ANY of them (OR logic). + file_glob: + Optional file pattern (e.g., ``"*.c"``, ``"*.h"``). If provided, + overrides default_extensions. + case_insensitive: + If *True*, perform case-insensitive matching (``-i`` flag). + word_boundary: + If *True*, match whole words only (``-w`` flag). + context_lines: + Lines of context around match (``-C`` flag). Default 0. + max_results: + Stop after this many matches (``-m`` flag). Default 50. + default_extensions: + List of file extensions to search when file_glob is not provided. + If *None*, searches ALL files (no --include filter). + If empty list ``[]``, searches ALL files (no --include filter). + + Returns + ------- + str + Raw grep output with matches found. + """ + cmd = ["grep", "-rn", "-I"] + + if case_insensitive: + cmd.append("-i") + if word_boundary: + cmd.append("-w") + + # Determine if searching a specific file (not a wildcard pattern) + # Specific file searches get asymmetric context (-B 2 -A 6) to see full code structures + # Wildcard searches use symmetric context (-C) to limit output volume + is_specific_file = file_glob and "*" not in file_glob + + if context_lines > 0: + if is_specific_file: + # More context after match to see full code block structure + cmd.extend(["-B", "2", "-A", "6"]) + else: + cmd.extend(["-C", str(context_lines)]) + + # If file_glob contains a path (e.g., "lib/connect.c"), split into: + # - path_filter: directory portion for post-filtering results (e.g., "lib/") + # - file_glob: filename only for --include (e.g., "connect.c") + path_filter = None + if file_glob and "/" in file_glob: + path_filter = file_glob.rsplit("/", 1)[0] + "/" + file_glob = file_glob.rsplit("/", 1)[1] + + if file_glob: + cmd.extend(["--include", file_glob]) + elif default_extensions is None: + pass # No filtering - search all files (caller should pass extensions explicitly) + elif default_extensions: + for ext in default_extensions: + cmd.extend(["--include", ext]) + + cmd.extend(["-m", str(max_results)]) + + # Handle single or multiple patterns + if isinstance(patterns, list): + for p in patterns: + cmd.extend(["-e", p]) + else: + cmd.extend(["-e", patterns]) + + cmd.append(".") + + def _run_grep() -> tuple[str, int]: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + errors="replace", + cwd=self._root, + timeout=30, + ) + return result.stdout, result.returncode + + try: + tracer = Context.get() + except Exception: + tracer = None + + if tracer: + with tracer.push_active_function("grep_native", input_data={"command": shlex.join(cmd), "path_filter": path_filter}) as span: + stdout, returncode = await asyncio.to_thread(_run_grep) + span.set_output({ + "return_code": returncode, + "match_count": stdout.count('\n') if stdout else 0, + }) + else: + stdout, returncode = await asyncio.to_thread(_run_grep) + + # Post-filter results to only include paths matching path_filter + if path_filter and stdout: + filtered_lines = [line for line in stdout.splitlines() if path_filter in line] + stdout = "\n".join(filtered_lines) + if filtered_lines: + stdout += "\n" + + return stdout + + def read_file( + self, + file_path: Path, + offset: int = 0, + max_lines: int | None = None, + ) -> str: + """Read file content starting from a line *offset*. + + Parameters + ---------- + file_path: + Absolute or relative path (resolved against *source_dir*). + offset: + 0-based line offset to start reading from. + max_lines: + Maximum number of lines to return. ``None`` means read to EOF. + """ + resolved = file_path if file_path.is_absolute() else (self._root / file_path) + if not resolved.resolve().is_relative_to(self._root.resolve()): + raise ValueError(f"Path escapes root directory: {file_path}") + lines = resolved.read_text(encoding="utf-8", errors="replace").splitlines() + end = (offset + max_lines) if max_lines is not None else len(lines) + return "\n".join(lines[offset:end]) diff --git a/src/vuln_analysis/tools/tests/__init__.py b/src/vuln_analysis/tools/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/vuln_analysis/tools/tests/mock_documents.py b/src/vuln_analysis/tools/tests/mock_documents.py new file mode 100644 index 000000000..fdf733382 --- /dev/null +++ b/src/vuln_analysis/tools/tests/mock_documents.py @@ -0,0 +1,282 @@ +from langchain.docstore.document import Document + +python_script_example = Document( + page_content="from flask import helpers\n" + "from werkzeug.formparser import MultiPartParser\n\n" + "helpers.redirect('/home', 302)\n\n" + "multi_part_parser = MultiPartParser()\n" + "print(multi_part_parser.max_form_parts)", + type='Document', metadata={'source': 'src/example_main.py', + 'content_type': 'simplified_code', + 'language': 'python'}) +python_class_example = Document( + page_content='class MultiPartParser:\n' + ' def __init__(\n' + ' self,\n' + ' stream_factory: TStreamFactory | None = None,\n' + ' max_form_memory_size: int | None = None,\n' + ' cls: type[MultiDict[str, t.Any]] | None = None,\n' + ' buffer_size: int = 64 * 1024,\n' + ' max_form_parts: int | None = None,\n' + ' ) -> None:\n' + ' self.max_form_memory_size = max_form_memory_size\n' + ' self.max_form_parts = max_form_parts\n\n' + ' if stream_factory is None:\n' + ' stream_factory = default_stream_factory\n\n' + ' self.stream_factory = stream_factory\n\n' + ' if cls is None:\n' + ' cls = t.cast("type[MultiDict[str, t.Any]]", MultiDict)\n\n' + ' self.cls = cls\n' + ' self.buffer_size = buffer_size\n\n' + ' def fail(self, message: str) -> te.NoReturn:\n' + ' raise ValueError(message)\n\n' + ' def get_part_charset(self, headers: Headers) -> str:\n' + ' # Figure out input charset for current part\n' + ' content_type = headers.get("content-type")\n\n' + ' if content_type:\n' + ' parameters = parse_options_header(content_type)[1]\n' + ' ct_charset = parameters.get("charset", "").lower()\n\n' + ' # A safe list of encodings. Modern clients should only send ASCII or UTF-8.\n' + ' # This list will not be extended further.\n' + ' if ct_charset in {"ascii", "us-ascii", "utf-8", "iso-8859-1"}:\n' + ' return ct_charset\n\n' + ' return "utf-8"\n\n' + ' def start_file_streaming(\n' + ' self, event: File, total_content_length: int | None\n' + ' ) -> t.IO[bytes]:\n' + ' content_type = event.headers.get("content-type")\n\n' + ' try:\n' + ' content_length = _plain_int(event.headers["content-length"])\n' + ' except (KeyError, ValueError):\n' + ' content_length = 0\n\n' + ' container = self.stream_factory(\n' + ' total_content_length=total_content_length,\n' + ' filename=event.filename,\n' + ' content_type=content_type,\n' + ' content_length=content_length,\n' + ' )\n' + ' return container\n\n' + ' def parse(\n' + ' self, stream: t.IO[bytes], boundary: bytes, content_length: int | None\n' + ' ) -> tuple[MultiDict[str, str], MultiDict[str, FileStorage]]:\n' + ' current_part: Field | File\n' + ' field_size: int | None = None\n' + ' container: t.IO[bytes] | list[bytes]\n' + ' _write: t.Callable[[bytes], t.Any]\n\n' + ' parser = MultipartDecoder(\n' + ' boundary,\n' + ' max_form_memory_size=self.max_form_memory_size,\n' + ' max_parts=self.max_form_parts,\n' + ' )\n\n' + ' fields = []\n' + ' files = []\n\n' + ' for data in _chunk_iter(stream.read, self.buffer_size):\n' + ' parser.receive_data(data)\n' + ' event = parser.next_event()\n' + ' while not isinstance(event, (Epilogue, NeedData)):\n' + ' if isinstance(event, Field):\n' + ' current_part = event\n' + ' field_size = 0\n' + ' container = []\n' + ' _write = container.append\n' + ' elif isinstance(event, File):\n' + ' current_part = event\n' + ' field_size = None\n' + ' container = self.start_file_streaming(event, content_length)\n' + ' _write = container.write\n' + ' elif isinstance(event, Data):\n' + ' if self.max_form_memory_size is not None and field_size is not None:\n' + ' # Ensure that accumulated data events do not exceed limit.\n' + ' # Also checked within single event in MultipartDecoder.\n' + ' field_size += len(event.data)\n\n' + ' if field_size > self.max_form_memory_size:\n' + ' raise RequestEntityTooLarge()\n\n' + ' _write(event.data)\n' + ' if not event.more_data:\n' + ' if isinstance(current_part, Field):\n' + ' value = b"".join(container).decode(\n' + ' self.get_part_charset(current_part.headers), "replace"\n' + ' )\n' + ' fields.append((current_part.name, value))\n' + ' else:\n' + ' container = t.cast(t.IO[bytes], container)\n' + ' container.seek(0)\n' + ' files.append(\n' + ' (\n' + ' current_part.name,\n' + ' FileStorage(\n' + ' container,\n' + ' current_part.filename,\n' + ' current_part.name,\n' + ' headers=current_part.headers,\n' + ' ),\n' + ' )\n' + ' )\n\n' + ' event = parser.next_event()\n\n' + ' return self.cls(fields), self.cls(files)', + type='Document', + metadata={'source': 'transitive_env/lib/python3.12/site-packages/werkzeug/formparser.py', + 'content_type': 'functions_classes', + 'language': 'python'}) +python_init_function_example = Document(page_content='def __init__(\n' + ' self,\n' + ' stream_factory: TStreamFactory | None = None,\n' + ' max_form_memory_size: int | None = None,\n' + ' cls: type[MultiDict[str, t.Any]] | None = None,\n' + ' buffer_size: int = 64 * 1024,\n' + ' max_form_parts: int | None = None,\n' + ') -> None:\n' + ' mock_function_in_use(str(stream_factory))\n' + ' self.max_form_memory_size = max_form_memory_size\n' + ' self.max_form_parts = max_form_parts\n\n' + ' if stream_factory is None:\n' + ' stream_factory = default_stream_factory\n\n' + ' self.stream_factory = stream_factory\n\n' + ' if cls is None:\n' + ' cls = t.cast("type[MultiDict[str, t.Any]]", MultiDict)\n\n' + ' self.cls = cls\n' + ' self.buffer_size = buffer_size\n' + '#(class: MultiPartParser)', + type='Document', + metadata={ + 'source': 'transitive_env/lib/python3.12/site-packages/werkzeug/formparser.py', + 'content_type': 'functions_classes', + 'language': 'python'}) +python_full_document_example = Document(page_content='from __future__ import annotations\n\n' + 'from mock_package import mock_function_in_use' + 'import typing as t\n' + 'from io import BytesIO\n' + 'from urllib.parse import parse_qsl\n\n' + 'from ._internal import _plain_int\nfrom .datastructures import FileStorage\n' + 'from .datastructures import Headers\n' + 'from .datastructures import MultiDict\n' + 'from .exceptions import RequestEntityTooLarge\n' + 'from .http import parse_options_header\n' + 'from .sansio.multipart import Data\n' + 'from .sansio.multipart import Epilogue\n' + 'from .sansio.multipart import Field\n' + 'from .sansio.multipart import File\n' + 'from .sansio.multipart import MultipartDecoder\n' + 'from .sansio.multipart import NeedData\n' + 'from .wsgi import get_content_length\n' + 'from .wsgi import get_input_stream\n\n' + '# there are some platforms where SpooledTemporaryFile is not available.\n' + '# In that case we need to provide a fallback.\n' + 'try:\n' + ' from tempfile import SpooledTemporaryFile\nexcept ImportError:\n' + ' from tempfile import TemporaryFile\n\n SpooledTemporaryFile = None # type: ignore\n\nif t.TYPE_CHECKING:\n import typing as te\n\n from _typeshed.wsgi import WSGIEnvironment\n\n t_parse_result = tuple[\n t.IO[bytes], MultiDict[str, str], MultiDict[str, FileStorage]\n ]\n\n class TStreamFactory(te.Protocol):\n def __call__(\n self,\n total_content_length: int | None,\n content_type: str | None,\n filename: str | None,\n content_length: int | None = None,\n ) -> t.IO[bytes]: ...\n\n\nF = t.TypeVar("F", bound=t.Callable[..., t.Any])\n\n\n# Code for: def default_stream_factory(\n\n\n# Code for: def parse_form_data(\n\n\n# Code for: class FormDataParser:\n\n\n# Code for: class MultiPartParser:\n\n\n# Code for: def _chunk_iter(read: t.Callable[[int], bytes], size: int) -> t.Iterator[bytes | None]:', + type='Document', + metadata={ + 'source': 'transitive_env/lib/python3.12/site-packages/werkzeug/formparser.py', + 'content_type': 'simplified_code', + 'language': 'python'}) +python_parse_function_example = Document(page_content='def parse(\n' + ' self, stream: t.IO[bytes], boundary: bytes, content_length: int | None\n' + ') -> tuple[MultiDict[str, str], MultiDict[str, FileStorage]]:\n' + ' current_part: Field | File\n field_size: int | None = None\n' + ' container: t.IO[bytes] | list[bytes]\n' + ' _write: t.Callable[[bytes], t.Any]\n\n' + ' parser = MultipartDecoder(\n' + ' boundary,\n' + ' max_form_memory_size=self.max_form_memory_size,\n' + ' max_parts=self.max_form_parts,\n )\n\n' + ' fields = []\n files = []\n\n' + ' for data in _chunk_iter(stream.read, self.buffer_size):\n' + ' parser.receive_data(data)\n' + ' event = parser.next_event()\n' + ' while not isinstance(event, (Epilogue, NeedData)):\n' + ' if isinstance(event, Field):\n' + ' current_part = event\n' + ' field_size = 0\n' + ' container = []\n' + ' _write = container.append\n' + ' elif isinstance(event, File):\n' + ' current_part = event\n' + ' field_size = None\n' + ' container = self.start_file_streaming(event, content_length)\n' + ' _write = container.write\n' + ' elif isinstance(event, Data):\n' + ' if self.max_form_memory_size is not None and field_size is not None:\n' + ' # Ensure that accumulated data events do not exceed limit.\n' + ' # Also checked within single event in MultipartDecoder.\n' + ' field_size += len(event.data)\n\n' + ' if field_size > self.max_form_memory_size:\n' + ' raise RequestEntityTooLarge()\n\n' + ' _write(event.data)\n' + ' if not event.more_data:\n' + ' if isinstance(current_part, Field):\n' + ' value = b"".join(container).decode(\n' + ' self.get_part_charset(current_part.headers), "replace"\n' + ' )\n' + ' fields.append((current_part.name, value))\n' + ' else:\n' + ' container = t.cast(t.IO[bytes], container)\n' + ' container.seek(0)\n' + ' files.append(\n' + ' (\n' + ' current_part.name,\n' + ' FileStorage(\n' + ' container,\n' + ' current_part.filename,\n' + ' current_part.name,\n' + ' headers=current_part.headers,\n' + ' ),\n' + ' )\n )\n\n' + ' event = parser.next_event()\n\n' + ' return self.cls(fields), self.cls(files)\n' + '#(class: MultiPartParser)', + type='Document', + metadata={ + 'source': 'transitive_env/lib/python3.12/site-packages/werkzeug/formparser.py', + 'content_type': 'functions_classes', + 'language': 'python'}) +python_mock_function_in_use = Document(page_content='def mock_function_in_use(stream_factory: str) -> None:\n' + ' print stream_factory', + type='Document', + metadata={ + 'source': 'transitive_env/lib/python3.12/site-packages/mock_package/mock_file.py', + 'content_type': 'functions_classes', + 'language': 'python'}) +python_mock_file = Document(page_content='def mock_function_in_use(stream_factory: str) -> None:\n' + ' print stream_factory', + type='Document', + metadata={'source': 'transitive_env/lib/python3.12/site-packages/mock_package/mock_file.py', + 'content_type': 'simplified_code', + 'language': 'python'}) + +# Mock documents for python-dateutil (PyPI name) / dateutil (import name) +# Simulates a package where PyPI name != import name +python_dateutil_function = Document( + page_content='def parse(timestr, parserinfo=None, **kwargs):\n' + ' return _parse(timestr, parserinfo, **kwargs)', + type='Document', + metadata={'source': 'transitive_env/lib/python3.12/site-packages/dateutil/parser.py', + 'content_type': 'functions_classes', + 'language': 'python'}) + +python_dateutil_full_source = Document( + page_content='from dateutil.parser import parse\n\n' + 'import datetime\n', + type='Document', + metadata={'source': 'transitive_env/lib/python3.12/site-packages/dateutil/parser.py', + 'content_type': 'simplified_code', + 'language': 'python'}) + +python_app_using_dateutil = Document( + page_content='from dateutil.parser import parse\n\n' + 'def process_date(raw: str) -> str:\n' + ' dt = parse(raw)\n' + ' return dt.isoformat()', + type='Document', + metadata={'source': 'src/app.py', + 'content_type': 'simplified_code', + 'language': 'python'}) + +python_app_function_using_dateutil = Document( + page_content='def process_date(raw: str) -> str:\n' + ' dt = parse(raw)\n' + ' return dt.isoformat()', + type='Document', + metadata={'source': 'src/app.py', + 'content_type': 'functions_classes', + 'language': 'python'}) diff --git a/src/vuln_analysis/tools/tests/test_concurrency.py b/src/vuln_analysis/tools/tests/test_concurrency.py new file mode 100644 index 000000000..4e62fc9f7 --- /dev/null +++ b/src/vuln_analysis/tools/tests/test_concurrency.py @@ -0,0 +1,702 @@ +"""Concurrency tests for transitive code search infrastructure. + +Tests that retrievers are immutable after __init__ (mutable state lives in per-search +context objects), per-repo build serialization, and ecosystem-aware cache keys. +""" +import asyncio +import time +import threading + +import pytest +from unittest.mock import MagicMock, patch +from langchain_core.documents import Document + +from exploit_iq_commons.data_models.input import SourceDocumentsInfo +from exploit_iq_commons.utils.chain_of_calls_retriever import ChainOfCallsRetriever +from exploit_iq_commons.utils.java_chain_of_calls_retriever import JavaChainOfCallsRetriever +from exploit_iq_commons.utils.transitive_code_searcher_tool import TransitiveCodeSearcher + +from vuln_analysis.data_models.state import AgentMorpheusEngineState +from vuln_analysis.tools.transitive_code_search import ( + _build_or_get_cached, + _build_searcher, + get_git_and_pickle_base_dirs, + _searcher_cache, + _searcher_building, + _repo_build_locks, +) + +_DEFAULT_THRESHOLD = AgentMorpheusEngineState.model_fields["uber_jar_file_threshold"].default + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _build_fake_java_retriever(): + """Build a minimal JavaChainOfCallsRetriever (bypassing __init__) with + the immutable structure used after Strategy C refactor. + tree_dict stores parents directly (not [parents, exclusions, method_exclusions]). + No mutable per-search state on the instance. + """ + r = JavaChainOfCallsRetriever.__new__(JavaChainOfCallsRetriever) + + # Immutable after __init__ + r.tree_dict = { + "com.example:lib-a:1.0": ["root:app:1.0"], + "com.example:lib-b:2.0": ["root:app:1.0"], + "root:app:1.0": ["__root__"], + } + r.documents = [ + Document(page_content="public void foo(){}", metadata={"source": "lib-a-1.0-sources/Foo.java", "content_type": "functions_classes"}), + Document(page_content="public void bar(){}", metadata={"source": "lib-b-2.0-sources/Bar.java", "content_type": "functions_classes"}), + Document(page_content="public void main(){}", metadata={"source": "src/main/java/App.java", "content_type": "functions_classes"}), + ] + r._root_docs = [r.documents[2]] + r._jar_to_docs = { + "lib-a:1.0": [r.documents[0]], + "lib-b:2.0": [r.documents[1]], + } + r.supported_packages = list(r.tree_dict.keys()) + + # Read-only shared state + r.ecosystem = MagicMock() + r.dependency_tree = MagicMock() + r.language_parser = MagicMock() + r.manifest_path = "/fake" + r.documents_of_functions = r.documents[:] + r.documents_of_types = [] + r.documents_of_full_sources = {} + r.type_inheritance = {} + r.types_classes_fields_mapping = {} + r.functions_local_variables_index = {} + + return r + + +def _build_fake_go_retriever(): + """Build a minimal ChainOfCallsRetriever (non-Java, bypassing __init__) with + the immutable structure used after Strategy C refactor. + tree_dict stores parents directly (not [parents, exclusions]). + No mutable per-search state on the instance. + """ + r = ChainOfCallsRetriever.__new__(ChainOfCallsRetriever) + + r.tree_dict = { + "crypto/x509": ["__root__"], + "net/http": ["__root__"], + } + r.documents = [ + Document(page_content="func ParsePKCS1PrivateKey(){}", metadata={"source": "vendor/crypto/x509/pkcs1.go", "content_type": "functions_classes"}), + Document(page_content="func ListenAndServe(){}", metadata={"source": "vendor/net/http/server.go", "content_type": "functions_classes"}), + Document(page_content="func main(){}", metadata={"source": "cmd/main.go", "content_type": "functions_classes"}), + ] + r.supported_packages = list(r.tree_dict.keys()) + + # Read-only shared state + r.ecosystem = MagicMock() + r.dependency_tree = MagicMock() + r.language_parser = MagicMock() + r.manifest_path = "/fake" + r.documents_of_functions = r.documents[:] + r.documents_of_types = [] + r.documents_of_full_sources = {} + r.types_classes_fields_mapping = {} + r.functions_local_variables_index = {} + + return r + + +def _make_si(git_repo: str, ref: str = "main"): + """Create a minimal source_info list for cache key derivation.""" + return [SourceDocumentsInfo(type="code", git_repo=git_repo, ref=ref, + include=["**/*.java"], exclude=[])] + + +def _clear_caches(): + """Reset module-level caches between tests.""" + _searcher_cache.clear() + _searcher_building.clear() + _repo_build_locks.clear() + + +def _make_java_searcher(): + """Create a mock TransitiveCodeSearcher with a Java retriever.""" + mock = MagicMock() + mock.chain_of_calls_retriever = MagicMock(spec=JavaChainOfCallsRetriever) + return mock + + +def _make_nonjava_searcher(): + """Create a mock TransitiveCodeSearcher with a non-Java retriever.""" + mock = MagicMock() + mock.chain_of_calls_retriever = MagicMock(spec=ChainOfCallsRetriever) + return mock + + +def _make_slow_builder(build_log, sleep_secs=0.2, java=True): + """Create a synchronous _build_searcher replacement that records timing. + + Runs inside asyncio.to_thread (in a real thread), so uses time.sleep. + """ + lock = threading.Lock() + + def slow_build(si, query, uber_jar_file_threshold=_DEFAULT_THRESHOLD,base_dirs=()): + tag = query.split(",")[0] + start = time.monotonic() + time.sleep(sleep_secs) + end = time.monotonic() + with lock: + build_log.append((tag, start, end)) + return _make_java_searcher() if java else _make_nonjava_searcher() + + return slow_build + + +# --------------------------------------------------------------------------- +# Immutability tests — Java (no mutable per-search state on instance) +# --------------------------------------------------------------------------- + +def test_java_retriever_has_no_mutable_search_state(): + """JavaChainOfCallsRetriever should not have found_path, + last_visited_parent_package_indexes, or exclusion lists on the instance.""" + r = _build_fake_java_retriever() + assert not hasattr(r, 'found_path'), "found_path should not be on instance" + assert not hasattr(r, 'last_visited_parent_package_indexes'), \ + "last_visited_parent_package_indexes should not be on instance" + + +def test_java_tree_dict_stores_parents_directly(): + """tree_dict values should be parent lists, not [parents, exclusions, method_exclusions].""" + r = _build_fake_java_retriever() + for pkg, value in r.tree_dict.items(): + assert isinstance(value, list), f"tree_dict[{pkg}] should be a list of parents" + # Parents are strings, not nested lists + assert all(isinstance(p, str) for p in value), \ + f"tree_dict[{pkg}] should contain parent strings, got {value}" + + +def test_java_retriever_shared_instance_is_safe(): + """Two references to the same Java retriever should be safe since + mutable state lives in _JavaSearchCtx, not on the instance.""" + original = _build_fake_java_retriever() + # Both references point to same object — this is safe now + ref_a = original + ref_b = original + assert ref_a is ref_b + assert ref_a.tree_dict is ref_b.tree_dict + assert ref_a.documents is ref_b.documents + + +# --------------------------------------------------------------------------- +# Immutability tests — non-Java (no mutable per-search state on instance) +# --------------------------------------------------------------------------- + +def test_nonjava_retriever_has_no_mutable_search_state(): + """ChainOfCallsRetriever should not have found_path, + last_visited_parent_package_indexes, or last_visited on the instance.""" + r = _build_fake_go_retriever() + assert not hasattr(r, 'found_path'), "found_path should not be on instance" + assert not hasattr(r, 'last_visited_parent_package_indexes'), \ + "last_visited_parent_package_indexes should not be on instance" + assert not hasattr(r, 'last_visited'), "last_visited should not be on instance" + + +def test_nonjava_tree_dict_stores_parents_directly(): + """tree_dict values should be parent lists, not [parents, exclusions].""" + r = _build_fake_go_retriever() + for pkg, value in r.tree_dict.items(): + assert isinstance(value, list), f"tree_dict[{pkg}] should be a list of parents" + assert all(isinstance(p, str) for p in value), \ + f"tree_dict[{pkg}] should contain parent strings, got {value}" + + +def test_nonjava_retriever_shared_instance_is_safe(): + """Two references to the same non-Java retriever should be safe since + mutable state lives in _SearchCtx, not on the instance.""" + original = _build_fake_go_retriever() + ref_a = original + ref_b = original + assert ref_a is ref_b + assert ref_a.tree_dict is ref_b.tree_dict + assert ref_a.documents is ref_b.documents + + +# --------------------------------------------------------------------------- +# _build_or_get_cached concurrency tests — Java (per-package cache keys) +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_java_same_repo_different_packages_are_serialized(): + """Two Java builds for different packages on the same repo must NOT run + concurrently and must each produce a separate cache entry.""" + _clear_caches() + si = _make_si("https://github.com/example/repo") + build_log = [] + + with patch("vuln_analysis.tools.transitive_code_search._build_searcher", + side_effect=_make_slow_builder(build_log, java=True)): + task1 = asyncio.create_task(_build_or_get_cached(si=si, query="pkg-a:art-a:1.0,ClassA.foo", uber_jar_file_threshold=_DEFAULT_THRESHOLD )) + task2 = asyncio.create_task(_build_or_get_cached(si=si, query="pkg-b:art-b:2.0,ClassB.bar", uber_jar_file_threshold=_DEFAULT_THRESHOLD)) + await asyncio.gather(task1, task2) + + assert len(build_log) == 2, f"Expected 2 Java builds (different packages), got {len(build_log)}" + # Builds must NOT overlap + (tag1, start1, end1) = build_log[0] + (tag2, start2, end2) = build_log[1] + assert start2 >= end1 or start1 >= end2, ( + f"Builds overlapped! {tag1}: [{start1:.4f}, {end1:.4f}], " + f"{tag2}: [{start2:.4f}, {end2:.4f}]" + ) + + +@pytest.mark.asyncio +async def test_different_repos_can_build_concurrently(): + """Builds for different repos should NOT be serialized.""" + _clear_caches() + si_a = _make_si("https://github.com/example/repo-a") + si_b = _make_si("https://github.com/example/repo-b") + build_log = [] + + def slow_build(si, query, uber_jar_file_threshold=_DEFAULT_THRESHOLD,base_dirs=()): + tag = si[0].git_repo.split("/")[-1] + start = time.monotonic() + time.sleep(0.2) + end = time.monotonic() + build_log.append((tag, start, end)) + return _make_nonjava_searcher() + + with patch("vuln_analysis.tools.transitive_code_search._build_searcher", + side_effect=slow_build): + task1 = asyncio.create_task(_build_or_get_cached(si_a, query="pkg-a:art-a:1.0,Foo.bar", uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=())) + task2 = asyncio.create_task(_build_or_get_cached(si_b, query="pkg-b:art-b:2.0,Baz.qux", uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=())) + await asyncio.gather(task1, task2) + + assert len(build_log) == 2 + (_, start1, end1) = build_log[0] + (_, start2, end2) = build_log[1] + assert start2 < end1, ( + f"Different-repo builds should run concurrently but were serialized: " + f"build1=[{start1:.4f}, {end1:.4f}], build2=[{start2:.4f}, {end2:.4f}]" + ) + + +@pytest.mark.asyncio +async def test_same_key_deduplicates_build(): + """Two concurrent requests for the exact same query should only build once.""" + _clear_caches() + si = _make_si("https://github.com/example/repo") + query = "pkg-a:art-a:1.0,ClassA.foo" + build_count = 0 + count_lock = threading.Lock() + + def counting_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD,base_dirs=()): + nonlocal build_count + with count_lock: + build_count += 1 + time.sleep(0.1) + return _make_nonjava_searcher() + + with patch("vuln_analysis.tools.transitive_code_search._build_searcher", + side_effect=counting_build): + task1 = asyncio.create_task(_build_or_get_cached(si, query=query, uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=())) + task2 = asyncio.create_task(_build_or_get_cached(si, query=query, uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=())) + results = await asyncio.gather(task1, task2) + + assert build_count == 1, f"Expected 1 build (deduplicated), got {build_count}" + assert results[0] is results[1], "Both tasks should return the same cached searcher" + + +@pytest.mark.asyncio +async def test_cache_hit_skips_build(): + """If the cache already has an entry for the repo key, no build should happen.""" + _clear_caches() + si = _make_si("https://github.com/example/repo") + query = "pkg-a:art-a:1.0,ClassA.foo" + # Non-Java cache uses repo-level key (git_repo, ref) + repo_key = ("https://github.com/example/repo", "main") + + pre_cached = _make_nonjava_searcher() + _searcher_cache[repo_key] = pre_cached + + build_count = 0 + + def counting_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD,base_dirs=()): + nonlocal build_count + build_count += 1 + return _make_nonjava_searcher() + + with patch("vuln_analysis.tools.transitive_code_search._build_searcher", + side_effect=counting_build): + result = await _build_or_get_cached(si, query=query, uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=()) + + assert build_count == 0, "Build should not run when cache hit exists" + assert result is pre_cached, "Should return the pre-cached searcher" + + +@pytest.mark.asyncio +async def test_java_cache_hit_skips_build(): + """If the cache already has a Java entry for the full key, no build should happen.""" + _clear_caches() + si = _make_si("https://github.com/example/repo") + query = "pkg-a:art-a:1.0,ClassA.foo" + full_key = ("https://github.com/example/repo", "main", "pkg-a:art-a:1.0") + + pre_cached = _make_java_searcher() + _searcher_cache[full_key] = pre_cached + + build_count = 0 + + def counting_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD,base_dirs=()): + nonlocal build_count + build_count += 1 + return _make_java_searcher() + + with patch("vuln_analysis.tools.transitive_code_search._build_searcher", + side_effect=counting_build): + result = await _build_or_get_cached(si, query=query, uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=()) + + assert build_count == 0, "Build should not run when Java cache hit exists" + assert result is pre_cached, "Should return the pre-cached Java searcher" + + +@pytest.mark.asyncio +async def test_build_failure_cleans_up_building_marker(): + """If _build_searcher raises, the building marker must be cleaned up.""" + _clear_caches() + si = _make_si("https://github.com/example/repo") + query = "pkg-a:art-a:1.0,ClassA.foo" + full_key = ("https://github.com/example/repo", "main", "pkg-a:art-a:1.0") + repo_key = ("https://github.com/example/repo", "main") + + def failing_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD,base_dirs=()): + raise RuntimeError("Maven failed") + + with patch("vuln_analysis.tools.transitive_code_search._build_searcher", + side_effect=failing_build): + with pytest.raises(RuntimeError, match="Maven failed"): + await _build_or_get_cached(si, query=query, uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=()) + + assert full_key not in _searcher_building, "Building marker not cleaned up after failure" + assert full_key not in _searcher_cache, "Failed build should not be cached" + assert repo_key not in _searcher_cache, "Failed build should not be cached" + + +@pytest.mark.asyncio +async def test_java_repo_lock_recheck_avoids_redundant_build(): + """When Java task B waits for the repo lock while task A builds, task B should + recheck the cache after acquiring the lock and skip its own build if A + already cached a result for B's key.""" + _clear_caches() + si = _make_si("https://github.com/example/repo") + query_a = "pkg-a:art-a:1.0,ClassA.foo" + query_b = "pkg-b:art-b:2.0,ClassB.bar" + full_key_b = ("https://github.com/example/repo", "main", "pkg-b:art-b:2.0") + + build_count = 0 + count_lock = threading.Lock() + + def build_that_precaches_b(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD,base_dirs=()): + nonlocal build_count + with count_lock: + build_count += 1 + current = build_count + if current == 1: + time.sleep(0.1) + _searcher_cache[full_key_b] = _make_java_searcher() + return _make_java_searcher() + + with patch("vuln_analysis.tools.transitive_code_search._build_searcher", + side_effect=build_that_precaches_b): + task1 = asyncio.create_task(_build_or_get_cached(si, query=query_a, uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=())) + await asyncio.sleep(0.01) + task2 = asyncio.create_task(_build_or_get_cached(si, query=query_b, uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=())) + await asyncio.gather(task1, task2) + + assert build_count == 1, ( + f"Expected 1 build (B should hit cache after repo lock), got {build_count}" + ) + + +# --------------------------------------------------------------------------- +# Non-Java cache sharing tests +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_nonjava_same_repo_different_packages_share_cache(): + """For non-Java ecosystems, different packages on the same repo should + share a single cache entry (repo_key), avoiding redundant builds.""" + _clear_caches() + si = _make_si("https://github.com/example/repo") + build_count = 0 + count_lock = threading.Lock() + + def counting_build(build_si, q, uber_jar_file_threshold=_DEFAULT_THRESHOLD,base_dirs=()): + nonlocal build_count + with count_lock: + build_count += 1 + time.sleep(0.1) + return _make_nonjava_searcher() + + with patch("vuln_analysis.tools.transitive_code_search._build_searcher", + side_effect=counting_build): + task1 = asyncio.create_task( + _build_or_get_cached(si, query="crypto/x509,ParsePKCS1PrivateKey", uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=())) + task2 = asyncio.create_task(_build_or_get_cached(si, query="net/http,ListenAndServe", uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=())) + results = await asyncio.gather(task1, task2) + + # Non-Java: both should resolve to the same (repo, ref) cache entry. + # Second task either deduplicates via _searcher_building or finds the + # repo_key in cache after the repo lock. + assert build_count == 1, ( + f"Expected 1 build for non-Java (shared repo_key), got {build_count}" + ) + assert results[0] is results[1], "Both tasks should return the same cached searcher" + + +@pytest.mark.asyncio +async def test_nonjava_caches_under_repo_key(): + """Non-Java builds should cache under (git_repo, ref), not (git_repo, ref, package).""" + _clear_caches() + si = _make_si("https://github.com/example/repo") + repo_key = ("https://github.com/example/repo", "main") + full_key = ("https://github.com/example/repo", "main", "crypto/x509") + + with patch("vuln_analysis.tools.transitive_code_search._build_searcher", + return_value=_make_nonjava_searcher()): + await _build_or_get_cached(si, query="crypto/x509,ParsePKCS1PrivateKey", uber_jar_file_threshold=_DEFAULT_THRESHOLD, base_dirs=()) + + assert repo_key in _searcher_cache, "Non-Java should cache under repo_key" + assert full_key not in _searcher_cache, "Non-Java should NOT cache under full_key" + + +# --------------------------------------------------------------------------- +# Base Dirs Cache Path Alignment Tests +# --------------------------------------------------------------------------- + +class TestGetGitAndPickleBaseDirs: + """Tests for get_git_and_pickle_base_dirs — reads config from Builder.""" + + @pytest.mark.asyncio + async def test_returns_empty_tuple_when_builder_is_none(self): + result = await get_git_and_pickle_base_dirs(None) + assert result == () + + @pytest.mark.asyncio + async def test_returns_empty_tuple_when_config_is_not_segmentation(self): + builder = MagicMock() + builder.get_function_config.return_value = "some_other_config" + result = await get_git_and_pickle_base_dirs(builder) + assert result == () + builder.get_function_config.assert_called_once_with("cve_segmentation") + + @pytest.mark.asyncio + async def test_returns_dirs_from_segmentation_config(self): + from vuln_analysis.functions.cve_segmentation import CVESegmentationConfig + config = CVESegmentationConfig( + agent_name="cve_agent_executor", + embedder_name="nim_embedder", + base_git_dir="/data/git", + base_pickle_dir="/data/pickle", + ) + builder = MagicMock() + builder.get_function_config.return_value = config + result = await get_git_and_pickle_base_dirs(builder) + assert result == ("/data/git", "/data/pickle") + + +class TestBuildSearcherBaseDirs: + """Tests for _build_searcher — ensures configured base_dirs reach DocumentEmbedding.""" + + def test_empty_base_dirs_uses_defaults(self): + """Empty tuple should create DocumentEmbedding with default dirs.""" + si = [SourceDocumentsInfo(git_repo="https://github.com/example/repo", ref="main", type="code")] + with patch("vuln_analysis.tools.transitive_code_search.get_call_of_chains_retriever") as mock_get_coc, \ + patch("vuln_analysis.tools.transitive_code_search.DocumentEmbedding") as mock_de: + mock_get_coc.return_value = MagicMock() + _build_searcher(si, "pkg,Func", _DEFAULT_THRESHOLD, base_dirs=()) + mock_de.assert_called_once_with(embedding=None) + + def test_valid_base_dirs_passed_to_document_embedding(self): + """Two-element tuple should pass git_directory and pickle_cache_directory.""" + si = [SourceDocumentsInfo(git_repo="https://github.com/example/repo", ref="main", type="code")] + with patch("vuln_analysis.tools.transitive_code_search.get_call_of_chains_retriever") as mock_get_coc, \ + patch("vuln_analysis.tools.transitive_code_search.DocumentEmbedding") as mock_de: + mock_get_coc.return_value = MagicMock() + _build_searcher(si, "pkg,Func", _DEFAULT_THRESHOLD, base_dirs=("/custom/git", "/custom/pickle")) + mock_de.assert_called_once_with( + embedding=None, + pickle_cache_directory="/custom/pickle", + git_directory="/custom/git", + ) + + def test_single_element_tuple_falls_back_to_defaults(self): + """Tuple with wrong length should fall back to defaults, not crash.""" + si = [SourceDocumentsInfo(git_repo="https://github.com/example/repo", ref="main", type="code")] + with patch("vuln_analysis.tools.transitive_code_search.get_call_of_chains_retriever") as mock_get_coc, \ + patch("vuln_analysis.tools.transitive_code_search.DocumentEmbedding") as mock_de: + mock_get_coc.return_value = MagicMock() + _build_searcher(si, "pkg,Func", _DEFAULT_THRESHOLD, base_dirs=("/only/one",)) + mock_de.assert_called_once_with(embedding=None) + + def test_default_parameter_uses_defaults(self): + """Omitting base_dirs entirely should use defaults.""" + si = [SourceDocumentsInfo(git_repo="https://github.com/example/repo", ref="main", type="code")] + with patch("vuln_analysis.tools.transitive_code_search.get_call_of_chains_retriever") as mock_get_coc, \ + patch("vuln_analysis.tools.transitive_code_search.DocumentEmbedding") as mock_de: + mock_get_coc.return_value = MagicMock() + _build_searcher(si, "pkg,Func", _DEFAULT_THRESHOLD) + mock_de.assert_called_once_with(embedding=None) + + +# --------------------------------------------------------------------------- +# Split Clone/Segmentation Pipeline Tests +# --------------------------------------------------------------------------- + +class TestRefScopedCloning: + """Tests for ref-scoped directory paths and concurrent cloning.""" + + def test_different_refs_produce_different_paths(self): + """Same repo with different refs should have different directory paths.""" + from exploit_iq_commons.utils.git_utils import get_repo_path_with_ref + from pathlib import Path + + base_dir = Path("/cache/git") + git_url = "https://github.com/example/repo" + + path_v1 = get_repo_path_with_ref(base_dir, git_url, "v1.0") + path_v2 = get_repo_path_with_ref(base_dir, git_url, "v2.0") + path_main = get_repo_path_with_ref(base_dir, git_url, "main") + + assert path_v1 != path_v2, "Different refs should produce different paths" + assert path_v1 != path_main, "Different refs should produce different paths" + assert path_v2 != path_main, "Different refs should produce different paths" + + assert "v1.0" in str(path_v1) or "v1_0" in str(path_v1) + assert "v2.0" in str(path_v2) or "v2_0" in str(path_v2) + + def test_same_ref_produces_same_path(self): + """Same repo and ref should always produce the same path.""" + from exploit_iq_commons.utils.git_utils import get_repo_path_with_ref + from pathlib import Path + + base_dir = Path("/cache/git") + git_url = "https://github.com/example/repo" + ref = "v1.0" + + path1 = get_repo_path_with_ref(base_dir, git_url, ref) + path2 = get_repo_path_with_ref(base_dir, git_url, ref) + + assert path1 == path2, "Same repo+ref should always produce same path" + + def test_ref_with_slashes_is_sanitized(self): + """Refs with slashes (e.g., refs/tags/v1.0) should be sanitized for filesystem.""" + from exploit_iq_commons.utils.git_utils import get_repo_path_with_ref + from pathlib import Path + + base_dir = Path("/cache/git") + git_url = "https://github.com/example/repo" + ref = "refs/tags/v1.0" + + path = get_repo_path_with_ref(base_dir, git_url, ref) + + assert "/" not in path.name or path.name.count("/") == 0, \ + "Ref slashes should be sanitized in directory name" + + +class TestCloneAndDepsLocking: + """Tests for per-repo locking in DocumentEmbedding.""" + + def test_get_repo_lock_returns_same_lock_for_same_key(self): + """Same (git_repo, ref) should return the same lock instance.""" + from exploit_iq_commons.utils.document_embedding import DocumentEmbedding + + lock1 = DocumentEmbedding._get_repo_lock("https://github.com/a/b", "main") + lock2 = DocumentEmbedding._get_repo_lock("https://github.com/a/b", "main") + + assert lock1 is lock2, "Same repo+ref should return same lock" + + def test_get_repo_lock_returns_different_locks_for_different_refs(self): + """Different refs should get different locks (can clone concurrently).""" + from exploit_iq_commons.utils.document_embedding import DocumentEmbedding + + lock_v1 = DocumentEmbedding._get_repo_lock("https://github.com/a/b", "v1.0") + lock_v2 = DocumentEmbedding._get_repo_lock("https://github.com/a/b", "v2.0") + + assert lock_v1 is not lock_v2, "Different refs should get different locks" + + def test_get_repo_lock_returns_different_locks_for_different_repos(self): + """Different repos should get different locks.""" + from exploit_iq_commons.utils.document_embedding import DocumentEmbedding + + lock_a = DocumentEmbedding._get_repo_lock("https://github.com/a/repo", "main") + lock_b = DocumentEmbedding._get_repo_lock("https://github.com/b/repo", "main") + + assert lock_a is not lock_b, "Different repos should get different locks" + + +class TestVulnerabilityRouting: + """Tests for conditional routing based on vulnerability status.""" + + def _make_vuln_dep(self, has_vulns: bool): + """Create a mock vulnerable dependency.""" + mock = MagicMock() + mock.vulnerable_sbom_packages = ["pkg-1.0"] if has_vulns else [] + return mock + + def _make_engine_input(self, vuln_deps): + """Create a mock AgentMorpheusEngineInput with vulnerable_dependencies.""" + mock = MagicMock() + mock.info.vulnerable_dependencies = vuln_deps + return mock + + def test_route_to_segmentation_when_any_vulnerable(self): + """Should route to segmentation if any CVE has vulnerable packages.""" + state = self._make_engine_input([ + self._make_vuln_dep(has_vulns=False), + self._make_vuln_dep(has_vulns=True), + self._make_vuln_dep(has_vulns=False), + ]) + + vuln_deps = state.info.vulnerable_dependencies + any_vulnerable = any(len(v.vulnerable_sbom_packages) > 0 for v in vuln_deps) + + assert any_vulnerable is True, "Should detect at least one vulnerable" + + def test_route_to_llm_engine_when_none_vulnerable(self): + """Should skip segmentation if no CVE has vulnerable packages.""" + state = self._make_engine_input([ + self._make_vuln_dep(has_vulns=False), + self._make_vuln_dep(has_vulns=False), + ]) + + vuln_deps = state.info.vulnerable_dependencies + any_vulnerable = any(len(v.vulnerable_sbom_packages) > 0 for v in vuln_deps) + + assert any_vulnerable is False, "Should detect no vulnerables" + + def test_route_to_segmentation_when_vuln_deps_is_none(self): + """Should route to segmentation when vulnerable_dependencies is None (unknown state).""" + state = self._make_engine_input(None) + + vuln_deps = state.info.vulnerable_dependencies + if vuln_deps is None: + route = "segmentation" + else: + any_vulnerable = any(len(v.vulnerable_sbom_packages) > 0 for v in vuln_deps) + route = "segmentation" if any_vulnerable else "llm_engine" + + assert route == "segmentation", "None vuln_deps should route to segmentation" + + def test_route_to_llm_engine_when_empty_vuln_deps(self): + """Should skip segmentation when vulnerable_dependencies is empty list.""" + state = self._make_engine_input([]) + + vuln_deps = state.info.vulnerable_dependencies + if vuln_deps is None: + route = "segmentation" + else: + any_vulnerable = any(len(v.vulnerable_sbom_packages) > 0 for v in vuln_deps) + route = "segmentation" if any_vulnerable else "llm_engine" + + assert route == "llm_engine", "Empty vuln_deps should skip segmentation" diff --git a/src/vuln_analysis/tools/tests/test_credential_client.py b/src/vuln_analysis/tools/tests/test_credential_client.py new file mode 100644 index 000000000..d088e6749 --- /dev/null +++ b/src/vuln_analysis/tools/tests/test_credential_client.py @@ -0,0 +1,249 @@ +import base64 +import logging +from unittest.mock import MagicMock, patch + +import pytest +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +from exploit_iq_commons.utils.credential_client import ( + AuthenticationError, + CredentialNotFoundError, + DecryptionError, + fetch_and_decrypt_credential, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_ENCRYPTION_KEY = "test-encryption-key-32-bytes!!!!" # exactly 32 UTF-8 bytes +_IV = b"\x00" * 12 # deterministic IV for tests + + +def _encrypt(plaintext: str, key: str = _ENCRYPTION_KEY, iv: bytes = _IV) -> bytes: + """Encrypt plaintext using AES-256-GCM with the same parameters as the backend.""" + aesgcm = AESGCM(key.encode("utf-8")[:32]) + return aesgcm.encrypt(iv, plaintext.encode("utf-8"), None) + + +def _make_response( + plaintext: str, + credential_type: str = "PAT", + username: str | None = "github-bot", + user_id: str = "alice@example.com", + key: str = _ENCRYPTION_KEY, + iv: bytes = _IV, +) -> dict: + encrypted = _encrypt(plaintext, key=key, iv=iv) + return { + "encryptedSecretValue": base64.b64encode(encrypted).decode(), + "iv": base64.b64encode(iv).decode(), + "username": username, + "credentialType": credential_type, + "userId": user_id, + } + + +def _mock_http_ok(payload: dict) -> MagicMock: + resp = MagicMock() + resp.status_code = 200 + resp.ok = True + resp.json.return_value = payload + return resp + + +def _mock_http_error(status: int) -> MagicMock: + resp = MagicMock() + resp.status_code = status + resp.ok = False + return resp + + +# --------------------------------------------------------------------------- +# Tests: successful decryption +# --------------------------------------------------------------------------- + +class TestFetchAndDecryptCredential: + + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_pat_decryption_success(self, mock_get, mock_ca): + plaintext = "ghp_myPersonalAccessToken123" + mock_get.return_value = _mock_http_ok(_make_response(plaintext, credential_type="PAT")) + + result = fetch_and_decrypt_credential( + credential_id="cred-uuid-1", + jwt_token="scan.jwt.token", + backend_url="https://backend.example.com", + encryption_key=_ENCRYPTION_KEY, + ) + + assert result["secret_value"] == plaintext + assert result["username"] == "github-bot" + assert result["credential_type"] == "PAT" + assert result["user_id"] == "alice@example.com" + + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_ssh_key_decryption_success(self, mock_get, mock_ca): + plaintext = "-----BEGIN OPENSSH PRIVATE KEY-----\nabc123\n-----END OPENSSH PRIVATE KEY-----" + mock_get.return_value = _mock_http_ok( + _make_response(plaintext, credential_type="SSH_KEY", username=None) + ) + + result = fetch_and_decrypt_credential( + credential_id="cred-uuid-2", + jwt_token="scan.jwt.token", + backend_url="https://backend.example.com", + encryption_key=_ENCRYPTION_KEY, + ) + + assert result["secret_value"] == plaintext + assert result["username"] is None + assert result["credential_type"] == "SSH_KEY" + + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_correct_url_and_auth_header(self, mock_get, mock_ca): + mock_get.return_value = _mock_http_ok(_make_response("token")) + + fetch_and_decrypt_credential( + credential_id="cred-uuid-3", + jwt_token="my.jwt", + backend_url="https://backend.example.com", + encryption_key=_ENCRYPTION_KEY, + ) + + mock_get.assert_called_once_with( + "https://backend.example.com/api/v1/credentials/cred-uuid-3", + headers={"Authorization": "Bearer my.jwt"}, + timeout=10, + verify=False, + ) + + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_backend_url_trailing_slash_stripped(self, mock_get, mock_ca): + mock_get.return_value = _mock_http_ok(_make_response("token")) + + fetch_and_decrypt_credential( + credential_id="cred-uuid-4", + jwt_token="my.jwt", + backend_url="https://backend.example.com/", # trailing slash + encryption_key=_ENCRYPTION_KEY, + ) + + call_url = mock_get.call_args[0][0] + assert "//" not in call_url.split("://")[1], "Double slash in URL" + + +# --------------------------------------------------------------------------- +# Tests: error handling +# --------------------------------------------------------------------------- + +class TestErrorHandling: + + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_404_raises_credential_not_found(self, mock_get, mock_ca): + mock_get.return_value = _mock_http_error(404) + + with pytest.raises(CredentialNotFoundError): + fetch_and_decrypt_credential( + credential_id="cred-id", + jwt_token="jwt", + backend_url="https://backend.example.com", + encryption_key=_ENCRYPTION_KEY, + ) + + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_401_raises_authentication_error(self, mock_get, mock_ca): + mock_get.return_value = _mock_http_error(401) + + with pytest.raises(AuthenticationError): + fetch_and_decrypt_credential( + credential_id="cred-id", + jwt_token="jwt", + backend_url="https://backend.example.com", + encryption_key=_ENCRYPTION_KEY, + ) + + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_403_raises_authentication_error(self, mock_get, mock_ca): + mock_get.return_value = _mock_http_error(403) + + with pytest.raises(AuthenticationError): + fetch_and_decrypt_credential( + credential_id="cred-id", + jwt_token="jwt", + backend_url="https://backend.example.com", + encryption_key=_ENCRYPTION_KEY, + ) + + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_500_raises_runtime_error(self, mock_get, mock_ca): + mock_get.return_value = _mock_http_error(500) + + with pytest.raises(RuntimeError): + fetch_and_decrypt_credential( + credential_id="cred-id", + jwt_token="jwt", + backend_url="https://backend.example.com", + encryption_key=_ENCRYPTION_KEY, + ) + + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_wrong_key_raises_decryption_error(self, mock_get, mock_ca): + mock_get.return_value = _mock_http_ok(_make_response("secret", key=_ENCRYPTION_KEY)) + + with pytest.raises(DecryptionError): + fetch_and_decrypt_credential( + credential_id="cred-id", + jwt_token="jwt", + backend_url="https://backend.example.com", + encryption_key="wrong-key-32-bytes-xxxxxxxxxxxxxx", + ) + + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_network_error_raises_runtime_error(self, mock_get, mock_ca): + import requests as req_lib + mock_get.side_effect = req_lib.RequestException("Connection refused") + + with pytest.raises(RuntimeError, match="Network error"): + fetch_and_decrypt_credential( + credential_id="cred-id", + jwt_token="jwt", + backend_url="https://backend.example.com", + encryption_key=_ENCRYPTION_KEY, + ) + + +# --------------------------------------------------------------------------- +# Tests: secret_value never appears in logs +# --------------------------------------------------------------------------- + +class TestNoPlaintextInLogs: + + @patch("exploit_iq_commons.utils.credential_client._validate_ca_bundle", return_value=False) + @patch("exploit_iq_commons.utils.credential_client.requests.get") + def test_secret_not_logged(self, mock_get, mock_ca, caplog): + secret = "super-secret-token-must-not-appear-in-logs" + mock_get.return_value = _mock_http_ok(_make_response(secret)) + + with caplog.at_level(logging.DEBUG, logger="exploit_iq_commons.utils.credential_client"): + fetch_and_decrypt_credential( + credential_id="cred-uuid-log", + jwt_token="jwt", + backend_url="https://backend.example.com", + encryption_key=_ENCRYPTION_KEY, + ) + + for record in caplog.records: + assert secret not in record.getMessage(), ( + f"Secret value leaked into log message: {record.getMessage()}" + ) diff --git a/src/vuln_analysis/tools/tests/test_segmenter.py b/src/vuln_analysis/tools/tests/test_segmenter.py new file mode 100644 index 000000000..2e4fdcfed --- /dev/null +++ b/src/vuln_analysis/tools/tests/test_segmenter.py @@ -0,0 +1,215 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# dummy comment to trigger a build 2 +from pathlib import Path +import pytest +from exploit_iq_commons.utils.c_segmenter_custom import CSegmenterExtended +from exploit_iq_commons.utils.functions_parsers.c_lang_function_parsers import _FUNCTION_PATTERN_REGEX +from langchain_community.document_loaders.parsers.language.c import CSegmenter + +def _get_function_name_from_segment(segment: str) -> str: + """ + Extract function name from C code segment using regex pattern. + This parser function can be used for testing. + """ + # Normalize line endings + code = segment.replace('\r\n', '\n') + m = _FUNCTION_PATTERN_REGEX.search(code, timeout=0.05) + return m.group('name') if m else "" + + +# --- Test Data Fixtures --- +# These fixtures load the test data once per test session. +# They assume the .c files are in the same directory as this test script. + +@pytest.fixture(scope="module") +def c_test_files_dir(): + """Returns the directory of this test file.""" + return Path(__file__).parent / "tests_data" + +@pytest.fixture(scope="module") +def err_c_code(c_test_files_dir): + file_path = c_test_files_dir / "err.c" + return file_path.read_text(encoding="utf-8") + +@pytest.fixture(scope="module") +def ess_lib_c_code(c_test_files_dir): + file_path = c_test_files_dir / "ess_lib.c" + return file_path.read_text(encoding="utf-8") + +@pytest.fixture(scope="module") +def parser_c_code(c_test_files_dir): + file_path = c_test_files_dir / "parser.c" + return file_path.read_text(encoding="utf-8") + +@pytest.fixture(scope="module") +def pk7_asn1_c_code(c_test_files_dir): + file_path = c_test_files_dir / "pk7_asn1.c" + return file_path.read_text(encoding="utf-8") + +# --- Integration Tests for CSegmenterExtended --- + +def test_integration_err_c(err_c_code: str): + """ + Tests segmentation on err.c. + Checks for specific functions and ensures comments are gone. + """ + segmenter = CSegmenterExtended(err_c_code) + segments = segmenter.extract_functions_classes() + + # Preprocess code: remove comments and macro blocks + original_segmenter = CSegmenter(err_c_code) + original_segments = original_segmenter.extract_functions_classes() + + assert len(segments) >= len(original_segments), f"Should find more or equal than {len(original_segments)} segments in err.c, found {len(segments)}" + + names = set() + for seg in segments: + assert "//" not in seg + assert "/*" not in seg + name = _get_function_name_from_segment(seg) + if name: + names.add(name) + + assert "OSSL_ERR_STATE_free" in names + assert "err_cleanup" in names + assert "ossl_err_get_state_int" in names + assert "err_load_strings" in names + assert "err_string_data_hash" in names + assert "ERR_get_error" in names + assert "ERR_get_state" in names + + +def test_integration_parser_c(parser_c_code): + """ + Tests segmentation on the large parser.c file. + This validates the "hidden segment" logic by ensuring + it can find many functions in a complex file. + """ + segmenter = CSegmenterExtended(parser_c_code) + segments = segmenter.extract_functions_classes() + # Preprocess code: remove comments and macro blocks + original_segmenter = CSegmenter(parser_c_code) + original_segments = original_segmenter.extract_functions_classes() + + assert len(segments) >= len(original_segments), f"Should find more or equal than {len(original_segments)} segments in parser.c, found {len(segments)}" + + names = {_get_function_name_from_segment(seg) for seg in segments if _get_function_name_from_segment(seg)} + + # Check for a mix of public and static functions + assert "xmlParseDocument" in names + assert "xmlParseStartTag" in names + assert "xmlParsePI" in names + assert "xmlParserEntityCheck" in names # static function + assert "xmlParseCommentComplex" in names # static function + assert "xmlCtxtResetPush" in names # public function + assert "xmlParseMemory" in names # parses XML from memory buffer + assert "xmlReadMemory" in names # reads XML from memory + assert "xmlCreateMemoryParserCtxt" in names # creates parser context from memory + assert "xmlParseFile" in names # popular public API + assert "xmlReadFile" in names # popular public API + assert "xmlParseEntityRef" in names # important parsing function + assert "xmlParseElement" in names # core parsing function + assert "xmlParseAttribute" in names # common parsing function + + +def test_integration_pk7_asn1_c_bug(pk7_asn1_c_code): + """ + Tests segmentation on pk7_asn1.c. + + This test *should fail* with the original implementation. + The `remove_macro_blocks` function incorrectly removes + definitions like `ASN1_ADB(PKCS7) = { ... }` and + `IMPLEMENT_ASN1_FUNCTIONS(PKCS7)`. + + This test now PASSES because the bug is fixed. + """ + segmenter = CSegmenterExtended(pk7_asn1_c_code) + segments = segmenter.extract_functions_classes() + + original_segmenter = CSegmenter(pk7_asn1_c_code) + original_segments = original_segmenter.extract_functions_classes() + + assert len(segments) >= len(original_segments), f"Should find more or equal than {len(original_segments)} segments in pk7_asn1.c, found {len(segments)}" + + + + names = {_get_function_name_from_segment(seg) for seg in segments if _get_function_name_from_segment(seg)} + + # These functions will be missed if remove_macro_blocks is buggy + assert "d2i_PKCS7" in names + assert "i2d_PKCS7" in names + assert "PKCS7_new" in names + assert "PKCS7_free" in names + assert "si_cb" in names # static function + + +def test_integration_ess_lib_c(ess_lib_c_code): + """ + Tests segmentation on ess_lib.c. + Checks for public and static functions. + """ + segmenter = CSegmenterExtended(ess_lib_c_code) + segments = segmenter.extract_functions_classes() + + original_segmenter = CSegmenter(ess_lib_c_code) + original_segments = original_segmenter.extract_functions_classes() + + assert len(segments) >= len(original_segments), f"Should find more or equal than {len(original_segments)} segments in ess_lib.c, found {len(segments)}" + + names = {_get_function_name_from_segment(seg) for seg in segments if _get_function_name_from_segment(seg)} + + assert "OSSL_ESS_signing_cert_new_init" in names + assert "ESS_CERT_ID_new_init" in names # static function + assert "find" in names # static function + assert "OSSL_ESS_check_signing_certs" in names + assert "OSSL_ESS_signing_cert_v2_new_init" in names # public function (v2 version) + assert "ESS_CERT_ID_V2_new_init" in names # static function (v2 version) + assert "ess_issuer_serial_cmp" in names # static comparison function + + +@pytest.fixture(scope="module") +def jsonpath_c_code(c_test_files_dir): + file_path = c_test_files_dir / "jsonpath.c" + return file_path.read_text(encoding="utf-8") + + +def test_jsonpath_c_define_macros_captured(jsonpath_c_code): + """ + Lowercase function-like #define macros (read_byte, read_int32, read_int32_n) + should be captured as dummy function segments. The raw ``#define`` text + must not leak through — only the synthesized ``void name(args) { body }`` form. + Functions that *use* these macros (e.g. jspInitByBuffer) must still be captured. + """ + segmenter = CSegmenterExtended(jsonpath_c_code) + segments = segmenter.extract_functions_classes() + + assert len(segments) > 0, "Should find segments in jsonpath.c" + + for seg in segments: + assert "#define read_byte" not in seg, ( + "Raw #define read_byte should not appear — only the dummy function form" + ) + + names = {_get_function_name_from_segment(seg) for seg in segments if _get_function_name_from_segment(seg)} + + assert "read_byte" in names, "read_byte macro should be captured as a dummy function segment" + assert "read_int32" in names, "read_int32 macro should be captured as a dummy function segment" + assert "read_int32_n" in names, "read_int32_n macro should be captured as a dummy function segment" + + assert "jspInitByBuffer" in names, ( + "jspInitByBuffer (which uses read_byte) should be captured as a function segment" + ) + assert "jspInit" in names diff --git a/src/vuln_analysis/tools/tests/test_source_code_git_loader.py b/src/vuln_analysis/tools/tests/test_source_code_git_loader.py new file mode 100644 index 000000000..2858753f0 --- /dev/null +++ b/src/vuln_analysis/tools/tests/test_source_code_git_loader.py @@ -0,0 +1,73 @@ +import pytest +import shutil + +from exploit_iq_commons.utils.source_code_git_loader import SourceCodeGitLoader + + +@pytest.mark.parametrize("https_url,expected_ssh", [ + ( + "https://github.com/org/repo.git", + "git@github.com:org/repo.git", + ), + ( + "https://github.com/org/repo", + "git@github.com:org/repo", + ), + ( + "https://gitlab.com/group/subgroup/repo.git", + "git@gitlab.com:group/subgroup/repo.git", + ), + ( + "git@github.com:org/repo.git", + "git@github.com:org/repo.git", # already SSH — unchanged + ), +]) +def test_https_to_ssh_url(https_url, expected_ssh): + assert SourceCodeGitLoader._https_to_ssh_url(https_url) == expected_ssh + + +def test_public_clone_fails_instead_of_prompting(tmp_path): + """Unauthenticated clone of a missing/private repo must fail fast, not hang on a prompt.""" + clone_url = "https://github.com/example/missing" + repo_path = tmp_path / "repo" + + loader = SourceCodeGitLoader(repo_path=repo_path, clone_url=clone_url, ref="main") + + with pytest.raises(ValueError, match=r"Failed to clone repository .+missing"): + loader.load_repo() + + assert not repo_path.exists() or not (repo_path / ".git").exists() + + +def test_load_repo_branch_checkout(tmp_path): + """Branch names must be checkable after a shallow clone (APPENG-4896).""" + repo_url = "https://github.com/medik8s/node-remediation-console" + repo_path = tmp_path / "branch_checkout_test" + branch = "release-0.6" + + loader = SourceCodeGitLoader(repo_path=repo_path, clone_url=repo_url, ref=branch) + repo = loader.load_repo() + assert repo.head.commit.hexsha == repo.commit(branch).hexsha + + shutil.rmtree(repo_path) + + +@pytest.mark.parametrize("refs", [ + ("3.1.43", "3.1.42", "tag switch"), + ("ba5c10d3655c4fec714294cbc2ae0829c44dc046", + "5b098ff41266d6990237cbade5b6e27cab23557f", "sha switch"), +]) +def test_load_repo_ref_switch(tmp_path, refs): + repo_url = "https://github.com/gitpython-developers/GitPython" + repo_path = tmp_path / "git_python_test_repo" + ref_a, ref_b, desc = refs + + loader_a = SourceCodeGitLoader(repo_path=repo_path, clone_url=repo_url, ref=ref_a) + repo_a = loader_a.load_repo() + assert repo_a.head.commit.hexsha == repo_a.commit(ref_a).hexsha, f"Initial {desc} failed" + + loader_b = SourceCodeGitLoader(repo_path=repo_path, clone_url=repo_url, ref=ref_b) + repo_b = loader_b.load_repo() + assert repo_b.head.commit.hexsha == repo_b.commit(ref_b).hexsha, f"Ref switch {desc} failed" + + shutil.rmtree(repo_path) diff --git a/src/vuln_analysis/tools/tests/test_transitive_code_search.py b/src/vuln_analysis/tools/tests/test_transitive_code_search.py new file mode 100644 index 000000000..80d3dd8f9 --- /dev/null +++ b/src/vuln_analysis/tools/tests/test_transitive_code_search.py @@ -0,0 +1,1274 @@ +import logging +from contextlib import aclosing +import pytest +from unittest.mock import patch, MagicMock +from langchain_core.documents import Document + +from exploit_iq_commons.data_models.common import AnalysisType +from vuln_analysis.data_models.state import AgentMorpheusEngineState +from vuln_analysis.tools.transitive_code_search import transitive_search, TransitiveCodeSearchToolConfig, _searcher_cache +from exploit_iq_commons.data_models.input import (AgentMorpheusEngineInput, AgentMorpheusInput, + ImageInfoInput, SourceDocumentsInfo, ManualSBOMInfoInput +, SBOMPackage, ScanInfoInput, VulnInfo, AgentMorpheusInfo) +from vuln_analysis.runtime_context import ctx_state +from vuln_analysis.tools.tests.mock_documents import (python_script_example, python_init_function_example, + python_full_document_example, python_parse_function_example, + python_mock_function_in_use, python_mock_file, + python_dateutil_function, python_dateutil_full_source, + python_app_using_dateutil, python_app_function_using_dateutil + ) + +from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager +from exploit_iq_commons.utils.dep_tree import PythonDependencyTreeBuilder +from exploit_iq_commons.utils.document_embedding import DocumentEmbedding +from exploit_iq_commons.utils.git_utils import sanitize_git_url_for_path, get_repo_path_with_ref +from pathlib import Path + +@pytest.fixture(autouse=True) +def patch_repo_path_with_fallback(): + """ + Patch get_repo_path to fall back to old cache format for existing CI caches. + + New format: {base_dir}/{sanitized_url}_{sanitized_ref} + Old format: {base_dir}/{sanitized_url} + """ + def get_repo_path_with_fallback(self, source_info): + new_path = get_repo_path_with_ref( + self._git_directory, source_info.git_repo, source_info.ref + ) + if new_path.exists(): + return new_path + + old_path = self._git_directory / sanitize_git_url_for_path(source_info.git_repo) + if old_path.exists(): + return old_path + + return new_path + + with patch.object(DocumentEmbedding, 'get_repo_path', get_repo_path_with_fallback): + yield + + +transitive_code_search_runner_coroutine = None + + +@pytest.mark.asyncio +async def test_transitive_search_golang_1(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + logging.basicConfig(level=logging.DEBUG) + set_input_for_next_run(git_repository="https://github.com/kuadrant/authorino", + git_ref="f792cd138891dc1ead99fd089aa757fbca3aace9", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"]) + result = await transitive_code_search_runner_coroutine("crypto/x509,ParsePKCS1PrivateKey") + (path_found, list_path) = result + assert path_found is True + assert len(list_path) > 1 + assert not list_path[-1].startswith("vendor/") + + +@pytest.mark.asyncio +async def test_transitive_search_golang_2(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run(git_repository="https://github.com/openshift/oc", + git_ref="0000b3ef257d07f423dfdf9b6d274214d1b0c846", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"]) + result = await transitive_code_search_runner_coroutine("crypto/rsa,Verify") + (path_found, list_path) = result + + assert path_found is False + assert len(list_path) == 1 + assert "NOTE" in list_path[0], "Should warn about dummy-branch fallback for stdlib package" + + +@pytest.mark.asyncio +async def test_transitive_search_golang_3(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run(git_repository="https://github.com/zvigrinberg/router", + git_ref="b49f382f59d6479af9ea26f067ee5cb4e1dd13d9", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"]) + result = await transitive_code_search_runner_coroutine("github.com/beorn7/perks,NewTargeted") + (path_found, list_path) = result + + assert path_found is True + assert len(list_path) == 5 + + +@pytest.mark.asyncio +async def test_transitive_search_golang_4(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run(git_repository="https://github.com/openshift/oc-mirror", + git_ref="b137a53a5360a41a70432ea2bfc98a6cee6f7a4a", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"]) + result = await transitive_code_search_runner_coroutine("github.com/mholt/archiver,Unarchive") + (path_found, list_path) = result + + assert path_found is True + assert len(list_path) == 2 + + +@pytest.mark.asyncio +async def test_transitive_search_golang_5(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run(git_repository="https://github.com/openshift/metallb", + git_ref="3be9d86e5752c6974a2fea99d9373af4f2225e6b", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"]) + result = await transitive_code_search_runner_coroutine("github.com/jpillora/backoff,ForAttempt") + (path_found, list_path) = result + + assert path_found is False + assert len(list_path) is 1 + +# Test fix of https://issues.redhat.com/browse/APPENG-3435 +@pytest.mark.asyncio +async def test_transitive_search_golang_6(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run(git_repository="https://github.com/open-telemetry/opentelemetry-operator", + git_ref="32443f9b956f7aba768fcc1b350d490a34218939", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"]) + result = await transitive_code_search_runner_coroutine("net/http,ListenAndServe") + (path_found, list_path) = result + print(result) + assert path_found is True + assert len(list_path) is 2 + + +def set_input_for_next_run(git_repository: str, git_ref: str, included_extensions: list[str], + excluded_extensions: list[str], sbom_packages: list[SBOMPackage] = None): + source_code_info = [SourceDocumentsInfo(type='code', git_repo=git_repository, + ref=git_ref, include=included_extensions, + exclude=excluded_extensions)] + if sbom_packages: + sbom_info_input = ManualSBOMInfoInput(packages=sbom_packages) + else: + sbom_info_input = ManualSBOMInfoInput(packages=[SBOMPackage(name="a", version="1.0", system="blabla")]) + morpheus_input = AgentMorpheusInput(image=ImageInfoInput(source_info=source_code_info, + sbom_info=sbom_info_input, + analysis_type=AnalysisType.IMAGE), + scan=ScanInfoInput(vulns=[VulnInfo(vuln_id="CVE-2025-1234")])) + engine_input = AgentMorpheusEngineInput(input=morpheus_input, info=AgentMorpheusInfo()) + state: AgentMorpheusEngineState = AgentMorpheusEngineState(original_input=engine_input, + code_vdb_path="", + doc_vdb_path="", + code_index_path="", + cve_intel=[]) + ctx_state.set(state) + + +async def get_transitive_code_runner_function(): + transitive_code_search = transitive_search(config=TransitiveCodeSearchToolConfig(), builder=None) + # aclosing automatically calls gen.aclose() + async with aclosing(transitive_code_search.gen) as gen: + async for function in gen: + return function.single_fn + +python_dependency_tree_mock_output = ( + 'deptree==0.0.12 # deptree\n' + ' importlib-metadata==8.7.0 # importlib-metadata\n' + ' zipp==3.23.0 # zipp>=3.20\n' + ' setuptools==80.9.0 # setuptools\n' + 'flask==3.1.2 # flask\n' + ' blinker==1.9.0 # blinker>=1.9.0\n' + ' click==8.3.0 # click>=8.1.3\n' + ' itsdangerous==2.2.0 # itsdangerous>=2.2.0\n' + ' jinja2==3.1.6 # jinja2>=3.1.2\n' + ' markupsafe==3.0.3 # MarkupSafe>=2.0\n' + ' markupsafe==3.0.3 # markupsafe>=2.1.1\n' + ' werkzeug==3.1.3 # werkzeug>=3.1.0\n' + ' markupsafe==3.0.3 # MarkupSafe>=2.1.1\n' + ' mock-package==1.1.1' +) +java_script_dependency_tree_mock_output = '''{ + "name": "root_project", + "version": "1.0.0", + "dependencies": { + "vulnerable-pkg": { + "version": "1.0.0" + }, + "another-pkg": { + "version": "2.0.0" + }, + "deep-pkg": { + "version": "1.0.0" + }, + "middle-pkg": { + "version": "1.0.0", + "dependencies": { + "deep-pkg": { + "version": "1.0.0" + } + } + } + } +}''' + + +def mock_file_open(*args, **kwargs): + file_path = args[0] if args else kwargs.get('file', '') + mock_file = MagicMock() + mock_file.__enter__ = MagicMock(return_value=mock_file) + mock_file.__exit__ = MagicMock(return_value=None) + if 'ecosystem_data.txt' in str(file_path): + mock_file.read.return_value = "PYTHON" + elif 'requirements.txt' in str(file_path): + mock_file.__iter__ = MagicMock(return_value=iter([ + "flask==2.0.1\n", + "werkzeug==2.0.1\n", + "requests==2.25.1\n" + ])) + else: + mock_file.read.return_value = "" + return mock_file + + +@pytest.mark.asyncio +@pytest.mark.parametrize("test_case", [ + { + "name": "python_1", + "search_query": "werkzeug,formparser.MultiPartParser", + "expected_path_found": True, + "expected_list_length": 2, + "mock_documents": [python_script_example, python_init_function_example, python_full_document_example] + }, + { + "name": "python_2", + "search_query": "werkzeug,formparser.MultiPartParser.parse", + "expected_path_found": False, + "expected_list_length": 1, + "mock_documents": [python_script_example, python_init_function_example, python_full_document_example, python_parse_function_example] + }, + { + "name": "python_3", + "search_query": "mock_package,mock_function_in_use", + "expected_path_found": True, + "expected_list_length": 3, + "mock_documents": [python_script_example, python_init_function_example, python_full_document_example, python_parse_function_example, python_mock_function_in_use, python_mock_file] + } +]) +@patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=python_dependency_tree_mock_output) +@patch('builtins.open', side_effect=mock_file_open) +async def test_transitive_search_python_parameterized(mock_open, mock_run_command,test_case): + """Parameterized test that runs all existing test cases with their respective configurations.""" + _searcher_cache.clear() + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + logging.basicConfig(level=logging.DEBUG) + + set_input_for_next_run( + git_repository='https://github.com/something/something', + git_ref='f792cd138891dc1ead99fd089aa757fbca3aace9', + included_extensions=['**/*.py'], + excluded_extensions=['**/*test*.py'] + ) + + with patch('exploit_iq_commons.utils.document_embedding.retrieve_from_cache', + return_value=(test_case["mock_documents"], True)): + result = await transitive_code_search_runner_coroutine(test_case["search_query"]) + + (path_found, list_path) = result + print(f"DEBUG: Test case {test_case['name']}") + print(f"DEBUG: path_found = {path_found}") + print(f"DEBUG: list_path = {list_path}") + print(f"DEBUG: len(list_path) = {len(list_path)}") + assert path_found == test_case["expected_path_found"] + assert len(list_path) == test_case["expected_list_length"] + +@pytest.mark.asyncio +async def test_python_transitive_search(): + """Test that runs with a real repository""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + logging.basicConfig(level=logging.DEBUG) + + set_input_for_next_run( + git_repository='https://github.com/TamarW0/example-repo', + git_ref='f0bf2e6cb4ae65c53fa1764db6302b73cd52c6c0', + included_extensions=['**/*.py'], + excluded_extensions=['**/*test*.py'] + ) + + result = await transitive_code_search_runner_coroutine("werkzeug,formparser.MultiPartParser") + + (path_found, list_path) = result + print(f"DEBUG: path_found = {path_found}") + print(f"DEBUG: list_path = {list_path}") + print(f"DEBUG: len(list_path) = {len(list_path)}") + assert path_found == True + assert len(list_path) == 2 + +@pytest.mark.asyncio +async def test_repeated_runs_of_transitive_search(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + logging.basicConfig(level=logging.DEBUG) + + set_input_for_next_run( + git_repository='https://github.com/TamarW0/example-repo', + git_ref='f0bf2e6cb4ae65c53fa1764db6302b73cd52c6c0', + included_extensions=['**/*.py'], + excluded_extensions=['**/*test*.py'] + ) + + input = "werkzeug,formparser.MultiPartParser" + result_1 = await transitive_code_search_runner_coroutine(input) + result_2 = await transitive_code_search_runner_coroutine(input) + + (path_found_1, list_path_1) = result_1 + (path_found_2, list_path_2) = result_2 + print(f"DEBUG: {path_found_1=}, {path_found_2=}") + print(f"DEBUG: {list_path_1=}, {list_path_2=}") + print(f"DEBUG: {len(list_path_1)=}, {len(list_path_2)=}") + assert path_found_1 == path_found_2 == True + assert len(list_path_1) == len(list_path_2) == 2 + + +@pytest.mark.asyncio +async def test_c_transitive_search(): + """Test that runs with a real postgres repository""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + logging.basicConfig(level=logging.DEBUG) + + #create sample sbom packages + sbom_list = [ + SBOMPackage(name='openssl', version='3.5.1-5.el9', path=None, system='rpm'), + SBOMPackage(name='libxml2', version='2.9.13-9.el9_6', path=None, system='rpm'), + SBOMPackage(name='libxslt', version='1.1.34-13.el9_6', path=None, system='rpm') + ] + + set_input_for_next_run( + git_repository='https://github.com/postgres/postgres', + git_ref='REL_13_14', + included_extensions=['**/*.c','**/*.h'], + excluded_extensions=['**/*test*.c',"**/docs/**/*","**/doc/**/*"] + ) + # set rpm cache directory + RPMDependencyManager.get_instance().set_rpm_cache_dir(".cache/am_cache/rpms") + # enable unit test mode + RPMDependencyManager.get_instance().enableUnitTestMode() + # set sbom list + RPMDependencyManager.get_instance().sbom = sbom_list + # run transitive code search + result = await transitive_code_search_runner_coroutine("libxml2,xmlParseComment") + (path_found, list_path) = result + print(f"DEBUG: path_found = {path_found}") + print(f"DEBUG: list_path = {list_path}") + print(f"DEBUG: len(list_path) = {len(list_path)}") + assert len(list_path) == 2 + assert path_found == True + + +@pytest.mark.asyncio +async def test_c_transitive_search_2(): + """Test that runs with a real postgres repository""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + logging.basicConfig(level=logging.DEBUG) + + #create sample sbom packages + sbom_list = [ + SBOMPackage(name='openssl', version='3.5.1-5.el9', path=None, system='rpm'), + SBOMPackage(name='libxml2', version='2.9.13-9.el9_6', path=None, system='rpm'), + SBOMPackage(name='libxslt', version='1.1.34-13.el9_6', path=None, system='rpm') + ] + + set_input_for_next_run( + git_repository='https://github.com/postgres/postgres', + git_ref='REL_13_14', + included_extensions=['**/*.c','**/*.h'], + excluded_extensions=['**/*test*.c',"**/docs/**/*","**/doc/**/*"] + ) + RPMDependencyManager.get_instance().set_rpm_cache_dir(".cache/am_cache/rpms") + RPMDependencyManager.get_instance().enableUnitTestMode() + RPMDependencyManager.get_instance().sbom = sbom_list + result = await transitive_code_search_runner_coroutine("libxslt,xsltParseStylesheetPI") + (path_found, list_path) = result + print(f"DEBUG: path_found = {path_found}") + print(f"DEBUG: list_path = {list_path}") + print(f"DEBUG: len(list_path) = {len(list_path)}") + assert len(list_path) == 1 + assert path_found == False + +@pytest.mark.asyncio +async def test_transitive_search_java_1(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run(git_repository="https://github.com/cryostatio/cryostat", + git_ref="8f753753379e9381429b476aacbf6890ef101438", + included_extensions=["**/*.java"], + excluded_extensions=["target/**/*", + "build/**/*", + "*.class", + ".gradle/**/*", + ".mvn/**/*", + ".gitignore", + "test/**/*", + "tests/**/*", + "src/test/**/*", + "pom.xml", + "build.gradle"]) + result = await transitive_code_search_runner_coroutine("commons-beanutils:commons-beanutils:1.9.4,org.apache.commons.beanutils.PropertyUtilsBean.getProperty") + (path_found, list_path) = result + print(result) + assert path_found is False + assert len(list_path) is 1 + +@pytest.mark.asyncio +async def test_transitive_search_java_2(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run(git_repository="https://github.com/cryostatio/cryostat", + git_ref="8f753753379e9381429b476aacbf6890ef101438", + included_extensions=["**/*.java"], + excluded_extensions=["target/**/*", + "build/**/*", + "*.class", + ".gradle/**/*", + ".mvn/**/*", + ".gitignore", + "test/**/*", + "tests/**/*", + "src/test/**/*", + "pom.xml", + "build.gradle"]) + result = await transitive_code_search_runner_coroutine("org.apache.commons:commons-lang3:3.14.0,org.apache.commons.lang3.StringUtils.isBlank") + (path_found, list_path) = result + print(result) + assert path_found is True + assert len(list_path) is 2 + # list_path contains concise strings: "source_path :: function_signature" + assert 'src/main/java/io/cryostat' in list_path[-1] + assert 'StringUtils' in list_path[0] and 'isBlank(' in list_path[0] + +@pytest.mark.skip +@pytest.mark.asyncio +async def test_transitive_search_java_3(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run(git_repository="https://github.com/cryostatio/cryostat", + git_ref="8f753753379e9381429b476aacbf6890ef101438", + included_extensions=["**/*.java"], + excluded_extensions=["target/**/*", + "build/**/*", + "*.class", + ".gradle/**/*", + ".mvn/**/*", + ".gitignore", + "test/**/*", + "tests/**/*", + "src/test/**/*", + "pom.xml", + "build.gradle"]) + result = await transitive_code_search_runner_coroutine(",java.net.URLEncoder.encode") + (path_found, list_path) = result + print(result) + assert path_found is True + assert len(list_path) > 1 + assert 'src/main/java/io/cryostat' in list_path[-1] + +@pytest.mark.asyncio +async def test_transitive_search_java_4(): + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run(git_repository="https://github.com/cryostatio/cryostat", + git_ref="8f753753379e9381429b476aacbf6890ef101438", + included_extensions=["**/*.java"], + excluded_extensions=["target/**/*", + "build/**/*", + "*.class", + ".gradle/**/*", + ".mvn/**/*", + ".gitignore", + "test/**/*", + "tests/**/*", + "src/test/**/*", + "pom.xml", + "build.gradle"]) + result = await transitive_code_search_runner_coroutine("org.openjdk.jmc:flightrecorder.configuration:9.0.0,org.openjdk.jmc.flightrecorder.configuration.events.SchemaVersion.fromBeanVersion") + (path_found, list_path) = result + print(result) + assert path_found is True + assert len(list_path) > 1 + assert 'src/main/java/io/cryostat' in list_path[-1] + +@pytest.mark.asyncio +async def test_java_script_transitive_search_1(): + """Test that runs with a real repository""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + logging.basicConfig(level=logging.DEBUG) + + set_input_for_next_run( + git_repository="https://github.com/trustification/exhort-javascript-api", + git_ref='b951581c8f1bf77721537ea135a546f39e1d7a59', + included_extensions=['**/*.js'], + excluded_extensions=['**/*test*.js'] + ) + + for input in ["PackageURL.constructor", "PackageURL", "PackageURL.fromString"]: + result = await transitive_code_search_runner_coroutine(f"packageurl-js,{input}") + + (path_found, list_path) = result + print(f"DEBUG: path_found = {path_found}") + print(f"DEBUG: list_path = {list_path}") + print(f"DEBUG: len(list_path) = {len(list_path)}") + assert path_found == True + assert len(list_path) == 2 + + +@pytest.mark.asyncio +async def test_java_script_transitive_search_2(): + """Test that runs with a real repository""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + logging.basicConfig(level=logging.DEBUG) + + set_input_for_next_run( + git_repository="https://github.com/openshift/monitoring-plugin", + git_ref='87571978825e63392a56f6920e068af3a71cda6e', + included_extensions=['**/*.js', '**/*.ts', '**/*.tsx'], + excluded_extensions=['**/*test*.js', '**/*test*.ts', '**/*test*.tsx', '**/*.spec.*'] + ) + + result = await transitive_code_search_runner_coroutine("lodash-es,filter") + + (path_found, list_path) = result + print(f"DEBUG: path_found = {path_found}") + print(f"DEBUG: list_path = {list_path}") + print(f"DEBUG: len(list_path) = {len(list_path)}") + assert path_found == True + assert len(list_path) == 2 + + +def mock_javascript_file_open(*args, **kwargs): + file_path = args[0] if args else kwargs.get('file', '') + mock_file = MagicMock() + mock_file.__enter__ = MagicMock(return_value=mock_file) + mock_file.__exit__ = MagicMock(return_value=None) + if 'ecosystem_data.txt' in str(file_path): + mock_file.read.return_value = "JAVASCRIPT" + return mock_file + + +def mock_javascript_path_exists(self): + path_str = str(self) + exists_files = ['package.json', 'ecosystem_data.txt', 'node_modules'] + if any([path in path_str for path in exists_files]): + return True + return False + + +@pytest.mark.asyncio +@patch('pathlib.Path.exists', mock_javascript_path_exists) +@patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=java_script_dependency_tree_mock_output) +@patch('builtins.open', side_effect=mock_javascript_file_open) +async def test_javascript_basic_reachable(mock_open, mock_npm_ls): + + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + + set_input_for_next_run( + git_repository='https://github.com/test/javascript-basic-call', + git_ref='abc123', + included_extensions=['**/*.js'], + excluded_extensions=['**/*test*.js'] + ) + + mock_documents = [ + # Application function that calls the vulnerable package + Document( + page_content="function processUserInput(userInput) {\n const result = dangerousFunction(userInput);\n return result;\n}", + metadata={'source': 'app.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Vulnerable package function + Document( + page_content="function dangerousFunction(input) {\n console.log('Executing: ' + input);\n return eval(input);\n}", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Simplified code for app.js (for import resolution and showing calls) + Document( + page_content="const { dangerousFunction, safeFunction } = require('vulnerable-pkg');\n\nfunction processUserInput(userInput) {\n const result = dangerousFunction(userInput);\n // ... rest of code\n}\n\nfunction processData(data) {\n return safeFunction(data);\n // ... rest of code\n}", + metadata={'source': 'app.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + # Simplified code for vulnerable-pkg/index.js + Document( + page_content="function dangerousFunction(input) {\n return eval(input);\n}\n\nfunction safeFunction(data) {\n return data.toUpperCase();\n}\n\nmodule.exports = { dangerousFunction, safeFunction };", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + ] + + with patch('exploit_iq_commons.utils.document_embedding.retrieve_from_cache', return_value=(mock_documents, True)): + result = await transitive_code_search_runner_coroutine("vulnerable-pkg,dangerousFunction") + + (path_found, list_path) = result + assert path_found == True + assert len(list_path) >= 2 + + +@pytest.mark.asyncio +@patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=java_script_dependency_tree_mock_output) +@patch('pathlib.Path.exists', mock_javascript_path_exists) +@patch('builtins.open', side_effect=mock_javascript_file_open) +async def test_javascript_unreachable(mock_open, mock_npm_ls): + + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + + set_input_for_next_run( + git_repository='https://github.com/test/javascript-unreachable', + git_ref='def456', + included_extensions=['**/*.js'], + excluded_extensions=['**/*test*.js'] + ) + + mock_documents = [ + # Application function that only calls publicAPI + Document( + page_content="function main() {\n return publicAPI();\n}", + metadata={'source': 'app.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Package's public API that doesn't call vulnerableFunc + Document( + page_content="function publicAPI() {\n return helperFunc();\n}", + metadata={'source': 'node_modules/another-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Helper function + Document( + page_content="function helperFunc() {\n return 'helper result';\n}", + metadata={'source': 'node_modules/another-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Vulnerable function that is NOT called by anyone + Document( + page_content="function vulnerableFunc(cmd) {\n return require('child_process').execSync(cmd).toString();\n}", + metadata={'source': 'node_modules/another-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Simplified code for app.js + Document( + page_content="const { publicAPI } = require('another-pkg');\n// Code for: function main() {...}", + metadata={'source': 'app.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + # Simplified code for another-pkg/index.js + Document( + page_content="function publicAPI() {\n return helperFunc();\n}\n\nfunction helperFunc() {\n return 'helper result';\n}\n\nfunction vulnerableFunc(cmd) {\n return require('child_process').execSync(cmd).toString();\n}\n\nmodule.exports = { publicAPI, helperFunc, vulnerableFunc };", + metadata={'source': 'node_modules/another-pkg/index.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ) + ] + + with patch('exploit_iq_commons.utils.document_embedding.retrieve_from_cache', return_value=(mock_documents, True)): + result = await transitive_code_search_runner_coroutine("another-pkg,vulnerableFunc") + + (path_found, list_path) = result + assert path_found == False, "vulnerableFunc should NOT be reachable from application code" + assert len(list_path) == 1, "Should have only the target function (not reachable)" + + +@pytest.mark.asyncio +@patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=java_script_dependency_tree_mock_output) +@patch('pathlib.Path.exists', mock_javascript_path_exists) +@patch('builtins.open', side_effect=mock_javascript_file_open) +async def test_javascript_multi_file_chain(mock_open, mock_npm_ls): + + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + + set_input_for_next_run( + git_repository='https://github.com/test/javascript-multi-file', + git_ref='ghi789', + included_extensions=['**/*.js'], + excluded_extensions=['**/*test*.js'] + ) + + mock_documents = [ + # App level + Document( + page_content="function appMain() {\n return processRequest();\n}", + metadata={'source': 'app.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Middleware level + Document( + page_content="function processRequest() {\n return vulnerableDeepFunc();\n}", + metadata={'source': 'node_modules/middle-pkg/middleware.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Deep package level + Document( + page_content="function vulnerableDeepFunc() {\n return eval(arguments[0]);\n}", + metadata={'source': 'node_modules/deep-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Simplified code for app.js + Document( + page_content="import { processRequest } from 'middle-pkg';\nfunction appMain() {\n return processRequest();\n // ... rest of code\n}", + metadata={'source': 'app.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + # Simplified code for middleware.js + Document( + page_content="const { vulnerableDeepFunc } = require('deep-pkg');\n\nfunction processRequest() {\n return vulnerableDeepFunc();\n // ... rest of code\n}", + metadata={'source': 'node_modules/middle-pkg/middleware.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + # Simplified code for middle-pkg/index.js (re-exports from middleware) + Document( + page_content="const { processRequest } = require('./middleware');\n\nmodule.exports = { processRequest };", + metadata={'source': 'node_modules/middle-pkg/index.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + # Simplified code for deep-pkg/index.js + Document( + page_content="export function vulnerableDeepFunc() {\n return eval(arguments[0]);\n}\n\nmodule.exports = { vulnerableDeepFunc };", + metadata={'source': 'node_modules/deep-pkg/index.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ) + ] + + + with patch('exploit_iq_commons.utils.document_embedding.retrieve_from_cache', return_value=(mock_documents, True)): + result = await transitive_code_search_runner_coroutine("deep-pkg,vulnerableDeepFunc") + + (path_found, list_path) = result + # Expected call chain: vulnerableDeepFunc -> processRequest -> appMain (3 functions) + assert path_found == True + assert len(list_path) == 3, f"Should have complete call chain of 3 functions, got {len(list_path)}" + + +@pytest.mark.asyncio +@patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=java_script_dependency_tree_mock_output) +@patch('pathlib.Path.exists', mock_javascript_path_exists) +@patch('builtins.open', side_effect=mock_javascript_file_open) +async def test_javascript_class_transitive_call(mock_open, mock_npm_ls): + """ + Test transitive search for JavaScript class methods where: + - A dependency package contains a class with two methods: + - func_a: vulnerable function + - func_b: triggers func_a + - Source code imports the class and calls func_b + + Expected: The vulnerable func_a should be reachable via the call chain: + app.js -> VulnerableClass.func_b -> VulnerableClass.func_a + """ + from langchain_core.documents import Document + + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + + set_input_for_next_run( + git_repository='https://github.com/test/javascript-class-transitive', + git_ref='class123', + included_extensions=['**/*.js'], + excluded_extensions=['**/*test*.js'] + ) + + mock_documents = [ + # func_a: vulnerable function (class method) + Document( + page_content="func_a(input) {\n" + " // This is the vulnerable function\n" + " return eval(input);\n" + "}\n" + "//(class: VulnerableClass)", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # func_b: triggers func_a (class method) + Document( + page_content="func_b(data) {\n" + " // This method triggers the vulnerable func_a\n" + " const processed = data.toString();\n" + " return this.func_a(processed);\n" + "}\n" + "//(class: VulnerableClass)", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Source code function that imports and uses the class + Document( + page_content="function processUserData(userData) {\n" + " const handler = new VulnerableClass();\n" + " return handler.func_b(userData);\n" + "}", + metadata={'source': 'app.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Simplified code for dependency package + Document( + page_content="class VulnerableClass {\n" + " func_a(input) {\n" + " return eval(input);\n" + " }\n" + " \n" + " func_b(data) {\n" + " const processed = data.toString();\n" + " return this.func_a(processed);\n" + " }\n" + "}\n" + "\n" + "module.exports = { VulnerableClass };", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + # Simplified code for app.js + Document( + page_content="const { VulnerableClass } = require('vulnerable-pkg');\n" + "\n" + "function processUserData(userData) {\n" + " const handler = new VulnerableClass();\n" + " return handler.func_b(userData);\n" + "}\n" + "\n" + "module.exports = { processUserData };", + metadata={'source': 'app.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + ] + + with patch('exploit_iq_commons.utils.document_embedding.retrieve_from_cache', return_value=(mock_documents, True)): + result = await transitive_code_search_runner_coroutine("vulnerable-pkg,VulnerableClass.func_a") + + (path_found, list_path) = result + print(f"DEBUG: path_found = {path_found}") + print(f"DEBUG: list_path = {list_path}") + print(f"DEBUG: len(list_path) = {len(list_path)}") + + # Expected call chain: func_a -> func_b -> processUserData (3 functions) + assert path_found == True, "func_a should be reachable through func_b -> processUserData" + assert len(list_path) == 3, f"Should have call chain of 3 functions (func_a -> func_b -> processUserData), got {len(list_path)}" + + +@pytest.mark.asyncio +@patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=java_script_dependency_tree_mock_output) +@patch('pathlib.Path.exists', mock_javascript_path_exists) +@patch('builtins.open', side_effect=mock_javascript_file_open) +async def test_javascript_class_inheritance_transitive_call(mock_open, mock_npm_ls): + """ + Test transitive search for JavaScript class inheritance where: + - A dependency package contains a base class with a vulnerable method: + - VulnerableBase.executeCommand: vulnerable function using eval + - Source code has a child class that extends the base class: + - CustomHandler extends VulnerableBase + - Another source file creates an instance of child class and calls the inherited method + + Expected: The vulnerable executeCommand should be reachable via the call chain: + app.js -> processRequest -> CustomHandler (instance) -> VulnerableBase.executeCommand + """ + from langchain_core.documents import Document + + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + + set_input_for_next_run( + git_repository='https://github.com/test/javascript-inheritance', + git_ref='inherit456', + included_extensions=['**/*.js'], + excluded_extensions=['**/*test*.js'] + ) + + mock_documents = [ + # executeCommand: vulnerable function in base class (dependency) + Document( + page_content="executeCommand(cmd) {\n" + " // Vulnerable: directly evaluates user input\n" + " return eval(cmd);\n" + "}\n" + "//(class: VulnerableBase)", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # CustomHandler: child class that extends VulnerableBase (source code) + # Note: Inheritance is shown in simplified_code, not in functions_classes + # processRequest: function that creates instance and calls inherited method + Document( + page_content="function processRequest(userInput) {\n" + " const handler = new CustomHandler();\n" + " return handler.executeCommand(userInput);\n" + "}", + metadata={'source': 'app.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Simplified code for dependency package (base class) + Document( + page_content="class VulnerableBase {\n" + " executeCommand(cmd) {\n" + " return eval(cmd);\n" + " }\n" + "}\n" + "\n" + "module.exports = { VulnerableBase };", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + # Simplified code for CustomHandler class (child class) + Document( + page_content="const { VulnerableBase } = require('vulnerable-pkg');\n" + "\n" + "class CustomHandler extends VulnerableBase {\n" + " constructor() {\n" + " super();\n" + " }\n" + " \n" + " // Inherits executeCommand from VulnerableBase\n" + "}\n" + "\n" + "module.exports = { CustomHandler };", + metadata={'source': 'src/handlers/CustomHandler.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + # Simplified code for app.js (usage) + Document( + page_content="const { CustomHandler } = require('./handlers/CustomHandler');\n" + "\n" + "function processRequest(userInput) {\n" + " const handler = new CustomHandler();\n" + " return handler.executeCommand(userInput);\n" + "}\n" + "\n" + "module.exports = { processRequest };", + metadata={'source': 'app.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + ] + + with patch('exploit_iq_commons.utils.document_embedding.retrieve_from_cache', return_value=(mock_documents, True)): + result = await transitive_code_search_runner_coroutine("vulnerable-pkg,VulnerableBase.executeCommand") + + (path_found, list_path) = result + print(f"DEBUG: path_found = {path_found}") + print(f"DEBUG: list_path = {list_path}") + print(f"DEBUG: len(list_path) = {len(list_path)}") + + # Expected call chain: executeCommand (from VulnerableBase) -> processRequest (2 functions) + # Note: The child class CustomHandler doesn't redefine the method, so it's inherited + assert path_found == True, "executeCommand should be reachable through CustomHandler instance in processRequest" + assert len(list_path) == 2, f"Should have call chain of 2 functions (executeCommand -> processRequest), got {len(list_path)}" + + +@pytest.mark.asyncio +@patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=java_script_dependency_tree_mock_output) +@patch('pathlib.Path.exists', mock_javascript_path_exists) +@patch('builtins.open', side_effect=mock_javascript_file_open) +async def test_javascript_object_literal_transitive_call(mock_open, mock_npm_ls): + """ + Test transitive search for JavaScript object literal methods where: + - A dependency package contains an object literal with two methods: + - vulnerableMethod: vulnerable function using eval + - wrapperMethod: triggers vulnerableMethod via this.vulnerableMethod + - Source code imports the object and calls wrapperMethod + + Expected: The vulnerable method should be reachable via the call chain: + app.js -> dangerousUtils.wrapperMethod -> dangerousUtils.vulnerableMethod + """ + from langchain_core.documents import Document + + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + + set_input_for_next_run( + git_repository='https://github.com/test/javascript-object-transitive', + git_ref='object123', + included_extensions=['**/*.js'], + excluded_extensions=['**/*test*.js'] + ) + + mock_documents = [ + # vulnerableMethod: vulnerable function (object method with unified class annotation) + Document( + page_content="vulnerableMethod(input) {\n" + " // This is the vulnerable function\n" + " return eval(input);\n" + "}\n" + "//(class: dangerousUtils)", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # wrapperMethod: triggers vulnerableMethod (object method with unified class annotation) + Document( + page_content="wrapperMethod(data) {\n" + " // This method triggers the vulnerable method\n" + " const processed = data.toString();\n" + " return this.vulnerableMethod(processed);\n" + "}\n" + "//(class: dangerousUtils)", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Source code function that imports and uses the object + Document( + page_content="function processUserInput(userData) {\n" + " return dangerousUtils.wrapperMethod(userData);\n" + "}", + metadata={'source': 'app.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Simplified code for dependency package (object literal) + Document( + page_content="const dangerousUtils = {\n" + " vulnerableMethod(input) {\n" + " return eval(input);\n" + " },\n" + " \n" + " wrapperMethod(data) {\n" + " const processed = data.toString();\n" + " return this.vulnerableMethod(processed);\n" + " }\n" + "};\n" + "\n" + "module.exports = { dangerousUtils };", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + # Simplified code for app.js + Document( + page_content="const { dangerousUtils } = require('vulnerable-pkg');\n" + "\n" + "function processUserInput(userData) {\n" + " return dangerousUtils.wrapperMethod(userData);\n" + "}\n" + "\n" + "module.exports = { processUserInput };", + metadata={'source': 'app.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + ] + + with patch('exploit_iq_commons.utils.document_embedding.retrieve_from_cache', return_value=(mock_documents, True)): + result = await transitive_code_search_runner_coroutine("vulnerable-pkg,dangerousUtils.vulnerableMethod") + + (path_found, list_path) = result + print(f"DEBUG: path_found = {path_found}") + print(f"DEBUG: list_path = {list_path}") + print(f"DEBUG: len(list_path) = {len(list_path)}") + + # Expected call chain: vulnerableMethod -> wrapperMethod -> processUserInput (3 functions) + assert path_found == True, "vulnerableMethod should be reachable through wrapperMethod -> processUserInput" + assert len(list_path) == 3, f"Should have call chain of 3 functions (vulnerableMethod -> wrapperMethod -> processUserInput), got {len(list_path)}" + + +@pytest.mark.asyncio +@patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=java_script_dependency_tree_mock_output) +@patch('pathlib.Path.exists', mock_javascript_path_exists) +@patch('builtins.open', side_effect=mock_javascript_file_open) +async def test_javascript_object_method_direct_call(mock_open, mock_npm_ls): + """ + Test transitive search for direct JavaScript object method call where: + - A dependency package contains an object literal with a vulnerable method + - Source code directly calls the vulnerable method + + Expected: The vulnerable method should be reachable directly from source: + app.js -> apiUtils.unsafeEval + """ + from langchain_core.documents import Document + + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + + set_input_for_next_run( + git_repository='https://github.com/test/javascript-object-direct', + git_ref='direct456', + included_extensions=['**/*.js'], + excluded_extensions=['**/*test*.js'] + ) + + mock_documents = [ + # unsafeEval: vulnerable function (object method with unified class annotation) + Document( + page_content="unsafeEval(code) {\n" + " return eval(code);\n" + "}\n" + "//(class: apiUtils)", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Source code function that directly calls the vulnerable method + Document( + page_content="function executeCode(userCode) {\n" + " return apiUtils.unsafeEval(userCode);\n" + "}", + metadata={'source': 'app.js', 'content_type': 'functions_classes', 'language': 'javascript'} + ), + # Simplified code for dependency package (object literal) + Document( + page_content="const apiUtils = {\n" + " unsafeEval(code) {\n" + " return eval(code);\n" + " }\n" + "};\n" + "\n" + "module.exports = { apiUtils };", + metadata={'source': 'node_modules/vulnerable-pkg/index.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + # Simplified code for app.js + Document( + page_content="const { apiUtils } = require('vulnerable-pkg');\n" + "\n" + "function executeCode(userCode) {\n" + " return apiUtils.unsafeEval(userCode);\n" + "}\n" + "\n" + "module.exports = { executeCode };", + metadata={'source': 'app.js', 'content_type': 'simplified_code', 'language': 'javascript'} + ), + ] + + with patch('exploit_iq_commons.utils.document_embedding.retrieve_from_cache', return_value=(mock_documents, True)): + result = await transitive_code_search_runner_coroutine("vulnerable-pkg,apiUtils.unsafeEval") + + (path_found, list_path) = result + print(f"DEBUG: path_found = {path_found}") + print(f"DEBUG: list_path = {list_path}") + print(f"DEBUG: len(list_path) = {len(list_path)}") + + # Expected call chain: unsafeEval -> executeCode (2 functions) + assert path_found == True, "unsafeEval should be reachable directly from executeCode" + assert len(list_path) == 2, f"Should have call chain of 2 functions (unsafeEval -> executeCode), got {len(list_path)}" + + +@pytest.mark.parametrize("raw_query, expected_cleaned", [ + ("'commons-beanutils:commons-beanutils:1.9.4'", "commons-beanutils:commons-beanutils:1.9.4"), + ('"commons-beanutils:commons-beanutils:1.9.4"', "commons-beanutils:commons-beanutils:1.9.4"), + ("‘commons-beanutils:commons-beanutils:1.9.4’", "commons-beanutils:commons-beanutils:1.9.4"), + ("“commons-beanutils:commons-beanutils:1.9.4”", "commons-beanutils:commons-beanutils:1.9.4"), + ("'commons-beanutils:commons-beanutils:1.9.4’", "commons-beanutils:commons-beanutils:1.9.4"), + ("\"commons-beanutils:commons-beanutils:1.9.4”", "commons-beanutils:commons-beanutils:1.9.4"), + ("commons-beanutils:commons-beanutils:1.9.4", "commons-beanutils:commons-beanutils:1.9.4"), + (" 'commons-beanutils:commons-beanutils:1.9.4' ", "commons-beanutils:commons-beanutils:1.9.4"), + ("'commons-beanutils:commons-beanutils:1.9.4'\nPlease wait...", "commons-beanutils:commons-beanutils:1.9.4"), +]) +def test_query_cleaning_strips_unicode_quotes(raw_query, expected_cleaned): + """Test that query cleaning handles both ASCII and Unicode smart quotes.""" + cleaned = raw_query.strip().split("\n")[0].strip().strip("'\"‘’“”").strip() + assert cleaned == expected_cleaned, f"For input {repr(raw_query)}: got {repr(cleaned)}, expected {repr(expected_cleaned)}" + + +python_hyphenated_deptree_output = ( + 'flask==3.1.2 # flask\n' + 'python-dateutil==2.9.0 # python-dateutil\n' + ' six==1.17.0 # six>=1.5\n' +) + + +def mock_file_open_with_dateutil(*args, **kwargs): + file_path = args[0] if args else kwargs.get('file', '') + mock_file = MagicMock() + mock_file.__enter__ = MagicMock(return_value=mock_file) + mock_file.__exit__ = MagicMock(return_value=None) + if 'ecosystem_data.txt' in str(file_path): + mock_file.read.return_value = "PYTHON" + elif 'requirements.txt' in str(file_path): + mock_file.__iter__ = MagicMock(return_value=iter([ + "flask==3.1.2\n", + "python-dateutil==2.9.0\n", + ])) + else: + mock_file.read.return_value = "" + return mock_file + + +@pytest.mark.asyncio +@patch('exploit_iq_commons.utils.dep_tree.run_command', return_value=python_hyphenated_deptree_output) +@patch('builtins.open', side_effect=mock_file_open_with_dateutil) +@patch.object(PythonDependencyTreeBuilder, '_find_site_packages', return_value=Path("/fake/site-packages")) +@patch.object(PythonDependencyTreeBuilder, '_find_module_dirs', side_effect=lambda pkg, sp: {"python-dateutil": ["dateutil"]}.get(pkg, [pkg])) +async def test_transitive_search_python_hyphenated_package(mock_find_module, mock_find_sp, mock_open, mock_run_command): + """Python packages with hyphens (python-dateutil) should be found via import name (dateutil).""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + logging.basicConfig(level=logging.DEBUG) + + set_input_for_next_run( + git_repository='https://github.com/test/test', + git_ref='abc123', + included_extensions=['**/*.py'], + excluded_extensions=['**/*test*.py'] + ) + + mock_documents = [ + python_dateutil_function, + python_dateutil_full_source, + python_app_using_dateutil, + python_app_function_using_dateutil, + ] + + with patch('exploit_iq_commons.utils.document_embedding.retrieve_from_cache', + return_value=(mock_documents, True)): + result = await transitive_code_search_runner_coroutine("python-dateutil,parse") + + (path_found, list_path) = result + assert path_found is True, f"Expected reachable path for python-dateutil,parse but got False. Path: {list_path}" + assert len(list_path) >= 2 + + +# --------------------------------------------------------------------------- +# Go CCA sub-package disambiguation tests +# +# These reproduce false-positive chains where CCA matches a function in +# the wrong Go sub-package (e.g. url.Parse instead of strvals.Parse). +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_go_cca_jwt_parse_not_reachable_in_hypershift(): + """CVE-2025-30204: golang-jwt/jwt ParseWithClaims is NOT reachable + in hypershift — jwt is only used for token signing (NewWithClaims), + never for parsing incoming tokens.""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run( + git_repository="https://github.com/openshift/hypershift", + git_ref="ffe47c7d9ded2a528862acbb8dd0a0f105dd801c", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"], + ) + result = await transitive_code_search_runner_coroutine("github.com/golang-jwt/jwt/v4,ParseWithClaims") + (path_found, list_path) = result + assert path_found is False, ( + f"ParseWithClaims should NOT be reachable in hypershift — " + f"jwt is only used for signing, not parsing. Path: {list_path}" + ) + + +@pytest.mark.asyncio +async def test_go_cca_strvals_parse_not_reachable_in_oc_mirror(): + """CVE-2022-23524: helm.sh/helm/v3/pkg/strvals.Parse is NOT reachable + in oc-mirror. CCA must not follow url.Parse or other same-named + functions from different packages.""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run( + git_repository="https://github.com/openshift/oc-mirror", + git_ref="0a2b069f4859606f9ac85e8d15761fdfb259cf55", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"], + ) + result = await transitive_code_search_runner_coroutine("helm.sh/helm/v3/pkg/strvals,Parse") + (path_found, list_path) = result + assert path_found is False, ( + f"strvals.Parse should NOT be reachable in oc-mirror — " + f"CCA likely followed url.Parse or another same-named function. " + f"Path: {list_path}" + ) + + +@pytest.mark.asyncio +async def test_go_cca_yaml_unmarshal_in_coredns(): + """CVE-2021-4235: gopkg.in/yaml.v2.Unmarshal in coredns. + + coredns imports sigs.k8s.io/yaml which wraps gopkg.in/yaml.v2, but + the CCA dep tree may not connect them. Before the sub-package + disambiguation fix this chain was found via a buggy same-package + shortcut (any "package yaml" file was treated as the callee). + After the fix CCA correctly requires import validation, so the + indirect chain through sigs.k8s.io/yaml may not resolve.""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run( + git_repository="https://github.com/openshift/coredns", + git_ref="e4c9977928b7f36962351b387fafff474b238311", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"], + ) + result = await transitive_code_search_runner_coroutine("gopkg.in/yaml.v2,Unmarshal") + (path_found, list_path) = result + if path_found: + assert len(list_path) > 1 + + +@pytest.mark.asyncio +async def test_go_cca_yaml_unmarshal_not_reachable_in_egress_router(): + """CVE-2022-3064: gopkg.in/yaml.v2.Unmarshal should NOT be reachable + in egress-router-cni — app code never imports yaml.v2, only JSON.""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run( + git_repository="https://github.com/openshift/egress-router-cni", + git_ref="a92e415791b531ca15ec84953550b71bd3534566", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"], + ) + result = await transitive_code_search_runner_coroutine("gopkg.in/yaml.v2,Unmarshal") + (path_found, list_path) = result + if path_found: + last_in_chain = list_path[-1] if list_path else "" + assert not last_in_chain.startswith("vendor/"), ( + f"yaml.Unmarshal chain should reach app code, not stop in vendor/. Path: {list_path}" + ) + + +@pytest.mark.asyncio +async def test_go_cca_quicgo_detectAndRemoveAckedPackets_reachable_in_caddy(): + """CVE-2025-29785: quic-go detectAndRemoveAckedPackets IS reachable + in caddy — caddy uses quic-go for HTTP/3 transport, and the function + is called internally when processing ACK frames from QUIC clients.""" + transitive_code_search_runner_coroutine = await get_transitive_code_runner_function() + set_input_for_next_run( + git_repository="https://github.com/caddyserver/caddy", + git_ref="8dc76676fb5156f971db49c50a5c509d1c93613b", + included_extensions=["**/*.go"], + excluded_extensions=["go.mod", "go.sum", "**/*test*.go"], + ) + result = await transitive_code_search_runner_coroutine("github.com/quic-go/quic-go,detectAndRemoveAckedPackets") + (path_found, list_path) = result + assert path_found is True, ( + f"detectAndRemoveAckedPackets should be reachable in caddy — " + f"caddy uses quic-go for HTTP/3 and the function processes ACK frames. " + f"Path: {list_path}" + ) + assert len(list_path) > 1 diff --git a/src/vuln_analysis/tools/tests/tests_data/err.c b/src/vuln_analysis/tools/tests/tests_data/err.c new file mode 100644 index 000000000..b95182d70 --- /dev/null +++ b/src/vuln_analysis/tools/tests/tests_data/err.c @@ -0,0 +1,898 @@ +/* + * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#define OSSL_FORCE_ERR_STATE + +#include +#include +#include +#include "crypto/cryptlib.h" +#include "internal/err.h" +#include "crypto/err.h" +#include +#include +#include +#include +#include +#include "internal/thread_once.h" +#include "crypto/ctype.h" +#include "internal/constant_time.h" +#include "internal/e_os.h" +#include "err_local.h" + +/* Forward declaration in case it's not published because of configuration */ +ERR_STATE *ERR_get_state(void); + +#ifndef OPENSSL_NO_ERR +static int err_load_strings(const ERR_STRING_DATA *str); +#endif + +#ifndef OPENSSL_NO_ERR +static ERR_STRING_DATA ERR_str_libraries[] = { + {ERR_PACK(ERR_LIB_NONE, 0, 0), "unknown library"}, + {ERR_PACK(ERR_LIB_SYS, 0, 0), "system library"}, + {ERR_PACK(ERR_LIB_BN, 0, 0), "bignum routines"}, + {ERR_PACK(ERR_LIB_RSA, 0, 0), "rsa routines"}, + {ERR_PACK(ERR_LIB_DH, 0, 0), "Diffie-Hellman routines"}, + {ERR_PACK(ERR_LIB_EVP, 0, 0), "digital envelope routines"}, + {ERR_PACK(ERR_LIB_BUF, 0, 0), "memory buffer routines"}, + {ERR_PACK(ERR_LIB_OBJ, 0, 0), "object identifier routines"}, + {ERR_PACK(ERR_LIB_PEM, 0, 0), "PEM routines"}, + {ERR_PACK(ERR_LIB_DSA, 0, 0), "dsa routines"}, + {ERR_PACK(ERR_LIB_X509, 0, 0), "x509 certificate routines"}, + {ERR_PACK(ERR_LIB_ASN1, 0, 0), "asn1 encoding routines"}, + {ERR_PACK(ERR_LIB_CONF, 0, 0), "configuration file routines"}, + {ERR_PACK(ERR_LIB_CRYPTO, 0, 0), "common libcrypto routines"}, + {ERR_PACK(ERR_LIB_EC, 0, 0), "elliptic curve routines"}, + {ERR_PACK(ERR_LIB_ECDSA, 0, 0), "ECDSA routines"}, + {ERR_PACK(ERR_LIB_ECDH, 0, 0), "ECDH routines"}, + {ERR_PACK(ERR_LIB_SSL, 0, 0), "SSL routines"}, + {ERR_PACK(ERR_LIB_BIO, 0, 0), "BIO routines"}, + {ERR_PACK(ERR_LIB_PKCS7, 0, 0), "PKCS7 routines"}, + {ERR_PACK(ERR_LIB_X509V3, 0, 0), "X509 V3 routines"}, + {ERR_PACK(ERR_LIB_PKCS12, 0, 0), "PKCS12 routines"}, + {ERR_PACK(ERR_LIB_RAND, 0, 0), "random number generator"}, + {ERR_PACK(ERR_LIB_DSO, 0, 0), "DSO support routines"}, + {ERR_PACK(ERR_LIB_TS, 0, 0), "time stamp routines"}, + {ERR_PACK(ERR_LIB_ENGINE, 0, 0), "engine routines"}, + {ERR_PACK(ERR_LIB_OCSP, 0, 0), "OCSP routines"}, + {ERR_PACK(ERR_LIB_UI, 0, 0), "UI routines"}, + {ERR_PACK(ERR_LIB_FIPS, 0, 0), "FIPS routines"}, + {ERR_PACK(ERR_LIB_CMS, 0, 0), "CMS routines"}, + {ERR_PACK(ERR_LIB_CRMF, 0, 0), "CRMF routines"}, + {ERR_PACK(ERR_LIB_CMP, 0, 0), "CMP routines"}, + {ERR_PACK(ERR_LIB_HMAC, 0, 0), "HMAC routines"}, + {ERR_PACK(ERR_LIB_CT, 0, 0), "CT routines"}, + {ERR_PACK(ERR_LIB_ASYNC, 0, 0), "ASYNC routines"}, + {ERR_PACK(ERR_LIB_KDF, 0, 0), "KDF routines"}, + {ERR_PACK(ERR_LIB_OSSL_STORE, 0, 0), "STORE routines"}, + {ERR_PACK(ERR_LIB_SM2, 0, 0), "SM2 routines"}, + {ERR_PACK(ERR_LIB_ESS, 0, 0), "ESS routines"}, + {ERR_PACK(ERR_LIB_PROV, 0, 0), "Provider routines"}, + {ERR_PACK(ERR_LIB_OSSL_ENCODER, 0, 0), "ENCODER routines"}, + {ERR_PACK(ERR_LIB_OSSL_DECODER, 0, 0), "DECODER routines"}, + {ERR_PACK(ERR_LIB_HTTP, 0, 0), "HTTP routines"}, + {0, NULL}, +}; + +/* + * Should make sure that all ERR_R_ reasons defined in include/openssl/err.h.in + * are listed. For maintainability, please keep all reasons in the same order. + */ +static ERR_STRING_DATA ERR_str_reasons[] = { + {ERR_R_SYS_LIB, "system lib"}, + {ERR_R_BN_LIB, "BN lib"}, + {ERR_R_RSA_LIB, "RSA lib"}, + {ERR_R_DH_LIB, "DH lib"}, + {ERR_R_EVP_LIB, "EVP lib"}, + {ERR_R_BUF_LIB, "BUF lib"}, + {ERR_R_OBJ_LIB, "OBJ lib"}, + {ERR_R_PEM_LIB, "PEM lib"}, + {ERR_R_DSA_LIB, "DSA lib"}, + {ERR_R_X509_LIB, "X509 lib"}, + {ERR_R_ASN1_LIB, "ASN1 lib"}, + {ERR_R_CRYPTO_LIB, "CRYPTO lib"}, + {ERR_R_EC_LIB, "EC lib"}, + {ERR_R_BIO_LIB, "BIO lib"}, + {ERR_R_PKCS7_LIB, "PKCS7 lib"}, + {ERR_R_X509V3_LIB, "X509V3 lib"}, + {ERR_R_ENGINE_LIB, "ENGINE lib"}, + {ERR_R_UI_LIB, "UI lib"}, + {ERR_R_ECDSA_LIB, "ECDSA lib"}, + {ERR_R_OSSL_STORE_LIB, "OSSL_STORE lib"}, + {ERR_R_OSSL_DECODER_LIB, "OSSL_DECODER lib"}, + + {ERR_R_FATAL, "fatal"}, + {ERR_R_MALLOC_FAILURE, "malloc failure"}, + {ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED, + "called a function you should not call"}, + {ERR_R_PASSED_NULL_PARAMETER, "passed a null parameter"}, + {ERR_R_INTERNAL_ERROR, "internal error"}, + {ERR_R_DISABLED, "called a function that was disabled at compile-time"}, + {ERR_R_INIT_FAIL, "init fail"}, + {ERR_R_PASSED_INVALID_ARGUMENT, "passed invalid argument"}, + {ERR_R_OPERATION_FAIL, "operation fail"}, + {ERR_R_INVALID_PROVIDER_FUNCTIONS, "invalid provider functions"}, + {ERR_R_INTERRUPTED_OR_CANCELLED, "interrupted or cancelled"}, + {ERR_R_NESTED_ASN1_ERROR, "nested asn1 error"}, + {ERR_R_MISSING_ASN1_EOS, "missing asn1 eos"}, + /* + * Something is unsupported, exactly what is expressed with additional data + */ + {ERR_R_UNSUPPORTED, "unsupported"}, + /* + * A fetch failed for other reasons than the name to be fetched being + * unsupported. + */ + {ERR_R_FETCH_FAILED, "fetch failed"}, + {ERR_R_INVALID_PROPERTY_DEFINITION, "invalid property definition"}, + {ERR_R_UNABLE_TO_GET_READ_LOCK, "unable to get read lock"}, + {ERR_R_UNABLE_TO_GET_WRITE_LOCK, "unable to get write lock"}, + {0, NULL}, +}; +#endif + +static CRYPTO_ONCE err_init = CRYPTO_ONCE_STATIC_INIT; +static int set_err_thread_local; +static CRYPTO_THREAD_LOCAL err_thread_local; + +static CRYPTO_ONCE err_string_init = CRYPTO_ONCE_STATIC_INIT; +static CRYPTO_RWLOCK *err_string_lock = NULL; + +#ifndef OPENSSL_NO_ERR +static ERR_STRING_DATA *int_err_get_item(const ERR_STRING_DATA *); +#endif + +/* + * The internal state + */ + +#ifndef OPENSSL_NO_ERR +static LHASH_OF(ERR_STRING_DATA) *int_error_hash = NULL; +#endif +static int int_err_library_number = ERR_LIB_USER; + +typedef enum ERR_GET_ACTION_e { + EV_POP, EV_PEEK, EV_PEEK_LAST +} ERR_GET_ACTION; + +static unsigned long get_error_values(ERR_GET_ACTION g, + const char **file, int *line, + const char **func, const char **data, + int *flags); + +#ifndef OPENSSL_NO_ERR +static unsigned long err_string_data_hash(const ERR_STRING_DATA *a) +{ + unsigned long ret, l; + + l = a->error; + ret = l ^ ERR_GET_LIB(l); + return (ret ^ ret % 19 * 13); +} + +static int err_string_data_cmp(const ERR_STRING_DATA *a, + const ERR_STRING_DATA *b) +{ + if (a->error == b->error) + return 0; + return a->error > b->error ? 1 : -1; +} + +static ERR_STRING_DATA *int_err_get_item(const ERR_STRING_DATA *d) +{ + ERR_STRING_DATA *p = NULL; + + if (!CRYPTO_THREAD_read_lock(err_string_lock)) + return NULL; + p = lh_ERR_STRING_DATA_retrieve(int_error_hash, d); + CRYPTO_THREAD_unlock(err_string_lock); + + return p; +} +#endif + +void OSSL_ERR_STATE_free(ERR_STATE *state) +{ + int i; + + if (state == NULL) + return; + for (i = 0; i < ERR_NUM_ERRORS; i++) { + err_clear(state, i, 1); + } + CRYPTO_free(state, OPENSSL_FILE, OPENSSL_LINE); +} + +DEFINE_RUN_ONCE_STATIC(do_err_strings_init) +{ + if (!OPENSSL_init_crypto(OPENSSL_INIT_BASE_ONLY, NULL)) + return 0; + err_string_lock = CRYPTO_THREAD_lock_new(); + if (err_string_lock == NULL) + return 0; +#ifndef OPENSSL_NO_ERR + int_error_hash = lh_ERR_STRING_DATA_new(err_string_data_hash, + err_string_data_cmp); + if (int_error_hash == NULL) { + CRYPTO_THREAD_lock_free(err_string_lock); + err_string_lock = NULL; + return 0; + } +#endif + return 1; +} + +void err_cleanup(void) +{ + if (set_err_thread_local != 0) + CRYPTO_THREAD_cleanup_local(&err_thread_local); + CRYPTO_THREAD_lock_free(err_string_lock); + err_string_lock = NULL; +#ifndef OPENSSL_NO_ERR + lh_ERR_STRING_DATA_free(int_error_hash); + int_error_hash = NULL; +#endif +} + +#ifndef OPENSSL_NO_ERR +/* + * Legacy; pack in the library. + */ +static void err_patch(int lib, ERR_STRING_DATA *str) +{ + unsigned long plib = ERR_PACK(lib, 0, 0); + + for (; str->error != 0; str++) + str->error |= plib; +} + +/* + * Hash in |str| error strings. Assumes the RUN_ONCE was done. + */ +static int err_load_strings(const ERR_STRING_DATA *str) +{ + if (!CRYPTO_THREAD_write_lock(err_string_lock)) + return 0; + for (; str->error; str++) + (void)lh_ERR_STRING_DATA_insert(int_error_hash, + (ERR_STRING_DATA *)str); + CRYPTO_THREAD_unlock(err_string_lock); + return 1; +} +#endif + +int ossl_err_load_ERR_strings(void) +{ +#ifndef OPENSSL_NO_ERR + if (!RUN_ONCE(&err_string_init, do_err_strings_init)) + return 0; + + err_load_strings(ERR_str_libraries); + err_load_strings(ERR_str_reasons); +#endif + return 1; +} + +int ERR_load_strings(int lib, ERR_STRING_DATA *str) +{ +#ifndef OPENSSL_NO_ERR + if (ossl_err_load_ERR_strings() == 0) + return 0; + + err_patch(lib, str); + err_load_strings(str); +#endif + + return 1; +} + +int ERR_load_strings_const(const ERR_STRING_DATA *str) +{ +#ifndef OPENSSL_NO_ERR + if (ossl_err_load_ERR_strings() == 0) + return 0; + err_load_strings(str); +#endif + + return 1; +} + +int ERR_unload_strings(int lib, ERR_STRING_DATA *str) +{ +#ifndef OPENSSL_NO_ERR + if (!RUN_ONCE(&err_string_init, do_err_strings_init)) + return 0; + + if (!CRYPTO_THREAD_write_lock(err_string_lock)) + return 0; + /* + * We don't need to ERR_PACK the lib, since that was done (to + * the table) when it was loaded. + */ + for (; str->error; str++) + (void)lh_ERR_STRING_DATA_delete(int_error_hash, str); + CRYPTO_THREAD_unlock(err_string_lock); +#endif + + return 1; +} + +void err_free_strings_int(void) +{ + /* obsolete */ +} + +/********************************************************/ + +void ERR_clear_error(void) +{ + int i; + ERR_STATE *es; + + es = ossl_err_get_state_int(); + if (es == NULL) + return; + + for (i = 0; i < ERR_NUM_ERRORS; i++) { + err_clear(es, i, 0); + } + es->top = es->bottom = 0; +} + +unsigned long ERR_get_error(void) +{ + return get_error_values(EV_POP, NULL, NULL, NULL, NULL, NULL); +} + +unsigned long ERR_get_error_all(const char **file, int *line, + const char **func, + const char **data, int *flags) +{ + return get_error_values(EV_POP, file, line, func, data, flags); +} + +#ifndef OPENSSL_NO_DEPRECATED_3_0 +unsigned long ERR_get_error_line(const char **file, int *line) +{ + return get_error_values(EV_POP, file, line, NULL, NULL, NULL); +} + +unsigned long ERR_get_error_line_data(const char **file, int *line, + const char **data, int *flags) +{ + return get_error_values(EV_POP, file, line, NULL, data, flags); +} +#endif + +unsigned long ERR_peek_error(void) +{ + return get_error_values(EV_PEEK, NULL, NULL, NULL, NULL, NULL); +} + +unsigned long ERR_peek_error_line(const char **file, int *line) +{ + return get_error_values(EV_PEEK, file, line, NULL, NULL, NULL); +} + +unsigned long ERR_peek_error_func(const char **func) +{ + return get_error_values(EV_PEEK, NULL, NULL, func, NULL, NULL); +} + +unsigned long ERR_peek_error_data(const char **data, int *flags) +{ + return get_error_values(EV_PEEK, NULL, NULL, NULL, data, flags); +} + +unsigned long ERR_peek_error_all(const char **file, int *line, + const char **func, + const char **data, int *flags) +{ + return get_error_values(EV_PEEK, file, line, func, data, flags); +} + +#ifndef OPENSSL_NO_DEPRECATED_3_0 +unsigned long ERR_peek_error_line_data(const char **file, int *line, + const char **data, int *flags) +{ + return get_error_values(EV_PEEK, file, line, NULL, data, flags); +} +#endif + +unsigned long ERR_peek_last_error(void) +{ + return get_error_values(EV_PEEK_LAST, NULL, NULL, NULL, NULL, NULL); +} + +unsigned long ERR_peek_last_error_line(const char **file, int *line) +{ + return get_error_values(EV_PEEK_LAST, file, line, NULL, NULL, NULL); +} + +unsigned long ERR_peek_last_error_func(const char **func) +{ + return get_error_values(EV_PEEK_LAST, NULL, NULL, func, NULL, NULL); +} + +unsigned long ERR_peek_last_error_data(const char **data, int *flags) +{ + return get_error_values(EV_PEEK_LAST, NULL, NULL, NULL, data, flags); +} + +unsigned long ERR_peek_last_error_all(const char **file, int *line, + const char **func, + const char **data, int *flags) +{ + return get_error_values(EV_PEEK_LAST, file, line, func, data, flags); +} + +#ifndef OPENSSL_NO_DEPRECATED_3_0 +unsigned long ERR_peek_last_error_line_data(const char **file, int *line, + const char **data, int *flags) +{ + return get_error_values(EV_PEEK_LAST, file, line, NULL, data, flags); +} +#endif + +static unsigned long get_error_values(ERR_GET_ACTION g, + const char **file, int *line, + const char **func, + const char **data, int *flags) +{ + int i = 0; + ERR_STATE *es; + unsigned long ret; + + es = ossl_err_get_state_int(); + if (es == NULL) + return 0; + + /* + * Clear anything that should have been cleared earlier. We do this + * here because this doesn't have constant-time issues. + */ + while (es->bottom != es->top) { + if (es->err_flags[es->top] & ERR_FLAG_CLEAR) { + err_clear(es, es->top, 0); + es->top = es->top > 0 ? es->top - 1 : ERR_NUM_ERRORS - 1; + continue; + } + i = (es->bottom + 1) % ERR_NUM_ERRORS; + if (es->err_flags[i] & ERR_FLAG_CLEAR) { + es->bottom = i; + err_clear(es, es->bottom, 0); + continue; + } + break; + } + + /* If everything has been cleared, the stack is empty. */ + if (es->bottom == es->top) + return 0; + + /* Which error, the top of stack (latest one) or the first one? */ + if (g == EV_PEEK_LAST) + i = es->top; + else + i = (es->bottom + 1) % ERR_NUM_ERRORS; + + ret = es->err_buffer[i]; + if (g == EV_POP) { + es->bottom = i; + es->err_buffer[i] = 0; + } + + if (file != NULL) { + *file = es->err_file[i]; + if (*file == NULL) + *file = ""; + } + if (line != NULL) + *line = es->err_line[i]; + if (func != NULL) { + *func = es->err_func[i]; + if (*func == NULL) + *func = ""; + } + if (flags != NULL) + *flags = es->err_data_flags[i]; + if (data == NULL) { + if (g == EV_POP) { + err_clear_data(es, i, 0); + } + } else { + *data = es->err_data[i]; + if (*data == NULL) { + *data = ""; + if (flags != NULL) + *flags = 0; + } + } + return ret; +} + +void ossl_err_string_int(unsigned long e, const char *func, + char *buf, size_t len) +{ + char lsbuf[64], rsbuf[256]; + const char *ls, *rs = NULL; + unsigned long l, r; + + if (len == 0) + return; + + l = ERR_GET_LIB(e); + ls = ERR_lib_error_string(e); + if (ls == NULL) { + BIO_snprintf(lsbuf, sizeof(lsbuf), "lib(%lu)", l); + ls = lsbuf; + } + + /* + * ERR_reason_error_string() can't safely return system error strings, + * since it would call openssl_strerror_r(), which needs a buffer for + * thread safety. So for system errors, we call openssl_strerror_r() + * directly instead. + */ + r = ERR_GET_REASON(e); +#ifndef OPENSSL_NO_ERR + if (ERR_SYSTEM_ERROR(e)) { + if (openssl_strerror_r(r, rsbuf, sizeof(rsbuf))) + rs = rsbuf; + } else { + rs = ERR_reason_error_string(e); + } +#endif + if (rs == NULL) { + BIO_snprintf(rsbuf, sizeof(rsbuf), "reason(%lu)", + r & ~(ERR_RFLAGS_MASK << ERR_RFLAGS_OFFSET)); + rs = rsbuf; + } + + BIO_snprintf(buf, len, "error:%08lX:%s:%s:%s", e, ls, func, rs); + if (strlen(buf) == len - 1) { + /* Didn't fit; use a minimal format. */ + BIO_snprintf(buf, len, "err:%lx:%lx:%lx:%lx", e, l, 0L, r); + } +} + + +void ERR_error_string_n(unsigned long e, char *buf, size_t len) +{ + ossl_err_string_int(e, "", buf, len); +} + +/* + * ERR_error_string_n should be used instead for ret != NULL as + * ERR_error_string cannot know how large the buffer is + */ +char *ERR_error_string(unsigned long e, char *ret) +{ + static char buf[256]; + + if (ret == NULL) + ret = buf; + ERR_error_string_n(e, ret, (int)sizeof(buf)); + return ret; +} + +const char *ERR_lib_error_string(unsigned long e) +{ +#ifndef OPENSSL_NO_ERR + ERR_STRING_DATA d, *p; + unsigned long l; + + if (!RUN_ONCE(&err_string_init, do_err_strings_init)) { + return NULL; + } + + l = ERR_GET_LIB(e); + d.error = ERR_PACK(l, 0, 0); + p = int_err_get_item(&d); + return ((p == NULL) ? NULL : p->string); +#else + return NULL; +#endif +} + +#ifndef OPENSSL_NO_DEPRECATED_3_0 +const char *ERR_func_error_string(unsigned long e) +{ + return NULL; +} +#endif + +const char *ERR_reason_error_string(unsigned long e) +{ +#ifndef OPENSSL_NO_ERR + ERR_STRING_DATA d, *p = NULL; + unsigned long l, r; + + if (!RUN_ONCE(&err_string_init, do_err_strings_init)) { + return NULL; + } + + /* + * ERR_reason_error_string() can't safely return system error strings, + * since openssl_strerror_r() needs a buffer for thread safety, and we + * haven't got one that would serve any sensible purpose. + */ + if (ERR_SYSTEM_ERROR(e)) + return NULL; + + l = ERR_GET_LIB(e); + r = ERR_GET_REASON(e); + d.error = ERR_PACK(l, 0, r); + p = int_err_get_item(&d); + if (p == NULL) { + d.error = ERR_PACK(0, 0, r); + p = int_err_get_item(&d); + } + return ((p == NULL) ? NULL : p->string); +#else + return NULL; +#endif +} + +static void err_delete_thread_state(void *unused) +{ + ERR_STATE *state = CRYPTO_THREAD_get_local(&err_thread_local); + if (state == NULL) + return; + + CRYPTO_THREAD_set_local(&err_thread_local, NULL); + OSSL_ERR_STATE_free(state); +} + +#ifndef OPENSSL_NO_DEPRECATED_1_1_0 +void ERR_remove_thread_state(void *dummy) +{ +} +#endif + +#ifndef OPENSSL_NO_DEPRECATED_1_0_0 +void ERR_remove_state(unsigned long pid) +{ +} +#endif + +DEFINE_RUN_ONCE_STATIC(err_do_init) +{ + set_err_thread_local = 1; + return CRYPTO_THREAD_init_local(&err_thread_local, NULL); +} + +ERR_STATE *ossl_err_get_state_int(void) +{ + ERR_STATE *state; + int saveerrno = get_last_sys_error(); + + if (!OPENSSL_init_crypto(OPENSSL_INIT_BASE_ONLY, NULL)) + return NULL; + + if (!RUN_ONCE(&err_init, err_do_init)) + return NULL; + + state = CRYPTO_THREAD_get_local(&err_thread_local); + if (state == (ERR_STATE*)-1) + return NULL; + + if (state == NULL) { + if (!CRYPTO_THREAD_set_local(&err_thread_local, (ERR_STATE*)-1)) + return NULL; + + state = OSSL_ERR_STATE_new(); + if (state == NULL) { + CRYPTO_THREAD_set_local(&err_thread_local, NULL); + return NULL; + } + + if (!ossl_init_thread_start(NULL, NULL, err_delete_thread_state) + || !CRYPTO_THREAD_set_local(&err_thread_local, state)) { + OSSL_ERR_STATE_free(state); + CRYPTO_THREAD_set_local(&err_thread_local, NULL); + return NULL; + } + + /* Ignore failures from these */ + OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CRYPTO_STRINGS, NULL); + } + + set_sys_error(saveerrno); + return state; +} + +#ifndef OPENSSL_NO_DEPRECATED_3_0 +ERR_STATE *ERR_get_state(void) +{ + return ossl_err_get_state_int(); +} +#endif + + +/* + * err_shelve_state returns the current thread local error state + * and freezes the error module until err_unshelve_state is called. + */ +int err_shelve_state(void **state) +{ + int saveerrno = get_last_sys_error(); + + /* + * Note, at present our only caller is OPENSSL_init_crypto(), indirectly + * via ossl_init_load_crypto_nodelete(), by which point the requested + * "base" initialization has already been performed, so the below call is a + * NOOP, that re-enters OPENSSL_init_crypto() only to quickly return. + * + * If are no other valid callers of this function, the call below can be + * removed, avoiding the re-entry into OPENSSL_init_crypto(). If there are + * potential uses that are not from inside OPENSSL_init_crypto(), then this + * call is needed, but some care is required to make sure that the re-entry + * remains a NOOP. + */ + if (!OPENSSL_init_crypto(OPENSSL_INIT_BASE_ONLY, NULL)) + return 0; + + if (!RUN_ONCE(&err_init, err_do_init)) + return 0; + + *state = CRYPTO_THREAD_get_local(&err_thread_local); + if (!CRYPTO_THREAD_set_local(&err_thread_local, (ERR_STATE*)-1)) + return 0; + + set_sys_error(saveerrno); + return 1; +} + +/* + * err_unshelve_state restores the error state that was returned + * by err_shelve_state previously. + */ +void err_unshelve_state(void* state) +{ + if (state != (void*)-1) + CRYPTO_THREAD_set_local(&err_thread_local, (ERR_STATE*)state); +} + +int ERR_get_next_error_library(void) +{ + int ret; + + if (!RUN_ONCE(&err_string_init, do_err_strings_init)) + return 0; + + if (!CRYPTO_THREAD_write_lock(err_string_lock)) + return 0; + ret = int_err_library_number++; + CRYPTO_THREAD_unlock(err_string_lock); + return ret; +} + +static int err_set_error_data_int(char *data, size_t size, int flags, + int deallocate) +{ + ERR_STATE *es; + + es = ossl_err_get_state_int(); + if (es == NULL) + return 0; + + err_clear_data(es, es->top, deallocate); + err_set_data(es, es->top, data, size, flags); + + return 1; +} + +void ERR_set_error_data(char *data, int flags) +{ + /* + * This function is void so we cannot propagate the error return. Since it + * is also in the public API we can't change the return type. + * + * We estimate the size of the data. If it's not flagged as allocated, + * then this is safe, and if it is flagged as allocated, then our size + * may be smaller than the actual allocation, but that doesn't matter + * too much, the buffer will remain untouched or will eventually be + * reallocated to a new size. + * + * callers should be advised that this function takes over ownership of + * the allocated memory, i.e. they can't count on the pointer to remain + * valid. + */ + err_set_error_data_int(data, strlen(data) + 1, flags, 1); +} + +void ERR_add_error_data(int num, ...) +{ + va_list args; + va_start(args, num); + ERR_add_error_vdata(num, args); + va_end(args); +} + +void ERR_add_error_vdata(int num, va_list args) +{ + int i, len, size; + int flags = ERR_TXT_MALLOCED | ERR_TXT_STRING; + char *str, *arg; + ERR_STATE *es; + + /* Get the current error data; if an allocated string get it. */ + es = ossl_err_get_state_int(); + if (es == NULL) + return; + i = es->top; + + /* + * If err_data is allocated already, reuse the space. + * Otherwise, allocate a small new buffer. + */ + if ((es->err_data_flags[i] & flags) == flags + && ossl_assert(es->err_data[i] != NULL)) { + str = es->err_data[i]; + size = es->err_data_size[i]; + + /* + * To protect the string we just grabbed from tampering by other + * functions we may call, or to protect them from freeing a pointer + * that may no longer be valid at that point, we clear away the + * data pointer and the flags. We will set them again at the end + * of this function. + */ + es->err_data[i] = NULL; + es->err_data_flags[i] = 0; + } else if ((str = OPENSSL_malloc(size = 81)) == NULL) { + return; + } else { + str[0] = '\0'; + } + len = strlen(str); + + while (--num >= 0) { + arg = va_arg(args, char *); + if (arg == NULL) + arg = ""; + len += strlen(arg); + if (len >= size) { + char *p; + + size = len + 20; + p = OPENSSL_realloc(str, size); + if (p == NULL) { + OPENSSL_free(str); + return; + } + str = p; + } + OPENSSL_strlcat(str, arg, (size_t)size); + } + if (!err_set_error_data_int(str, size, flags, 0)) + OPENSSL_free(str); +} + +void err_clear_last_constant_time(int clear) +{ + ERR_STATE *es; + int top; + + es = ossl_err_get_state_int(); + if (es == NULL) + return; + + top = es->top; + + /* + * Flag error as cleared but remove it elsewhere to avoid two errors + * accessing the same error stack location, revealing timing information. + */ + clear = constant_time_select_int(constant_time_eq_int(clear, 0), + 0, ERR_FLAG_CLEAR); + es->err_flags[top] |= clear; +} diff --git a/src/vuln_analysis/tools/tests/tests_data/ess_lib.c b/src/vuln_analysis/tools/tests/tests_data/ess_lib.c new file mode 100644 index 000000000..ff174470d --- /dev/null +++ b/src/vuln_analysis/tools/tests/tests_data/ess_lib.c @@ -0,0 +1,368 @@ +/* + * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include +#include +#include +#include +#include "internal/sizes.h" +#include "crypto/ess.h" +#include "crypto/x509.h" + +static ESS_CERT_ID *ESS_CERT_ID_new_init(const X509 *cert, + int set_issuer_serial); +static ESS_CERT_ID_V2 *ESS_CERT_ID_V2_new_init(const EVP_MD *hash_alg, + const X509 *cert, + int set_issuer_serial); + +ESS_SIGNING_CERT *OSSL_ESS_signing_cert_new_init(const X509 *signcert, + const STACK_OF(X509) *certs, + int set_issuer_serial) +{ + ESS_CERT_ID *cid = NULL; + ESS_SIGNING_CERT *sc; + int i; + + if ((sc = ESS_SIGNING_CERT_new()) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + goto err; + } + if (sc->cert_ids == NULL + && (sc->cert_ids = sk_ESS_CERT_ID_new_null()) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_CRYPTO_LIB); + goto err; + } + + if ((cid = ESS_CERT_ID_new_init(signcert, set_issuer_serial)) == NULL + || !sk_ESS_CERT_ID_push(sc->cert_ids, cid)) { + ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + goto err; + } + for (i = 0; i < sk_X509_num(certs); ++i) { + X509 *cert = sk_X509_value(certs, i); + + if ((cid = ESS_CERT_ID_new_init(cert, 1)) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + goto err; + } + if (!sk_ESS_CERT_ID_push(sc->cert_ids, cid)) { + ERR_raise(ERR_LIB_ESS, ERR_R_CRYPTO_LIB); + goto err; + } + } + + return sc; + err: + ESS_SIGNING_CERT_free(sc); + ESS_CERT_ID_free(cid); + return NULL; +} + +static ESS_CERT_ID *ESS_CERT_ID_new_init(const X509 *cert, + int set_issuer_serial) +{ + ESS_CERT_ID *cid = NULL; + GENERAL_NAME *name = NULL; + unsigned char cert_sha1[SHA_DIGEST_LENGTH]; + + if ((cid = ESS_CERT_ID_new()) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + goto err; + } + if (!X509_digest(cert, EVP_sha1(), cert_sha1, NULL)) { + ERR_raise(ERR_LIB_ESS, ERR_R_X509_LIB); + goto err; + } + if (!ASN1_OCTET_STRING_set(cid->hash, cert_sha1, SHA_DIGEST_LENGTH)) { + ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + goto err; + } + + /* Setting the issuer/serial if requested. */ + if (!set_issuer_serial) + return cid; + + if (cid->issuer_serial == NULL + && (cid->issuer_serial = ESS_ISSUER_SERIAL_new()) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + goto err; + } + if ((name = GENERAL_NAME_new()) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + goto err; + } + name->type = GEN_DIRNAME; + if ((name->d.dirn = X509_NAME_dup(X509_get_issuer_name(cert))) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_X509_LIB); + goto err; + } + if (!sk_GENERAL_NAME_push(cid->issuer_serial->issuer, name)) { + ERR_raise(ERR_LIB_ESS, ERR_R_CRYPTO_LIB); + goto err; + } + name = NULL; /* Ownership is lost. */ + ASN1_INTEGER_free(cid->issuer_serial->serial); + if ((cid->issuer_serial->serial + = ASN1_INTEGER_dup(X509_get0_serialNumber(cert))) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + goto err; + } + + return cid; + err: + GENERAL_NAME_free(name); + ESS_CERT_ID_free(cid); + return NULL; +} + +ESS_SIGNING_CERT_V2 *OSSL_ESS_signing_cert_v2_new_init(const EVP_MD *hash_alg, + const X509 *signcert, + const + STACK_OF(X509) *certs, + int set_issuer_serial) +{ + ESS_CERT_ID_V2 *cid = NULL; + ESS_SIGNING_CERT_V2 *sc; + int i; + + if ((sc = ESS_SIGNING_CERT_V2_new()) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + goto err; + } + cid = ESS_CERT_ID_V2_new_init(hash_alg, signcert, set_issuer_serial); + if (cid == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + goto err; + } + if (!sk_ESS_CERT_ID_V2_push(sc->cert_ids, cid)) { + ERR_raise(ERR_LIB_ESS, ERR_R_CRYPTO_LIB); + goto err; + } + cid = NULL; + + for (i = 0; i < sk_X509_num(certs); ++i) { + X509 *cert = sk_X509_value(certs, i); + + if ((cid = ESS_CERT_ID_V2_new_init(hash_alg, cert, 1)) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + goto err; + } + if (!sk_ESS_CERT_ID_V2_push(sc->cert_ids, cid)) { + ERR_raise(ERR_LIB_ESS, ERR_R_CRYPTO_LIB); + goto err; + } + cid = NULL; + } + + return sc; + err: + ESS_SIGNING_CERT_V2_free(sc); + ESS_CERT_ID_V2_free(cid); + return NULL; +} + +static ESS_CERT_ID_V2 *ESS_CERT_ID_V2_new_init(const EVP_MD *hash_alg, + const X509 *cert, + int set_issuer_serial) +{ + ESS_CERT_ID_V2 *cid; + GENERAL_NAME *name = NULL; + unsigned char hash[EVP_MAX_MD_SIZE]; + unsigned int hash_len = sizeof(hash); + X509_ALGOR *alg = NULL; + + memset(hash, 0, sizeof(hash)); + + if ((cid = ESS_CERT_ID_V2_new()) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + goto err; + } + + if (!EVP_MD_is_a(hash_alg, SN_sha256)) { + alg = X509_ALGOR_new(); + if (alg == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + goto err; + } + X509_ALGOR_set_md(alg, hash_alg); + if (alg->algorithm == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + goto err; + } + cid->hash_alg = alg; + alg = NULL; + } else { + cid->hash_alg = NULL; + } + + if (!X509_digest(cert, hash_alg, hash, &hash_len)) { + ERR_raise(ERR_LIB_ESS, ERR_R_X509_LIB); + goto err; + } + + if (!ASN1_OCTET_STRING_set(cid->hash, hash, hash_len)) { + ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + goto err; + } + + if (!set_issuer_serial) + return cid; + + if ((cid->issuer_serial = ESS_ISSUER_SERIAL_new()) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + goto err; + } + if ((name = GENERAL_NAME_new()) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + goto err; + } + name->type = GEN_DIRNAME; + if ((name->d.dirn = X509_NAME_dup(X509_get_issuer_name(cert))) == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + goto err; + } + if (!sk_GENERAL_NAME_push(cid->issuer_serial->issuer, name)) { + ERR_raise(ERR_LIB_ESS, ERR_R_CRYPTO_LIB); + goto err; + } + name = NULL; /* Ownership is lost. */ + ASN1_INTEGER_free(cid->issuer_serial->serial); + cid->issuer_serial->serial = ASN1_INTEGER_dup(X509_get0_serialNumber(cert)); + if (cid->issuer_serial->serial == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + goto err; + } + + return cid; + err: + X509_ALGOR_free(alg); + GENERAL_NAME_free(name); + ESS_CERT_ID_V2_free(cid); + return NULL; +} + +static int ess_issuer_serial_cmp(const ESS_ISSUER_SERIAL *is, const X509 *cert) +{ + GENERAL_NAME *issuer; + + if (is == NULL || cert == NULL || sk_GENERAL_NAME_num(is->issuer) != 1) + return -1; + + issuer = sk_GENERAL_NAME_value(is->issuer, 0); + if (issuer->type != GEN_DIRNAME + || X509_NAME_cmp(issuer->d.dirn, X509_get_issuer_name(cert)) != 0) + return -1; + + return ASN1_INTEGER_cmp(is->serial, X509_get0_serialNumber(cert)); +} + +/* + * Find the cert in |certs| referenced by |cid| if not NULL, else by |cid_v2|. + * The cert must be the first one in |certs| if and only if |index| is 0. + * Return 0 on not found, -1 on error, else 1 + the position in |certs|. + */ +static int find(const ESS_CERT_ID *cid, const ESS_CERT_ID_V2 *cid_v2, + int index, const STACK_OF(X509) *certs) +{ + const X509 *cert; + EVP_MD *md = NULL; + char name[OSSL_MAX_NAME_SIZE]; + unsigned char cert_digest[EVP_MAX_MD_SIZE]; + unsigned int len, cid_hash_len; + const ESS_ISSUER_SERIAL *is; + int i; + int ret = -1; + + if (cid == NULL && cid_v2 == NULL) { + ERR_raise(ERR_LIB_ESS, ERR_R_PASSED_INVALID_ARGUMENT); + return -1; + } + + if (cid != NULL) + strcpy(name, "SHA1"); + else if (cid_v2->hash_alg == NULL) + strcpy(name, "SHA256"); + else + OBJ_obj2txt(name, sizeof(name), cid_v2->hash_alg->algorithm, 0); + + (void)ERR_set_mark(); + md = EVP_MD_fetch(NULL, name, NULL); + + if (md == NULL) + md = (EVP_MD *)EVP_get_digestbyname(name); + + if (md == NULL) { + (void)ERR_clear_last_mark(); + ERR_raise(ERR_LIB_ESS, ESS_R_ESS_DIGEST_ALG_UNKNOWN); + goto end; + } + (void)ERR_pop_to_mark(); + + for (i = 0; i < sk_X509_num(certs); ++i) { + cert = sk_X509_value(certs, i); + + cid_hash_len = cid != NULL ? cid->hash->length : cid_v2->hash->length; + if (!X509_digest(cert, md, cert_digest, &len) + || cid_hash_len != len) { + ERR_raise(ERR_LIB_ESS, ESS_R_ESS_CERT_DIGEST_ERROR); + goto end; + } + + if (memcmp(cid != NULL ? cid->hash->data : cid_v2->hash->data, + cert_digest, len) == 0) { + is = cid != NULL ? cid->issuer_serial : cid_v2->issuer_serial; + /* Well, it's not really required to match the serial numbers. */ + if (is == NULL || ess_issuer_serial_cmp(is, cert) == 0) { + if ((i == 0) == (index == 0)) { + ret = i + 1; + goto end; + } + ERR_raise(ERR_LIB_ESS, ESS_R_ESS_CERT_ID_WRONG_ORDER); + goto end; + } + } + } + + ret = 0; + ERR_raise(ERR_LIB_ESS, ESS_R_ESS_CERT_ID_NOT_FOUND); +end: + EVP_MD_free(md); + return ret; +} + +int OSSL_ESS_check_signing_certs(const ESS_SIGNING_CERT *ss, + const ESS_SIGNING_CERT_V2 *ssv2, + const STACK_OF(X509) *chain, + int require_signing_cert) +{ + int n_v1 = ss == NULL ? -1 : sk_ESS_CERT_ID_num(ss->cert_ids); + int n_v2 = ssv2 == NULL ? -1 : sk_ESS_CERT_ID_V2_num(ssv2->cert_ids); + int i, ret; + + if (require_signing_cert && ss == NULL && ssv2 == NULL) { + ERR_raise(ERR_LIB_ESS, ESS_R_MISSING_SIGNING_CERTIFICATE_ATTRIBUTE); + return -1; + } + if (n_v1 == 0 || n_v2 == 0) { + ERR_raise(ERR_LIB_ESS, ESS_R_EMPTY_ESS_CERT_ID_LIST); + return -1; + } + /* If both ss and ssv2 exist, as required evaluate them independently. */ + for (i = 0; i < n_v1; i++) { + ret = find(sk_ESS_CERT_ID_value(ss->cert_ids, i), NULL, i, chain); + if (ret <= 0) + return ret; + } + for (i = 0; i < n_v2; i++) { + ret = find(NULL, sk_ESS_CERT_ID_V2_value(ssv2->cert_ids, i), i, chain); + if (ret <= 0) + return ret; + } + return 1; +} diff --git a/src/vuln_analysis/tools/tests/tests_data/jsonpath.c b/src/vuln_analysis/tools/tests/tests_data/jsonpath.c new file mode 100644 index 000000000..db06e6f29 --- /dev/null +++ b/src/vuln_analysis/tools/tests/tests_data/jsonpath.c @@ -0,0 +1,1076 @@ +/*------------------------------------------------------------------------- + * + * jsonpath.c + * Input/output and supporting routines for jsonpath + * + * jsonpath expression is a chain of path items. First path item is $, $var, + * literal or arithmetic expression. Subsequent path items are accessors + * (.key, .*, [subscripts], [*]), filters (? (predicate)) and methods (.type(), + * .size() etc). + * + * For instance, structure of path items for simple expression: + * + * $.a[*].type() + * + * is pretty evident: + * + * $ => .a => [*] => .type() + * + * Some path items such as arithmetic operations, predicates or array + * subscripts may comprise subtrees. For instance, more complex expression + * + * ($.a + $[1 to 5, 7] ? (@ > 3).double()).type() + * + * have following structure of path items: + * + * + => .type() + * ___/ \___ + * / \ + * $ => .a $ => [] => ? => .double() + * _||_ | + * / \ > + * to to / \ + * / \ / @ 3 + * 1 5 7 + * + * Binary encoding of jsonpath constitutes a sequence of 4-bytes aligned + * variable-length path items connected by links. Every item has a header + * consisting of item type (enum JsonPathItemType) and offset of next item + * (zero means no next item). After the header, item may have payload + * depending on item type. For instance, payload of '.key' accessor item is + * length of key name and key name itself. Payload of '>' arithmetic operator + * item is offsets of right and left operands. + * + * So, binary representation of sample expression above is: + * (bottom arrows are next links, top lines are argument links) + * + * _____ + * _____ ___/____ \ __ + * _ /_ \ _____/__/____ \ \ __ _ /_ \ + * / / \ \ / / / \ \ \ / \ / / \ \ + * +(LR) $ .a $ [](* to *, * to *) 1 5 7 ?(A) >(LR) @ 3 .double() .type() + * | | ^ | ^| ^| ^ ^ + * | |__| |__||________________________||___________________| | + * |_______________________________________________________________________| + * + * Copyright (c) 2019-2020, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/utils/adt/jsonpath.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "funcapi.h" +#include "lib/stringinfo.h" +#include "libpq/pqformat.h" +#include "miscadmin.h" +#include "utils/builtins.h" +#include "utils/json.h" +#include "utils/jsonpath.h" + + +static Datum jsonPathFromCstring(char *in, int len); +static char *jsonPathToCstring(StringInfo out, JsonPath *in, + int estimated_len); +static int flattenJsonPathParseItem(StringInfo buf, JsonPathParseItem *item, + int nestingLevel, bool insideArraySubscript); +static void alignStringInfoInt(StringInfo buf); +static int32 reserveSpaceForItemPointer(StringInfo buf); +static void printJsonPathItem(StringInfo buf, JsonPathItem *v, bool inKey, + bool printBracketes); +static int operationPriority(JsonPathItemType op); + + +/**************************** INPUT/OUTPUT ********************************/ + +/* + * jsonpath type input function + */ +Datum +jsonpath_in(PG_FUNCTION_ARGS) +{ + char *in = PG_GETARG_CSTRING(0); + int len = strlen(in); + + return jsonPathFromCstring(in, len); +} + +/* + * jsonpath type recv function + * + * The type is sent as text in binary mode, so this is almost the same + * as the input function, but it's prefixed with a version number so we + * can change the binary format sent in future if necessary. For now, + * only version 1 is supported. + */ +Datum +jsonpath_recv(PG_FUNCTION_ARGS) +{ + StringInfo buf = (StringInfo) PG_GETARG_POINTER(0); + int version = pq_getmsgint(buf, 1); + char *str; + int nbytes; + + if (version == JSONPATH_VERSION) + str = pq_getmsgtext(buf, buf->len - buf->cursor, &nbytes); + else + elog(ERROR, "unsupported jsonpath version number: %d", version); + + return jsonPathFromCstring(str, nbytes); +} + +/* + * jsonpath type output function + */ +Datum +jsonpath_out(PG_FUNCTION_ARGS) +{ + JsonPath *in = PG_GETARG_JSONPATH_P(0); + + PG_RETURN_CSTRING(jsonPathToCstring(NULL, in, VARSIZE(in))); +} + +/* + * jsonpath type send function + * + * Just send jsonpath as a version number, then a string of text + */ +Datum +jsonpath_send(PG_FUNCTION_ARGS) +{ + JsonPath *in = PG_GETARG_JSONPATH_P(0); + StringInfoData buf; + StringInfoData jtext; + int version = JSONPATH_VERSION; + + initStringInfo(&jtext); + (void) jsonPathToCstring(&jtext, in, VARSIZE(in)); + + pq_begintypsend(&buf); + pq_sendint8(&buf, version); + pq_sendtext(&buf, jtext.data, jtext.len); + pfree(jtext.data); + + PG_RETURN_BYTEA_P(pq_endtypsend(&buf)); +} + +/* + * Converts C-string to a jsonpath value. + * + * Uses jsonpath parser to turn string into an AST, then + * flattenJsonPathParseItem() does second pass turning AST into binary + * representation of jsonpath. + */ +static Datum +jsonPathFromCstring(char *in, int len) +{ + JsonPathParseResult *jsonpath = parsejsonpath(in, len); + JsonPath *res; + StringInfoData buf; + + initStringInfo(&buf); + enlargeStringInfo(&buf, 4 * len /* estimation */ ); + + appendStringInfoSpaces(&buf, JSONPATH_HDRSZ); + + if (!jsonpath) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for type %s: \"%s\"", "jsonpath", + in))); + + flattenJsonPathParseItem(&buf, jsonpath->expr, 0, false); + + res = (JsonPath *) buf.data; + SET_VARSIZE(res, buf.len); + res->header = JSONPATH_VERSION; + if (jsonpath->lax) + res->header |= JSONPATH_LAX; + + PG_RETURN_JSONPATH_P(res); +} + +/* + * Converts jsonpath value to a C-string. + * + * If 'out' argument is non-null, the resulting C-string is stored inside the + * StringBuffer. The resulting string is always returned. + */ +static char * +jsonPathToCstring(StringInfo out, JsonPath *in, int estimated_len) +{ + StringInfoData buf; + JsonPathItem v; + + if (!out) + { + out = &buf; + initStringInfo(out); + } + enlargeStringInfo(out, estimated_len); + + if (!(in->header & JSONPATH_LAX)) + appendBinaryStringInfo(out, "strict ", 7); + + jspInit(&v, in); + printJsonPathItem(out, &v, false, true); + + return out->data; +} + +/* + * Recursive function converting given jsonpath parse item and all its + * children into a binary representation. + */ +static int +flattenJsonPathParseItem(StringInfo buf, JsonPathParseItem *item, + int nestingLevel, bool insideArraySubscript) +{ + /* position from beginning of jsonpath data */ + int32 pos = buf->len - JSONPATH_HDRSZ; + int32 chld; + int32 next; + int argNestingLevel = 0; + + check_stack_depth(); + CHECK_FOR_INTERRUPTS(); + + appendStringInfoChar(buf, (char) (item->type)); + + /* + * We align buffer to int32 because a series of int32 values often goes + * after the header, and we want to read them directly by dereferencing + * int32 pointer (see jspInitByBuffer()). + */ + alignStringInfoInt(buf); + + /* + * Reserve space for next item pointer. Actual value will be recorded + * later, after next and children items processing. + */ + next = reserveSpaceForItemPointer(buf); + + switch (item->type) + { + case jpiString: + case jpiVariable: + case jpiKey: + appendBinaryStringInfo(buf, (char *) &item->value.string.len, + sizeof(item->value.string.len)); + appendBinaryStringInfo(buf, item->value.string.val, + item->value.string.len); + appendStringInfoChar(buf, '\0'); + break; + case jpiNumeric: + appendBinaryStringInfo(buf, (char *) item->value.numeric, + VARSIZE(item->value.numeric)); + break; + case jpiBool: + appendBinaryStringInfo(buf, (char *) &item->value.boolean, + sizeof(item->value.boolean)); + break; + case jpiAnd: + case jpiOr: + case jpiEqual: + case jpiNotEqual: + case jpiLess: + case jpiGreater: + case jpiLessOrEqual: + case jpiGreaterOrEqual: + case jpiAdd: + case jpiSub: + case jpiMul: + case jpiDiv: + case jpiMod: + case jpiStartsWith: + { + /* + * First, reserve place for left/right arg's positions, then + * record both args and sets actual position in reserved + * places. + */ + int32 left = reserveSpaceForItemPointer(buf); + int32 right = reserveSpaceForItemPointer(buf); + + chld = !item->value.args.left ? pos : + flattenJsonPathParseItem(buf, item->value.args.left, + nestingLevel + argNestingLevel, + insideArraySubscript); + *(int32 *) (buf->data + left) = chld - pos; + + chld = !item->value.args.right ? pos : + flattenJsonPathParseItem(buf, item->value.args.right, + nestingLevel + argNestingLevel, + insideArraySubscript); + *(int32 *) (buf->data + right) = chld - pos; + } + break; + case jpiLikeRegex: + { + int32 offs; + + appendBinaryStringInfo(buf, + (char *) &item->value.like_regex.flags, + sizeof(item->value.like_regex.flags)); + offs = reserveSpaceForItemPointer(buf); + appendBinaryStringInfo(buf, + (char *) &item->value.like_regex.patternlen, + sizeof(item->value.like_regex.patternlen)); + appendBinaryStringInfo(buf, item->value.like_regex.pattern, + item->value.like_regex.patternlen); + appendStringInfoChar(buf, '\0'); + + chld = flattenJsonPathParseItem(buf, item->value.like_regex.expr, + nestingLevel, + insideArraySubscript); + *(int32 *) (buf->data + offs) = chld - pos; + } + break; + case jpiFilter: + argNestingLevel++; + /* FALLTHROUGH */ + case jpiIsUnknown: + case jpiNot: + case jpiPlus: + case jpiMinus: + case jpiExists: + case jpiDatetime: + { + int32 arg = reserveSpaceForItemPointer(buf); + + chld = !item->value.arg ? pos : + flattenJsonPathParseItem(buf, item->value.arg, + nestingLevel + argNestingLevel, + insideArraySubscript); + *(int32 *) (buf->data + arg) = chld - pos; + } + break; + case jpiNull: + break; + case jpiRoot: + break; + case jpiAnyArray: + case jpiAnyKey: + break; + case jpiCurrent: + if (nestingLevel <= 0) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("@ is not allowed in root expressions"))); + break; + case jpiLast: + if (!insideArraySubscript) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("LAST is allowed only in array subscripts"))); + break; + case jpiIndexArray: + { + int32 nelems = item->value.array.nelems; + int offset; + int i; + + appendBinaryStringInfo(buf, (char *) &nelems, sizeof(nelems)); + + offset = buf->len; + + appendStringInfoSpaces(buf, sizeof(int32) * 2 * nelems); + + for (i = 0; i < nelems; i++) + { + int32 *ppos; + int32 topos; + int32 frompos = + flattenJsonPathParseItem(buf, + item->value.array.elems[i].from, + nestingLevel, true) - pos; + + if (item->value.array.elems[i].to) + topos = flattenJsonPathParseItem(buf, + item->value.array.elems[i].to, + nestingLevel, true) - pos; + else + topos = 0; + + ppos = (int32 *) &buf->data[offset + i * 2 * sizeof(int32)]; + + ppos[0] = frompos; + ppos[1] = topos; + } + } + break; + case jpiAny: + appendBinaryStringInfo(buf, + (char *) &item->value.anybounds.first, + sizeof(item->value.anybounds.first)); + appendBinaryStringInfo(buf, + (char *) &item->value.anybounds.last, + sizeof(item->value.anybounds.last)); + break; + case jpiType: + case jpiSize: + case jpiAbs: + case jpiFloor: + case jpiCeiling: + case jpiDouble: + case jpiKeyValue: + break; + default: + elog(ERROR, "unrecognized jsonpath item type: %d", item->type); + } + + if (item->next) + { + chld = flattenJsonPathParseItem(buf, item->next, nestingLevel, + insideArraySubscript) - pos; + *(int32 *) (buf->data + next) = chld; + } + + return pos; +} + +/* + * Align StringInfo to int by adding zero padding bytes + */ +static void +alignStringInfoInt(StringInfo buf) +{ + switch (INTALIGN(buf->len) - buf->len) + { + case 3: + appendStringInfoCharMacro(buf, 0); + /* FALLTHROUGH */ + case 2: + appendStringInfoCharMacro(buf, 0); + /* FALLTHROUGH */ + case 1: + appendStringInfoCharMacro(buf, 0); + /* FALLTHROUGH */ + default: + break; + } +} + +/* + * Reserve space for int32 JsonPathItem pointer. Now zero pointer is written, + * actual value will be recorded at '(int32 *) &buf->data[pos]' later. + */ +static int32 +reserveSpaceForItemPointer(StringInfo buf) +{ + int32 pos = buf->len; + int32 ptr = 0; + + appendBinaryStringInfo(buf, (char *) &ptr, sizeof(ptr)); + + return pos; +} + +/* + * Prints text representation of given jsonpath item and all its children. + */ +static void +printJsonPathItem(StringInfo buf, JsonPathItem *v, bool inKey, + bool printBracketes) +{ + JsonPathItem elem; + int i; + + check_stack_depth(); + CHECK_FOR_INTERRUPTS(); + + switch (v->type) + { + case jpiNull: + appendStringInfoString(buf, "null"); + break; + case jpiKey: + if (inKey) + appendStringInfoChar(buf, '.'); + escape_json(buf, jspGetString(v, NULL)); + break; + case jpiString: + escape_json(buf, jspGetString(v, NULL)); + break; + case jpiVariable: + appendStringInfoChar(buf, '$'); + escape_json(buf, jspGetString(v, NULL)); + break; + case jpiNumeric: + appendStringInfoString(buf, + DatumGetCString(DirectFunctionCall1(numeric_out, + NumericGetDatum(jspGetNumeric(v))))); + break; + case jpiBool: + if (jspGetBool(v)) + appendBinaryStringInfo(buf, "true", 4); + else + appendBinaryStringInfo(buf, "false", 5); + break; + case jpiAnd: + case jpiOr: + case jpiEqual: + case jpiNotEqual: + case jpiLess: + case jpiGreater: + case jpiLessOrEqual: + case jpiGreaterOrEqual: + case jpiAdd: + case jpiSub: + case jpiMul: + case jpiDiv: + case jpiMod: + case jpiStartsWith: + if (printBracketes) + appendStringInfoChar(buf, '('); + jspGetLeftArg(v, &elem); + printJsonPathItem(buf, &elem, false, + operationPriority(elem.type) <= + operationPriority(v->type)); + appendStringInfoChar(buf, ' '); + appendStringInfoString(buf, jspOperationName(v->type)); + appendStringInfoChar(buf, ' '); + jspGetRightArg(v, &elem); + printJsonPathItem(buf, &elem, false, + operationPriority(elem.type) <= + operationPriority(v->type)); + if (printBracketes) + appendStringInfoChar(buf, ')'); + break; + case jpiLikeRegex: + if (printBracketes) + appendStringInfoChar(buf, '('); + + jspInitByBuffer(&elem, v->base, v->content.like_regex.expr); + printJsonPathItem(buf, &elem, false, + operationPriority(elem.type) <= + operationPriority(v->type)); + + appendBinaryStringInfo(buf, " like_regex ", 12); + + escape_json(buf, v->content.like_regex.pattern); + + if (v->content.like_regex.flags) + { + appendBinaryStringInfo(buf, " flag \"", 7); + + if (v->content.like_regex.flags & JSP_REGEX_ICASE) + appendStringInfoChar(buf, 'i'); + if (v->content.like_regex.flags & JSP_REGEX_DOTALL) + appendStringInfoChar(buf, 's'); + if (v->content.like_regex.flags & JSP_REGEX_MLINE) + appendStringInfoChar(buf, 'm'); + if (v->content.like_regex.flags & JSP_REGEX_WSPACE) + appendStringInfoChar(buf, 'x'); + if (v->content.like_regex.flags & JSP_REGEX_QUOTE) + appendStringInfoChar(buf, 'q'); + + appendStringInfoChar(buf, '"'); + } + + if (printBracketes) + appendStringInfoChar(buf, ')'); + break; + case jpiPlus: + case jpiMinus: + if (printBracketes) + appendStringInfoChar(buf, '('); + appendStringInfoChar(buf, v->type == jpiPlus ? '+' : '-'); + jspGetArg(v, &elem); + printJsonPathItem(buf, &elem, false, + operationPriority(elem.type) <= + operationPriority(v->type)); + if (printBracketes) + appendStringInfoChar(buf, ')'); + break; + case jpiFilter: + appendBinaryStringInfo(buf, "?(", 2); + jspGetArg(v, &elem); + printJsonPathItem(buf, &elem, false, false); + appendStringInfoChar(buf, ')'); + break; + case jpiNot: + appendBinaryStringInfo(buf, "!(", 2); + jspGetArg(v, &elem); + printJsonPathItem(buf, &elem, false, false); + appendStringInfoChar(buf, ')'); + break; + case jpiIsUnknown: + appendStringInfoChar(buf, '('); + jspGetArg(v, &elem); + printJsonPathItem(buf, &elem, false, false); + appendBinaryStringInfo(buf, ") is unknown", 12); + break; + case jpiExists: + appendBinaryStringInfo(buf, "exists (", 8); + jspGetArg(v, &elem); + printJsonPathItem(buf, &elem, false, false); + appendStringInfoChar(buf, ')'); + break; + case jpiCurrent: + Assert(!inKey); + appendStringInfoChar(buf, '@'); + break; + case jpiRoot: + Assert(!inKey); + appendStringInfoChar(buf, '$'); + break; + case jpiLast: + appendBinaryStringInfo(buf, "last", 4); + break; + case jpiAnyArray: + appendBinaryStringInfo(buf, "[*]", 3); + break; + case jpiAnyKey: + if (inKey) + appendStringInfoChar(buf, '.'); + appendStringInfoChar(buf, '*'); + break; + case jpiIndexArray: + appendStringInfoChar(buf, '['); + for (i = 0; i < v->content.array.nelems; i++) + { + JsonPathItem from; + JsonPathItem to; + bool range = jspGetArraySubscript(v, &from, &to, i); + + if (i) + appendStringInfoChar(buf, ','); + + printJsonPathItem(buf, &from, false, false); + + if (range) + { + appendBinaryStringInfo(buf, " to ", 4); + printJsonPathItem(buf, &to, false, false); + } + } + appendStringInfoChar(buf, ']'); + break; + case jpiAny: + if (inKey) + appendStringInfoChar(buf, '.'); + + if (v->content.anybounds.first == 0 && + v->content.anybounds.last == PG_UINT32_MAX) + appendBinaryStringInfo(buf, "**", 2); + else if (v->content.anybounds.first == v->content.anybounds.last) + { + if (v->content.anybounds.first == PG_UINT32_MAX) + appendStringInfo(buf, "**{last}"); + else + appendStringInfo(buf, "**{%u}", + v->content.anybounds.first); + } + else if (v->content.anybounds.first == PG_UINT32_MAX) + appendStringInfo(buf, "**{last to %u}", + v->content.anybounds.last); + else if (v->content.anybounds.last == PG_UINT32_MAX) + appendStringInfo(buf, "**{%u to last}", + v->content.anybounds.first); + else + appendStringInfo(buf, "**{%u to %u}", + v->content.anybounds.first, + v->content.anybounds.last); + break; + case jpiType: + appendBinaryStringInfo(buf, ".type()", 7); + break; + case jpiSize: + appendBinaryStringInfo(buf, ".size()", 7); + break; + case jpiAbs: + appendBinaryStringInfo(buf, ".abs()", 6); + break; + case jpiFloor: + appendBinaryStringInfo(buf, ".floor()", 8); + break; + case jpiCeiling: + appendBinaryStringInfo(buf, ".ceiling()", 10); + break; + case jpiDouble: + appendBinaryStringInfo(buf, ".double()", 9); + break; + case jpiDatetime: + appendBinaryStringInfo(buf, ".datetime(", 10); + if (v->content.arg) + { + jspGetArg(v, &elem); + printJsonPathItem(buf, &elem, false, false); + } + appendStringInfoChar(buf, ')'); + break; + case jpiKeyValue: + appendBinaryStringInfo(buf, ".keyvalue()", 11); + break; + default: + elog(ERROR, "unrecognized jsonpath item type: %d", v->type); + } + + if (jspGetNext(v, &elem)) + printJsonPathItem(buf, &elem, true, true); +} + +const char * +jspOperationName(JsonPathItemType type) +{ + switch (type) + { + case jpiAnd: + return "&&"; + case jpiOr: + return "||"; + case jpiEqual: + return "=="; + case jpiNotEqual: + return "!="; + case jpiLess: + return "<"; + case jpiGreater: + return ">"; + case jpiLessOrEqual: + return "<="; + case jpiGreaterOrEqual: + return ">="; + case jpiPlus: + case jpiAdd: + return "+"; + case jpiMinus: + case jpiSub: + return "-"; + case jpiMul: + return "*"; + case jpiDiv: + return "/"; + case jpiMod: + return "%"; + case jpiStartsWith: + return "starts with"; + case jpiLikeRegex: + return "like_regex"; + case jpiType: + return "type"; + case jpiSize: + return "size"; + case jpiKeyValue: + return "keyvalue"; + case jpiDouble: + return "double"; + case jpiAbs: + return "abs"; + case jpiFloor: + return "floor"; + case jpiCeiling: + return "ceiling"; + case jpiDatetime: + return "datetime"; + default: + elog(ERROR, "unrecognized jsonpath item type: %d", type); + return NULL; + } +} + +static int +operationPriority(JsonPathItemType op) +{ + switch (op) + { + case jpiOr: + return 0; + case jpiAnd: + return 1; + case jpiEqual: + case jpiNotEqual: + case jpiLess: + case jpiGreater: + case jpiLessOrEqual: + case jpiGreaterOrEqual: + case jpiStartsWith: + return 2; + case jpiAdd: + case jpiSub: + return 3; + case jpiMul: + case jpiDiv: + case jpiMod: + return 4; + case jpiPlus: + case jpiMinus: + return 5; + default: + return 6; + } +} + +/******************* Support functions for JsonPath *************************/ + +/* + * Support macros to read stored values + */ + +#define read_byte(v, b, p) do { \ + (v) = *(uint8*)((b) + (p)); \ + (p) += 1; \ +} while(0) \ + +#define read_int32(v, b, p) do { \ + (v) = *(uint32*)((b) + (p)); \ + (p) += sizeof(int32); \ +} while(0) \ + +#define read_int32_n(v, b, p, n) do { \ + (v) = (void *)((b) + (p)); \ + (p) += sizeof(int32) * (n); \ +} while(0) \ + +/* + * Read root node and fill root node representation + */ +void +jspInit(JsonPathItem *v, JsonPath *js) +{ + Assert((js->header & ~JSONPATH_LAX) == JSONPATH_VERSION); + jspInitByBuffer(v, js->data, 0); +} + +/* + * Read node from buffer and fill its representation + */ +void +jspInitByBuffer(JsonPathItem *v, char *base, int32 pos) +{ + v->base = base + pos; + + read_byte(v->type, base, pos); + pos = INTALIGN((uintptr_t) (base + pos)) - (uintptr_t) base; + read_int32(v->nextPos, base, pos); + + switch (v->type) + { + case jpiNull: + case jpiRoot: + case jpiCurrent: + case jpiAnyArray: + case jpiAnyKey: + case jpiType: + case jpiSize: + case jpiAbs: + case jpiFloor: + case jpiCeiling: + case jpiDouble: + case jpiKeyValue: + case jpiLast: + break; + case jpiKey: + case jpiString: + case jpiVariable: + read_int32(v->content.value.datalen, base, pos); + /* FALLTHROUGH */ + case jpiNumeric: + case jpiBool: + v->content.value.data = base + pos; + break; + case jpiAnd: + case jpiOr: + case jpiAdd: + case jpiSub: + case jpiMul: + case jpiDiv: + case jpiMod: + case jpiEqual: + case jpiNotEqual: + case jpiLess: + case jpiGreater: + case jpiLessOrEqual: + case jpiGreaterOrEqual: + case jpiStartsWith: + read_int32(v->content.args.left, base, pos); + read_int32(v->content.args.right, base, pos); + break; + case jpiLikeRegex: + read_int32(v->content.like_regex.flags, base, pos); + read_int32(v->content.like_regex.expr, base, pos); + read_int32(v->content.like_regex.patternlen, base, pos); + v->content.like_regex.pattern = base + pos; + break; + case jpiNot: + case jpiExists: + case jpiIsUnknown: + case jpiPlus: + case jpiMinus: + case jpiFilter: + case jpiDatetime: + read_int32(v->content.arg, base, pos); + break; + case jpiIndexArray: + read_int32(v->content.array.nelems, base, pos); + read_int32_n(v->content.array.elems, base, pos, + v->content.array.nelems * 2); + break; + case jpiAny: + read_int32(v->content.anybounds.first, base, pos); + read_int32(v->content.anybounds.last, base, pos); + break; + default: + elog(ERROR, "unrecognized jsonpath item type: %d", v->type); + } +} + +void +jspGetArg(JsonPathItem *v, JsonPathItem *a) +{ + Assert(v->type == jpiFilter || + v->type == jpiNot || + v->type == jpiIsUnknown || + v->type == jpiExists || + v->type == jpiPlus || + v->type == jpiMinus || + v->type == jpiDatetime); + + jspInitByBuffer(a, v->base, v->content.arg); +} + +bool +jspGetNext(JsonPathItem *v, JsonPathItem *a) +{ + if (jspHasNext(v)) + { + Assert(v->type == jpiString || + v->type == jpiNumeric || + v->type == jpiBool || + v->type == jpiNull || + v->type == jpiKey || + v->type == jpiAny || + v->type == jpiAnyArray || + v->type == jpiAnyKey || + v->type == jpiIndexArray || + v->type == jpiFilter || + v->type == jpiCurrent || + v->type == jpiExists || + v->type == jpiRoot || + v->type == jpiVariable || + v->type == jpiLast || + v->type == jpiAdd || + v->type == jpiSub || + v->type == jpiMul || + v->type == jpiDiv || + v->type == jpiMod || + v->type == jpiPlus || + v->type == jpiMinus || + v->type == jpiEqual || + v->type == jpiNotEqual || + v->type == jpiGreater || + v->type == jpiGreaterOrEqual || + v->type == jpiLess || + v->type == jpiLessOrEqual || + v->type == jpiAnd || + v->type == jpiOr || + v->type == jpiNot || + v->type == jpiIsUnknown || + v->type == jpiType || + v->type == jpiSize || + v->type == jpiAbs || + v->type == jpiFloor || + v->type == jpiCeiling || + v->type == jpiDouble || + v->type == jpiDatetime || + v->type == jpiKeyValue || + v->type == jpiStartsWith || + v->type == jpiLikeRegex); + + if (a) + jspInitByBuffer(a, v->base, v->nextPos); + return true; + } + + return false; +} + +void +jspGetLeftArg(JsonPathItem *v, JsonPathItem *a) +{ + Assert(v->type == jpiAnd || + v->type == jpiOr || + v->type == jpiEqual || + v->type == jpiNotEqual || + v->type == jpiLess || + v->type == jpiGreater || + v->type == jpiLessOrEqual || + v->type == jpiGreaterOrEqual || + v->type == jpiAdd || + v->type == jpiSub || + v->type == jpiMul || + v->type == jpiDiv || + v->type == jpiMod || + v->type == jpiStartsWith); + + jspInitByBuffer(a, v->base, v->content.args.left); +} + +void +jspGetRightArg(JsonPathItem *v, JsonPathItem *a) +{ + Assert(v->type == jpiAnd || + v->type == jpiOr || + v->type == jpiEqual || + v->type == jpiNotEqual || + v->type == jpiLess || + v->type == jpiGreater || + v->type == jpiLessOrEqual || + v->type == jpiGreaterOrEqual || + v->type == jpiAdd || + v->type == jpiSub || + v->type == jpiMul || + v->type == jpiDiv || + v->type == jpiMod || + v->type == jpiStartsWith); + + jspInitByBuffer(a, v->base, v->content.args.right); +} + +bool +jspGetBool(JsonPathItem *v) +{ + Assert(v->type == jpiBool); + + return (bool) *v->content.value.data; +} + +Numeric +jspGetNumeric(JsonPathItem *v) +{ + Assert(v->type == jpiNumeric); + + return (Numeric) v->content.value.data; +} + +char * +jspGetString(JsonPathItem *v, int32 *len) +{ + Assert(v->type == jpiKey || + v->type == jpiString || + v->type == jpiVariable); + + if (len) + *len = v->content.value.datalen; + return v->content.value.data; +} + +bool +jspGetArraySubscript(JsonPathItem *v, JsonPathItem *from, JsonPathItem *to, + int i) +{ + Assert(v->type == jpiIndexArray); + + jspInitByBuffer(from, v->base, v->content.array.elems[i].from); + + if (!v->content.array.elems[i].to) + return false; + + jspInitByBuffer(to, v->base, v->content.array.elems[i].to); + + return true; +} diff --git a/src/vuln_analysis/tools/tests/tests_data/parser.c b/src/vuln_analysis/tools/tests/tests_data/parser.c new file mode 100644 index 000000000..1c5e036ea --- /dev/null +++ b/src/vuln_analysis/tools/tests/tests_data/parser.c @@ -0,0 +1,15570 @@ +/* + * parser.c : an XML 1.0 parser, namespaces and validity support are mostly + * implemented on top of the SAX interfaces + * + * References: + * The XML specification: + * http://www.w3.org/TR/REC-xml + * Original 1.0 version: + * http://www.w3.org/TR/1998/REC-xml-19980210 + * XML second edition working draft + * http://www.w3.org/TR/2000/WD-xml-2e-20000814 + * + * Okay this is a big file, the parser core is around 7000 lines, then it + * is followed by the progressive parser top routines, then the various + * high level APIs to call the parser and a few miscellaneous functions. + * A number of helper functions and deprecated ones have been moved to + * parserInternals.c to reduce this file size. + * As much as possible the functions are associated with their relative + * production in the XML specification. A few productions defining the + * different ranges of character are actually implanted either in + * parserInternals.h or parserInternals.c + * The DOM tree build is realized from the default SAX callbacks in + * the module SAX.c. + * The routines doing the validation checks are in valid.c and called either + * from the SAX callbacks or as standalone functions using a preparsed + * document. + * + * See Copyright for the status of this software. + * + * daniel@veillard.com + */ + +/* To avoid EBCDIC trouble when parsing on zOS */ +#if defined(__MVS__) +#pragma convert("ISO8859-1") +#endif + +#define IN_LIBXML +#include "libxml.h" + +#if defined(_WIN32) && !defined (__CYGWIN__) +#define XML_DIR_SEP '\\' +#else +#define XML_DIR_SEP '/' +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef LIBXML_CATALOG_ENABLED +#include +#endif +#ifdef LIBXML_SCHEMAS_ENABLED +#include +#include +#endif +#ifdef HAVE_CTYPE_H +#include +#endif +#ifdef HAVE_STDLIB_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif +#ifdef HAVE_FCNTL_H +#include +#endif +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_ZLIB_H +#include +#endif +#ifdef HAVE_LZMA_H +#include +#endif + +#include "buf.h" +#include "enc.h" + +static void +xmlFatalErr(xmlParserCtxtPtr ctxt, xmlParserErrors error, const char *info); + +static xmlParserCtxtPtr +xmlCreateEntityParserCtxtInternal(const xmlChar *URL, const xmlChar *ID, + const xmlChar *base, xmlParserCtxtPtr pctx); + +static void xmlHaltParser(xmlParserCtxtPtr ctxt); + +/************************************************************************ + * * + * Arbitrary limits set in the parser. See XML_PARSE_HUGE * + * * + ************************************************************************/ + +#define XML_PARSER_BIG_ENTITY 1000 +#define XML_PARSER_LOT_ENTITY 5000 + +/* + * XML_PARSER_NON_LINEAR is the threshold where the ratio of parsed entity + * replacement over the size in byte of the input indicates that you have + * and eponential behaviour. A value of 10 correspond to at least 3 entity + * replacement per byte of input. + */ +#define XML_PARSER_NON_LINEAR 10 + +/* + * xmlParserEntityCheck + * + * Function to check non-linear entity expansion behaviour + * This is here to detect and stop exponential linear entity expansion + * This is not a limitation of the parser but a safety + * boundary feature. It can be disabled with the XML_PARSE_HUGE + * parser option. + */ +static int +xmlParserEntityCheck(xmlParserCtxtPtr ctxt, size_t size, + xmlEntityPtr ent, size_t replacement) +{ + size_t consumed = 0; + + if ((ctxt == NULL) || (ctxt->options & XML_PARSE_HUGE)) + return (0); + if (ctxt->lastError.code == XML_ERR_ENTITY_LOOP) + return (1); + + /* + * This may look absurd but is needed to detect + * entities problems + */ + if ((ent != NULL) && (ent->etype != XML_INTERNAL_PREDEFINED_ENTITY) && + (ent->content != NULL) && (ent->checked == 0) && + (ctxt->errNo != XML_ERR_ENTITY_LOOP)) { + unsigned long oldnbent = ctxt->nbentities; + xmlChar *rep; + + ent->checked = 1; + + ++ctxt->depth; + rep = xmlStringDecodeEntities(ctxt, ent->content, + XML_SUBSTITUTE_REF, 0, 0, 0); + --ctxt->depth; + if (ctxt->errNo == XML_ERR_ENTITY_LOOP) { + ent->content[0] = 0; + } + + ent->checked = (ctxt->nbentities - oldnbent + 1) * 2; + if (rep != NULL) { + if (xmlStrchr(rep, '<')) + ent->checked |= 1; + xmlFree(rep); + rep = NULL; + } + } + if (replacement != 0) { + if (replacement < XML_MAX_TEXT_LENGTH) + return(0); + + /* + * If the volume of entity copy reaches 10 times the + * amount of parsed data and over the large text threshold + * then that's very likely to be an abuse. + */ + if (ctxt->input != NULL) { + consumed = ctxt->input->consumed + + (ctxt->input->cur - ctxt->input->base); + } + consumed += ctxt->sizeentities; + + if (replacement < XML_PARSER_NON_LINEAR * consumed) + return(0); + } else if (size != 0) { + /* + * Do the check based on the replacement size of the entity + */ + if (size < XML_PARSER_BIG_ENTITY) + return(0); + + /* + * A limit on the amount of text data reasonably used + */ + if (ctxt->input != NULL) { + consumed = ctxt->input->consumed + + (ctxt->input->cur - ctxt->input->base); + } + consumed += ctxt->sizeentities; + + if ((size < XML_PARSER_NON_LINEAR * consumed) && + (ctxt->nbentities * 3 < XML_PARSER_NON_LINEAR * consumed)) + return (0); + } else if (ent != NULL) { + /* + * use the number of parsed entities in the replacement + */ + size = ent->checked / 2; + + /* + * The amount of data parsed counting entities size only once + */ + if (ctxt->input != NULL) { + consumed = ctxt->input->consumed + + (ctxt->input->cur - ctxt->input->base); + } + consumed += ctxt->sizeentities; + + /* + * Check the density of entities for the amount of data + * knowing an entity reference will take at least 3 bytes + */ + if (size * 3 < consumed * XML_PARSER_NON_LINEAR) + return (0); + } else { + /* + * strange we got no data for checking + */ + if (((ctxt->lastError.code != XML_ERR_UNDECLARED_ENTITY) && + (ctxt->lastError.code != XML_WAR_UNDECLARED_ENTITY)) || + (ctxt->nbentities <= 10000)) + return (0); + } + xmlFatalErr(ctxt, XML_ERR_ENTITY_LOOP, NULL); + return (1); +} + +/** + * xmlParserMaxDepth: + * + * arbitrary depth limit for the XML documents that we allow to + * process. This is not a limitation of the parser but a safety + * boundary feature. It can be disabled with the XML_PARSE_HUGE + * parser option. + */ +unsigned int xmlParserMaxDepth = 256; + + + +#define SAX2 1 +#define XML_PARSER_BIG_BUFFER_SIZE 300 +#define XML_PARSER_BUFFER_SIZE 100 +#define SAX_COMPAT_MODE BAD_CAST "SAX compatibility mode document" + +/** + * XML_PARSER_CHUNK_SIZE + * + * When calling GROW that's the minimal amount of data + * the parser expected to have received. It is not a hard + * limit but an optimization when reading strings like Names + * It is not strictly needed as long as inputs available characters + * are followed by 0, which should be provided by the I/O level + */ +#define XML_PARSER_CHUNK_SIZE 100 + +/* + * List of XML prefixed PI allowed by W3C specs + */ + +static const char *xmlW3CPIs[] = { + "xml-stylesheet", + "xml-model", + NULL +}; + + +/* DEPR void xmlParserHandleReference(xmlParserCtxtPtr ctxt); */ +static xmlEntityPtr xmlParseStringPEReference(xmlParserCtxtPtr ctxt, + const xmlChar **str); + +static xmlParserErrors +xmlParseExternalEntityPrivate(xmlDocPtr doc, xmlParserCtxtPtr oldctxt, + xmlSAXHandlerPtr sax, + void *user_data, int depth, const xmlChar *URL, + const xmlChar *ID, xmlNodePtr *list); + +static int +xmlCtxtUseOptionsInternal(xmlParserCtxtPtr ctxt, int options, + const char *encoding); +#ifdef LIBXML_LEGACY_ENABLED +static void +xmlAddEntityReference(xmlEntityPtr ent, xmlNodePtr firstNode, + xmlNodePtr lastNode); +#endif /* LIBXML_LEGACY_ENABLED */ + +static xmlParserErrors +xmlParseBalancedChunkMemoryInternal(xmlParserCtxtPtr oldctxt, + const xmlChar *string, void *user_data, xmlNodePtr *lst); + +static int +xmlLoadEntityContent(xmlParserCtxtPtr ctxt, xmlEntityPtr entity); + +/************************************************************************ + * * + * Some factorized error routines * + * * + ************************************************************************/ + +/** + * xmlErrAttributeDup: + * @ctxt: an XML parser context + * @prefix: the attribute prefix + * @localname: the attribute localname + * + * Handle a redefinition of attribute error + */ +static void +xmlErrAttributeDup(xmlParserCtxtPtr ctxt, const xmlChar * prefix, + const xmlChar * localname) +{ + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + if (ctxt != NULL) + ctxt->errNo = XML_ERR_ATTRIBUTE_REDEFINED; + + if (prefix == NULL) + __xmlRaiseError(NULL, NULL, NULL, ctxt, NULL, XML_FROM_PARSER, + XML_ERR_ATTRIBUTE_REDEFINED, XML_ERR_FATAL, NULL, 0, + (const char *) localname, NULL, NULL, 0, 0, + "Attribute %s redefined\n", localname); + else + __xmlRaiseError(NULL, NULL, NULL, ctxt, NULL, XML_FROM_PARSER, + XML_ERR_ATTRIBUTE_REDEFINED, XML_ERR_FATAL, NULL, 0, + (const char *) prefix, (const char *) localname, + NULL, 0, 0, "Attribute %s:%s redefined\n", prefix, + localname); + if (ctxt != NULL) { + ctxt->wellFormed = 0; + if (ctxt->recovery == 0) + ctxt->disableSAX = 1; + } +} + +/** + * xmlFatalErr: + * @ctxt: an XML parser context + * @error: the error number + * @extra: extra information string + * + * Handle a fatal parser error, i.e. violating Well-Formedness constraints + */ +static void +xmlFatalErr(xmlParserCtxtPtr ctxt, xmlParserErrors error, const char *info) +{ + const char *errmsg; + + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + switch (error) { + case XML_ERR_INVALID_HEX_CHARREF: + errmsg = "CharRef: invalid hexadecimal value"; + break; + case XML_ERR_INVALID_DEC_CHARREF: + errmsg = "CharRef: invalid decimal value"; + break; + case XML_ERR_INVALID_CHARREF: + errmsg = "CharRef: invalid value"; + break; + case XML_ERR_INTERNAL_ERROR: + errmsg = "internal error"; + break; + case XML_ERR_PEREF_AT_EOF: + errmsg = "PEReference at end of document"; + break; + case XML_ERR_PEREF_IN_PROLOG: + errmsg = "PEReference in prolog"; + break; + case XML_ERR_PEREF_IN_EPILOG: + errmsg = "PEReference in epilog"; + break; + case XML_ERR_PEREF_NO_NAME: + errmsg = "PEReference: no name"; + break; + case XML_ERR_PEREF_SEMICOL_MISSING: + errmsg = "PEReference: expecting ';'"; + break; + case XML_ERR_ENTITY_LOOP: + errmsg = "Detected an entity reference loop"; + break; + case XML_ERR_ENTITY_NOT_STARTED: + errmsg = "EntityValue: \" or ' expected"; + break; + case XML_ERR_ENTITY_PE_INTERNAL: + errmsg = "PEReferences forbidden in internal subset"; + break; + case XML_ERR_ENTITY_NOT_FINISHED: + errmsg = "EntityValue: \" or ' expected"; + break; + case XML_ERR_ATTRIBUTE_NOT_STARTED: + errmsg = "AttValue: \" or ' expected"; + break; + case XML_ERR_LT_IN_ATTRIBUTE: + errmsg = "Unescaped '<' not allowed in attributes values"; + break; + case XML_ERR_LITERAL_NOT_STARTED: + errmsg = "SystemLiteral \" or ' expected"; + break; + case XML_ERR_LITERAL_NOT_FINISHED: + errmsg = "Unfinished System or Public ID \" or ' expected"; + break; + case XML_ERR_MISPLACED_CDATA_END: + errmsg = "Sequence ']]>' not allowed in content"; + break; + case XML_ERR_URI_REQUIRED: + errmsg = "SYSTEM or PUBLIC, the URI is missing"; + break; + case XML_ERR_PUBID_REQUIRED: + errmsg = "PUBLIC, the Public Identifier is missing"; + break; + case XML_ERR_HYPHEN_IN_COMMENT: + errmsg = "Comment must not contain '--' (double-hyphen)"; + break; + case XML_ERR_PI_NOT_STARTED: + errmsg = "xmlParsePI : no target name"; + break; + case XML_ERR_RESERVED_XML_NAME: + errmsg = "Invalid PI name"; + break; + case XML_ERR_NOTATION_NOT_STARTED: + errmsg = "NOTATION: Name expected here"; + break; + case XML_ERR_NOTATION_NOT_FINISHED: + errmsg = "'>' required to close NOTATION declaration"; + break; + case XML_ERR_VALUE_REQUIRED: + errmsg = "Entity value required"; + break; + case XML_ERR_URI_FRAGMENT: + errmsg = "Fragment not allowed"; + break; + case XML_ERR_ATTLIST_NOT_STARTED: + errmsg = "'(' required to start ATTLIST enumeration"; + break; + case XML_ERR_NMTOKEN_REQUIRED: + errmsg = "NmToken expected in ATTLIST enumeration"; + break; + case XML_ERR_ATTLIST_NOT_FINISHED: + errmsg = "')' required to finish ATTLIST enumeration"; + break; + case XML_ERR_MIXED_NOT_STARTED: + errmsg = "MixedContentDecl : '|' or ')*' expected"; + break; + case XML_ERR_PCDATA_REQUIRED: + errmsg = "MixedContentDecl : '#PCDATA' expected"; + break; + case XML_ERR_ELEMCONTENT_NOT_STARTED: + errmsg = "ContentDecl : Name or '(' expected"; + break; + case XML_ERR_ELEMCONTENT_NOT_FINISHED: + errmsg = "ContentDecl : ',' '|' or ')' expected"; + break; + case XML_ERR_PEREF_IN_INT_SUBSET: + errmsg = + "PEReference: forbidden within markup decl in internal subset"; + break; + case XML_ERR_GT_REQUIRED: + errmsg = "expected '>'"; + break; + case XML_ERR_CONDSEC_INVALID: + errmsg = "XML conditional section '[' expected"; + break; + case XML_ERR_EXT_SUBSET_NOT_FINISHED: + errmsg = "Content error in the external subset"; + break; + case XML_ERR_CONDSEC_INVALID_KEYWORD: + errmsg = + "conditional section INCLUDE or IGNORE keyword expected"; + break; + case XML_ERR_CONDSEC_NOT_FINISHED: + errmsg = "XML conditional section not closed"; + break; + case XML_ERR_XMLDECL_NOT_STARTED: + errmsg = "Text declaration '' expected"; + break; + case XML_ERR_EXT_ENTITY_STANDALONE: + errmsg = "external parsed entities cannot be standalone"; + break; + case XML_ERR_ENTITYREF_SEMICOL_MISSING: + errmsg = "EntityRef: expecting ';'"; + break; + case XML_ERR_DOCTYPE_NOT_FINISHED: + errmsg = "DOCTYPE improperly terminated"; + break; + case XML_ERR_LTSLASH_REQUIRED: + errmsg = "EndTag: 'errNo = error; + if (info == NULL) { + __xmlRaiseError(NULL, NULL, NULL, ctxt, NULL, XML_FROM_PARSER, error, + XML_ERR_FATAL, NULL, 0, info, NULL, NULL, 0, 0, "%s\n", + errmsg); + } else { + __xmlRaiseError(NULL, NULL, NULL, ctxt, NULL, XML_FROM_PARSER, error, + XML_ERR_FATAL, NULL, 0, info, NULL, NULL, 0, 0, "%s: %s\n", + errmsg, info); + } + if (ctxt != NULL) { + ctxt->wellFormed = 0; + if (ctxt->recovery == 0) + ctxt->disableSAX = 1; + } +} + +/** + * xmlFatalErrMsg: + * @ctxt: an XML parser context + * @error: the error number + * @msg: the error message + * + * Handle a fatal parser error, i.e. violating Well-Formedness constraints + */ +static void LIBXML_ATTR_FORMAT(3,0) +xmlFatalErrMsg(xmlParserCtxtPtr ctxt, xmlParserErrors error, + const char *msg) +{ + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + if (ctxt != NULL) + ctxt->errNo = error; + __xmlRaiseError(NULL, NULL, NULL, ctxt, NULL, XML_FROM_PARSER, error, + XML_ERR_FATAL, NULL, 0, NULL, NULL, NULL, 0, 0, "%s", msg); + if (ctxt != NULL) { + ctxt->wellFormed = 0; + if (ctxt->recovery == 0) + ctxt->disableSAX = 1; + } +} + +/** + * xmlWarningMsg: + * @ctxt: an XML parser context + * @error: the error number + * @msg: the error message + * @str1: extra data + * @str2: extra data + * + * Handle a warning. + */ +static void LIBXML_ATTR_FORMAT(3,0) +xmlWarningMsg(xmlParserCtxtPtr ctxt, xmlParserErrors error, + const char *msg, const xmlChar *str1, const xmlChar *str2) +{ + xmlStructuredErrorFunc schannel = NULL; + + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + if ((ctxt != NULL) && (ctxt->sax != NULL) && + (ctxt->sax->initialized == XML_SAX2_MAGIC)) + schannel = ctxt->sax->serror; + if (ctxt != NULL) { + __xmlRaiseError(schannel, + (ctxt->sax) ? ctxt->sax->warning : NULL, + ctxt->userData, + ctxt, NULL, XML_FROM_PARSER, error, + XML_ERR_WARNING, NULL, 0, + (const char *) str1, (const char *) str2, NULL, 0, 0, + msg, (const char *) str1, (const char *) str2); + } else { + __xmlRaiseError(schannel, NULL, NULL, + ctxt, NULL, XML_FROM_PARSER, error, + XML_ERR_WARNING, NULL, 0, + (const char *) str1, (const char *) str2, NULL, 0, 0, + msg, (const char *) str1, (const char *) str2); + } +} + +/** + * xmlValidityError: + * @ctxt: an XML parser context + * @error: the error number + * @msg: the error message + * @str1: extra data + * + * Handle a validity error. + */ +static void LIBXML_ATTR_FORMAT(3,0) +xmlValidityError(xmlParserCtxtPtr ctxt, xmlParserErrors error, + const char *msg, const xmlChar *str1, const xmlChar *str2) +{ + xmlStructuredErrorFunc schannel = NULL; + + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + if (ctxt != NULL) { + ctxt->errNo = error; + if ((ctxt->sax != NULL) && (ctxt->sax->initialized == XML_SAX2_MAGIC)) + schannel = ctxt->sax->serror; + } + if (ctxt != NULL) { + __xmlRaiseError(schannel, + ctxt->vctxt.error, ctxt->vctxt.userData, + ctxt, NULL, XML_FROM_DTD, error, + XML_ERR_ERROR, NULL, 0, (const char *) str1, + (const char *) str2, NULL, 0, 0, + msg, (const char *) str1, (const char *) str2); + ctxt->valid = 0; + } else { + __xmlRaiseError(schannel, NULL, NULL, + ctxt, NULL, XML_FROM_DTD, error, + XML_ERR_ERROR, NULL, 0, (const char *) str1, + (const char *) str2, NULL, 0, 0, + msg, (const char *) str1, (const char *) str2); + } +} + +/** + * xmlFatalErrMsgInt: + * @ctxt: an XML parser context + * @error: the error number + * @msg: the error message + * @val: an integer value + * + * Handle a fatal parser error, i.e. violating Well-Formedness constraints + */ +static void LIBXML_ATTR_FORMAT(3,0) +xmlFatalErrMsgInt(xmlParserCtxtPtr ctxt, xmlParserErrors error, + const char *msg, int val) +{ + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + if (ctxt != NULL) + ctxt->errNo = error; + __xmlRaiseError(NULL, NULL, NULL, + ctxt, NULL, XML_FROM_PARSER, error, XML_ERR_FATAL, + NULL, 0, NULL, NULL, NULL, val, 0, msg, val); + if (ctxt != NULL) { + ctxt->wellFormed = 0; + if (ctxt->recovery == 0) + ctxt->disableSAX = 1; + } +} + +/** + * xmlFatalErrMsgStrIntStr: + * @ctxt: an XML parser context + * @error: the error number + * @msg: the error message + * @str1: an string info + * @val: an integer value + * @str2: an string info + * + * Handle a fatal parser error, i.e. violating Well-Formedness constraints + */ +static void LIBXML_ATTR_FORMAT(3,0) +xmlFatalErrMsgStrIntStr(xmlParserCtxtPtr ctxt, xmlParserErrors error, + const char *msg, const xmlChar *str1, int val, + const xmlChar *str2) +{ + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + if (ctxt != NULL) + ctxt->errNo = error; + __xmlRaiseError(NULL, NULL, NULL, + ctxt, NULL, XML_FROM_PARSER, error, XML_ERR_FATAL, + NULL, 0, (const char *) str1, (const char *) str2, + NULL, val, 0, msg, str1, val, str2); + if (ctxt != NULL) { + ctxt->wellFormed = 0; + if (ctxt->recovery == 0) + ctxt->disableSAX = 1; + } +} + +/** + * xmlFatalErrMsgStr: + * @ctxt: an XML parser context + * @error: the error number + * @msg: the error message + * @val: a string value + * + * Handle a fatal parser error, i.e. violating Well-Formedness constraints + */ +static void LIBXML_ATTR_FORMAT(3,0) +xmlFatalErrMsgStr(xmlParserCtxtPtr ctxt, xmlParserErrors error, + const char *msg, const xmlChar * val) +{ + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + if (ctxt != NULL) + ctxt->errNo = error; + __xmlRaiseError(NULL, NULL, NULL, ctxt, NULL, + XML_FROM_PARSER, error, XML_ERR_FATAL, + NULL, 0, (const char *) val, NULL, NULL, 0, 0, msg, + val); + if (ctxt != NULL) { + ctxt->wellFormed = 0; + if (ctxt->recovery == 0) + ctxt->disableSAX = 1; + } +} + +/** + * xmlErrMsgStr: + * @ctxt: an XML parser context + * @error: the error number + * @msg: the error message + * @val: a string value + * + * Handle a non fatal parser error + */ +static void LIBXML_ATTR_FORMAT(3,0) +xmlErrMsgStr(xmlParserCtxtPtr ctxt, xmlParserErrors error, + const char *msg, const xmlChar * val) +{ + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + if (ctxt != NULL) + ctxt->errNo = error; + __xmlRaiseError(NULL, NULL, NULL, ctxt, NULL, + XML_FROM_PARSER, error, XML_ERR_ERROR, + NULL, 0, (const char *) val, NULL, NULL, 0, 0, msg, + val); +} + +/** + * xmlNsErr: + * @ctxt: an XML parser context + * @error: the error number + * @msg: the message + * @info1: extra information string + * @info2: extra information string + * + * Handle a fatal parser error, i.e. violating Well-Formedness constraints + */ +static void LIBXML_ATTR_FORMAT(3,0) +xmlNsErr(xmlParserCtxtPtr ctxt, xmlParserErrors error, + const char *msg, + const xmlChar * info1, const xmlChar * info2, + const xmlChar * info3) +{ + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + if (ctxt != NULL) + ctxt->errNo = error; + __xmlRaiseError(NULL, NULL, NULL, ctxt, NULL, XML_FROM_NAMESPACE, error, + XML_ERR_ERROR, NULL, 0, (const char *) info1, + (const char *) info2, (const char *) info3, 0, 0, msg, + info1, info2, info3); + if (ctxt != NULL) + ctxt->nsWellFormed = 0; +} + +/** + * xmlNsWarn + * @ctxt: an XML parser context + * @error: the error number + * @msg: the message + * @info1: extra information string + * @info2: extra information string + * + * Handle a namespace warning error + */ +static void LIBXML_ATTR_FORMAT(3,0) +xmlNsWarn(xmlParserCtxtPtr ctxt, xmlParserErrors error, + const char *msg, + const xmlChar * info1, const xmlChar * info2, + const xmlChar * info3) +{ + if ((ctxt != NULL) && (ctxt->disableSAX != 0) && + (ctxt->instate == XML_PARSER_EOF)) + return; + __xmlRaiseError(NULL, NULL, NULL, ctxt, NULL, XML_FROM_NAMESPACE, error, + XML_ERR_WARNING, NULL, 0, (const char *) info1, + (const char *) info2, (const char *) info3, 0, 0, msg, + info1, info2, info3); +} + +/************************************************************************ + * * + * Library wide options * + * * + ************************************************************************/ + +/** + * xmlHasFeature: + * @feature: the feature to be examined + * + * Examines if the library has been compiled with a given feature. + * + * Returns a non-zero value if the feature exist, otherwise zero. + * Returns zero (0) if the feature does not exist or an unknown + * unknown feature is requested, non-zero otherwise. + */ +int +xmlHasFeature(xmlFeature feature) +{ + switch (feature) { + case XML_WITH_THREAD: +#ifdef LIBXML_THREAD_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_TREE: +#ifdef LIBXML_TREE_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_OUTPUT: +#ifdef LIBXML_OUTPUT_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_PUSH: +#ifdef LIBXML_PUSH_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_READER: +#ifdef LIBXML_READER_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_PATTERN: +#ifdef LIBXML_PATTERN_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_WRITER: +#ifdef LIBXML_WRITER_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_SAX1: +#ifdef LIBXML_SAX1_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_FTP: +#ifdef LIBXML_FTP_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_HTTP: +#ifdef LIBXML_HTTP_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_VALID: +#ifdef LIBXML_VALID_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_HTML: +#ifdef LIBXML_HTML_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_LEGACY: +#ifdef LIBXML_LEGACY_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_C14N: +#ifdef LIBXML_C14N_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_CATALOG: +#ifdef LIBXML_CATALOG_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_XPATH: +#ifdef LIBXML_XPATH_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_XPTR: +#ifdef LIBXML_XPTR_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_XINCLUDE: +#ifdef LIBXML_XINCLUDE_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_ICONV: +#ifdef LIBXML_ICONV_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_ISO8859X: +#ifdef LIBXML_ISO8859X_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_UNICODE: +#ifdef LIBXML_UNICODE_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_REGEXP: +#ifdef LIBXML_REGEXP_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_AUTOMATA: +#ifdef LIBXML_AUTOMATA_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_EXPR: +#ifdef LIBXML_EXPR_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_SCHEMAS: +#ifdef LIBXML_SCHEMAS_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_SCHEMATRON: +#ifdef LIBXML_SCHEMATRON_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_MODULES: +#ifdef LIBXML_MODULES_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_DEBUG: +#ifdef LIBXML_DEBUG_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_DEBUG_MEM: +#ifdef DEBUG_MEMORY_LOCATION + return(1); +#else + return(0); +#endif + case XML_WITH_DEBUG_RUN: +#ifdef LIBXML_DEBUG_RUNTIME + return(1); +#else + return(0); +#endif + case XML_WITH_ZLIB: +#ifdef LIBXML_ZLIB_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_LZMA: +#ifdef LIBXML_LZMA_ENABLED + return(1); +#else + return(0); +#endif + case XML_WITH_ICU: +#ifdef LIBXML_ICU_ENABLED + return(1); +#else + return(0); +#endif + default: + break; + } + return(0); +} + +/************************************************************************ + * * + * SAX2 defaulted attributes handling * + * * + ************************************************************************/ + +/** + * xmlDetectSAX2: + * @ctxt: an XML parser context + * + * Do the SAX2 detection and specific intialization + */ +static void +xmlDetectSAX2(xmlParserCtxtPtr ctxt) { + if (ctxt == NULL) return; +#ifdef LIBXML_SAX1_ENABLED + if ((ctxt->sax) && (ctxt->sax->initialized == XML_SAX2_MAGIC) && + ((ctxt->sax->startElementNs != NULL) || + (ctxt->sax->endElementNs != NULL))) ctxt->sax2 = 1; +#else + ctxt->sax2 = 1; +#endif /* LIBXML_SAX1_ENABLED */ + + ctxt->str_xml = xmlDictLookup(ctxt->dict, BAD_CAST "xml", 3); + ctxt->str_xmlns = xmlDictLookup(ctxt->dict, BAD_CAST "xmlns", 5); + ctxt->str_xml_ns = xmlDictLookup(ctxt->dict, XML_XML_NAMESPACE, 36); + if ((ctxt->str_xml==NULL) || (ctxt->str_xmlns==NULL) || + (ctxt->str_xml_ns == NULL)) { + xmlErrMemory(ctxt, NULL); + } +} + +typedef struct _xmlDefAttrs xmlDefAttrs; +typedef xmlDefAttrs *xmlDefAttrsPtr; +struct _xmlDefAttrs { + int nbAttrs; /* number of defaulted attributes on that element */ + int maxAttrs; /* the size of the array */ +#if __STDC_VERSION__ >= 199901L + /* Using a C99 flexible array member avoids UBSan errors. */ + const xmlChar *values[]; /* array of localname/prefix/values/external */ +#else + const xmlChar *values[5]; +#endif +}; + +/** + * xmlAttrNormalizeSpace: + * @src: the source string + * @dst: the target string + * + * Normalize the space in non CDATA attribute values: + * If the attribute type is not CDATA, then the XML processor MUST further + * process the normalized attribute value by discarding any leading and + * trailing space (#x20) characters, and by replacing sequences of space + * (#x20) characters by a single space (#x20) character. + * Note that the size of dst need to be at least src, and if one doesn't need + * to preserve dst (and it doesn't come from a dictionary or read-only) then + * passing src as dst is just fine. + * + * Returns a pointer to the normalized value (dst) or NULL if no conversion + * is needed. + */ +static xmlChar * +xmlAttrNormalizeSpace(const xmlChar *src, xmlChar *dst) +{ + if ((src == NULL) || (dst == NULL)) + return(NULL); + + while (*src == 0x20) src++; + while (*src != 0) { + if (*src == 0x20) { + while (*src == 0x20) src++; + if (*src != 0) + *dst++ = 0x20; + } else { + *dst++ = *src++; + } + } + *dst = 0; + if (dst == src) + return(NULL); + return(dst); +} + +/** + * xmlAttrNormalizeSpace2: + * @src: the source string + * + * Normalize the space in non CDATA attribute values, a slightly more complex + * front end to avoid allocation problems when running on attribute values + * coming from the input. + * + * Returns a pointer to the normalized value (dst) or NULL if no conversion + * is needed. + */ +static const xmlChar * +xmlAttrNormalizeSpace2(xmlParserCtxtPtr ctxt, xmlChar *src, int *len) +{ + int i; + int remove_head = 0; + int need_realloc = 0; + const xmlChar *cur; + + if ((ctxt == NULL) || (src == NULL) || (len == NULL)) + return(NULL); + i = *len; + if (i <= 0) + return(NULL); + + cur = src; + while (*cur == 0x20) { + cur++; + remove_head++; + } + while (*cur != 0) { + if (*cur == 0x20) { + cur++; + if ((*cur == 0x20) || (*cur == 0)) { + need_realloc = 1; + break; + } + } else + cur++; + } + if (need_realloc) { + xmlChar *ret; + + ret = xmlStrndup(src + remove_head, i - remove_head + 1); + if (ret == NULL) { + xmlErrMemory(ctxt, NULL); + return(NULL); + } + xmlAttrNormalizeSpace(ret, ret); + *len = (int) strlen((const char *)ret); + return(ret); + } else if (remove_head) { + *len -= remove_head; + memmove(src, src + remove_head, 1 + *len); + return(src); + } + return(NULL); +} + +/** + * xmlAddDefAttrs: + * @ctxt: an XML parser context + * @fullname: the element fullname + * @fullattr: the attribute fullname + * @value: the attribute value + * + * Add a defaulted attribute for an element + */ +static void +xmlAddDefAttrs(xmlParserCtxtPtr ctxt, + const xmlChar *fullname, + const xmlChar *fullattr, + const xmlChar *value) { + xmlDefAttrsPtr defaults; + int len; + const xmlChar *name; + const xmlChar *prefix; + + /* + * Allows to detect attribute redefinitions + */ + if (ctxt->attsSpecial != NULL) { + if (xmlHashLookup2(ctxt->attsSpecial, fullname, fullattr) != NULL) + return; + } + + if (ctxt->attsDefault == NULL) { + ctxt->attsDefault = xmlHashCreateDict(10, ctxt->dict); + if (ctxt->attsDefault == NULL) + goto mem_error; + } + + /* + * split the element name into prefix:localname , the string found + * are within the DTD and then not associated to namespace names. + */ + name = xmlSplitQName3(fullname, &len); + if (name == NULL) { + name = xmlDictLookup(ctxt->dict, fullname, -1); + prefix = NULL; + } else { + name = xmlDictLookup(ctxt->dict, name, -1); + prefix = xmlDictLookup(ctxt->dict, fullname, len); + } + + /* + * make sure there is some storage + */ + defaults = xmlHashLookup2(ctxt->attsDefault, name, prefix); + if (defaults == NULL) { + defaults = (xmlDefAttrsPtr) xmlMalloc(sizeof(xmlDefAttrs) + + (4 * 5) * sizeof(const xmlChar *)); + if (defaults == NULL) + goto mem_error; + defaults->nbAttrs = 0; + defaults->maxAttrs = 4; + if (xmlHashUpdateEntry2(ctxt->attsDefault, name, prefix, + defaults, NULL) < 0) { + xmlFree(defaults); + goto mem_error; + } + } else if (defaults->nbAttrs >= defaults->maxAttrs) { + xmlDefAttrsPtr temp; + + temp = (xmlDefAttrsPtr) xmlRealloc(defaults, sizeof(xmlDefAttrs) + + (2 * defaults->maxAttrs * 5) * sizeof(const xmlChar *)); + if (temp == NULL) + goto mem_error; + defaults = temp; + defaults->maxAttrs *= 2; + if (xmlHashUpdateEntry2(ctxt->attsDefault, name, prefix, + defaults, NULL) < 0) { + xmlFree(defaults); + goto mem_error; + } + } + + /* + * Split the element name into prefix:localname , the string found + * are within the DTD and hen not associated to namespace names. + */ + name = xmlSplitQName3(fullattr, &len); + if (name == NULL) { + name = xmlDictLookup(ctxt->dict, fullattr, -1); + prefix = NULL; + } else { + name = xmlDictLookup(ctxt->dict, name, -1); + prefix = xmlDictLookup(ctxt->dict, fullattr, len); + } + + defaults->values[5 * defaults->nbAttrs] = name; + defaults->values[5 * defaults->nbAttrs + 1] = prefix; + /* intern the string and precompute the end */ + len = xmlStrlen(value); + value = xmlDictLookup(ctxt->dict, value, len); + defaults->values[5 * defaults->nbAttrs + 2] = value; + defaults->values[5 * defaults->nbAttrs + 3] = value + len; + if (ctxt->external) + defaults->values[5 * defaults->nbAttrs + 4] = BAD_CAST "external"; + else + defaults->values[5 * defaults->nbAttrs + 4] = NULL; + defaults->nbAttrs++; + + return; + +mem_error: + xmlErrMemory(ctxt, NULL); + return; +} + +/** + * xmlAddSpecialAttr: + * @ctxt: an XML parser context + * @fullname: the element fullname + * @fullattr: the attribute fullname + * @type: the attribute type + * + * Register this attribute type + */ +static void +xmlAddSpecialAttr(xmlParserCtxtPtr ctxt, + const xmlChar *fullname, + const xmlChar *fullattr, + int type) +{ + if (ctxt->attsSpecial == NULL) { + ctxt->attsSpecial = xmlHashCreateDict(10, ctxt->dict); + if (ctxt->attsSpecial == NULL) + goto mem_error; + } + + if (xmlHashLookup2(ctxt->attsSpecial, fullname, fullattr) != NULL) + return; + + xmlHashAddEntry2(ctxt->attsSpecial, fullname, fullattr, + (void *) (ptrdiff_t) type); + return; + +mem_error: + xmlErrMemory(ctxt, NULL); + return; +} + +/** + * xmlCleanSpecialAttrCallback: + * + * Removes CDATA attributes from the special attribute table + */ +static void +xmlCleanSpecialAttrCallback(void *payload, void *data, + const xmlChar *fullname, const xmlChar *fullattr, + const xmlChar *unused ATTRIBUTE_UNUSED) { + xmlParserCtxtPtr ctxt = (xmlParserCtxtPtr) data; + + if (((ptrdiff_t) payload) == XML_ATTRIBUTE_CDATA) { + xmlHashRemoveEntry2(ctxt->attsSpecial, fullname, fullattr, NULL); + } +} + +/** + * xmlCleanSpecialAttr: + * @ctxt: an XML parser context + * + * Trim the list of attributes defined to remove all those of type + * CDATA as they are not special. This call should be done when finishing + * to parse the DTD and before starting to parse the document root. + */ +static void +xmlCleanSpecialAttr(xmlParserCtxtPtr ctxt) +{ + if (ctxt->attsSpecial == NULL) + return; + + xmlHashScanFull(ctxt->attsSpecial, xmlCleanSpecialAttrCallback, ctxt); + + if (xmlHashSize(ctxt->attsSpecial) == 0) { + xmlHashFree(ctxt->attsSpecial, NULL); + ctxt->attsSpecial = NULL; + } + return; +} + +/** + * xmlCheckLanguageID: + * @lang: pointer to the string value + * + * Checks that the value conforms to the LanguageID production: + * + * NOTE: this is somewhat deprecated, those productions were removed from + * the XML Second edition. + * + * [33] LanguageID ::= Langcode ('-' Subcode)* + * [34] Langcode ::= ISO639Code | IanaCode | UserCode + * [35] ISO639Code ::= ([a-z] | [A-Z]) ([a-z] | [A-Z]) + * [36] IanaCode ::= ('i' | 'I') '-' ([a-z] | [A-Z])+ + * [37] UserCode ::= ('x' | 'X') '-' ([a-z] | [A-Z])+ + * [38] Subcode ::= ([a-z] | [A-Z])+ + * + * The current REC reference the sucessors of RFC 1766, currently 5646 + * + * http://www.rfc-editor.org/rfc/rfc5646.txt + * langtag = language + * ["-" script] + * ["-" region] + * *("-" variant) + * *("-" extension) + * ["-" privateuse] + * language = 2*3ALPHA ; shortest ISO 639 code + * ["-" extlang] ; sometimes followed by + * ; extended language subtags + * / 4ALPHA ; or reserved for future use + * / 5*8ALPHA ; or registered language subtag + * + * extlang = 3ALPHA ; selected ISO 639 codes + * *2("-" 3ALPHA) ; permanently reserved + * + * script = 4ALPHA ; ISO 15924 code + * + * region = 2ALPHA ; ISO 3166-1 code + * / 3DIGIT ; UN M.49 code + * + * variant = 5*8alphanum ; registered variants + * / (DIGIT 3alphanum) + * + * extension = singleton 1*("-" (2*8alphanum)) + * + * ; Single alphanumerics + * ; "x" reserved for private use + * singleton = DIGIT ; 0 - 9 + * / %x41-57 ; A - W + * / %x59-5A ; Y - Z + * / %x61-77 ; a - w + * / %x79-7A ; y - z + * + * it sounds right to still allow Irregular i-xxx IANA and user codes too + * The parser below doesn't try to cope with extension or privateuse + * that could be added but that's not interoperable anyway + * + * Returns 1 if correct 0 otherwise + **/ +int +xmlCheckLanguageID(const xmlChar * lang) +{ + const xmlChar *cur = lang, *nxt; + + if (cur == NULL) + return (0); + if (((cur[0] == 'i') && (cur[1] == '-')) || + ((cur[0] == 'I') && (cur[1] == '-')) || + ((cur[0] == 'x') && (cur[1] == '-')) || + ((cur[0] == 'X') && (cur[1] == '-'))) { + /* + * Still allow IANA code and user code which were coming + * from the previous version of the XML-1.0 specification + * it's deprecated but we should not fail + */ + cur += 2; + while (((cur[0] >= 'A') && (cur[0] <= 'Z')) || + ((cur[0] >= 'a') && (cur[0] <= 'z'))) + cur++; + return(cur[0] == 0); + } + nxt = cur; + while (((nxt[0] >= 'A') && (nxt[0] <= 'Z')) || + ((nxt[0] >= 'a') && (nxt[0] <= 'z'))) + nxt++; + if (nxt - cur >= 4) { + /* + * Reserved + */ + if ((nxt - cur > 8) || (nxt[0] != 0)) + return(0); + return(1); + } + if (nxt - cur < 2) + return(0); + /* we got an ISO 639 code */ + if (nxt[0] == 0) + return(1); + if (nxt[0] != '-') + return(0); + + nxt++; + cur = nxt; + /* now we can have extlang or script or region or variant */ + if ((nxt[0] >= '0') && (nxt[0] <= '9')) + goto region_m49; + + while (((nxt[0] >= 'A') && (nxt[0] <= 'Z')) || + ((nxt[0] >= 'a') && (nxt[0] <= 'z'))) + nxt++; + if (nxt - cur == 4) + goto script; + if (nxt - cur == 2) + goto region; + if ((nxt - cur >= 5) && (nxt - cur <= 8)) + goto variant; + if (nxt - cur != 3) + return(0); + /* we parsed an extlang */ + if (nxt[0] == 0) + return(1); + if (nxt[0] != '-') + return(0); + + nxt++; + cur = nxt; + /* now we can have script or region or variant */ + if ((nxt[0] >= '0') && (nxt[0] <= '9')) + goto region_m49; + + while (((nxt[0] >= 'A') && (nxt[0] <= 'Z')) || + ((nxt[0] >= 'a') && (nxt[0] <= 'z'))) + nxt++; + if (nxt - cur == 2) + goto region; + if ((nxt - cur >= 5) && (nxt - cur <= 8)) + goto variant; + if (nxt - cur != 4) + return(0); + /* we parsed a script */ +script: + if (nxt[0] == 0) + return(1); + if (nxt[0] != '-') + return(0); + + nxt++; + cur = nxt; + /* now we can have region or variant */ + if ((nxt[0] >= '0') && (nxt[0] <= '9')) + goto region_m49; + + while (((nxt[0] >= 'A') && (nxt[0] <= 'Z')) || + ((nxt[0] >= 'a') && (nxt[0] <= 'z'))) + nxt++; + + if ((nxt - cur >= 5) && (nxt - cur <= 8)) + goto variant; + if (nxt - cur != 2) + return(0); + /* we parsed a region */ +region: + if (nxt[0] == 0) + return(1); + if (nxt[0] != '-') + return(0); + + nxt++; + cur = nxt; + /* now we can just have a variant */ + while (((nxt[0] >= 'A') && (nxt[0] <= 'Z')) || + ((nxt[0] >= 'a') && (nxt[0] <= 'z'))) + nxt++; + + if ((nxt - cur < 5) || (nxt - cur > 8)) + return(0); + + /* we parsed a variant */ +variant: + if (nxt[0] == 0) + return(1); + if (nxt[0] != '-') + return(0); + /* extensions and private use subtags not checked */ + return (1); + +region_m49: + if (((nxt[1] >= '0') && (nxt[1] <= '9')) && + ((nxt[2] >= '0') && (nxt[2] <= '9'))) { + nxt += 3; + goto region; + } + return(0); +} + +/************************************************************************ + * * + * Parser stacks related functions and macros * + * * + ************************************************************************/ + +static xmlEntityPtr xmlParseStringEntityRef(xmlParserCtxtPtr ctxt, + const xmlChar ** str); + +#ifdef SAX2 +/** + * nsPush: + * @ctxt: an XML parser context + * @prefix: the namespace prefix or NULL + * @URL: the namespace name + * + * Pushes a new parser namespace on top of the ns stack + * + * Returns -1 in case of error, -2 if the namespace should be discarded + * and the index in the stack otherwise. + */ +static int +nsPush(xmlParserCtxtPtr ctxt, const xmlChar *prefix, const xmlChar *URL) +{ + if (ctxt->options & XML_PARSE_NSCLEAN) { + int i; + for (i = ctxt->nsNr - 2;i >= 0;i -= 2) { + if (ctxt->nsTab[i] == prefix) { + /* in scope */ + if (ctxt->nsTab[i + 1] == URL) + return(-2); + /* out of scope keep it */ + break; + } + } + } + if ((ctxt->nsMax == 0) || (ctxt->nsTab == NULL)) { + ctxt->nsMax = 10; + ctxt->nsNr = 0; + ctxt->nsTab = (const xmlChar **) + xmlMalloc(ctxt->nsMax * sizeof(xmlChar *)); + if (ctxt->nsTab == NULL) { + xmlErrMemory(ctxt, NULL); + ctxt->nsMax = 0; + return (-1); + } + } else if (ctxt->nsNr >= ctxt->nsMax) { + const xmlChar ** tmp; + ctxt->nsMax *= 2; + tmp = (const xmlChar **) xmlRealloc((char *) ctxt->nsTab, + ctxt->nsMax * sizeof(ctxt->nsTab[0])); + if (tmp == NULL) { + xmlErrMemory(ctxt, NULL); + ctxt->nsMax /= 2; + return (-1); + } + ctxt->nsTab = tmp; + } + ctxt->nsTab[ctxt->nsNr++] = prefix; + ctxt->nsTab[ctxt->nsNr++] = URL; + return (ctxt->nsNr); +} +/** + * nsPop: + * @ctxt: an XML parser context + * @nr: the number to pop + * + * Pops the top @nr parser prefix/namespace from the ns stack + * + * Returns the number of namespaces removed + */ +static int +nsPop(xmlParserCtxtPtr ctxt, int nr) +{ + int i; + + if (ctxt->nsTab == NULL) return(0); + if (ctxt->nsNr < nr) { + xmlGenericError(xmlGenericErrorContext, "Pbm popping %d NS\n", nr); + nr = ctxt->nsNr; + } + if (ctxt->nsNr <= 0) + return (0); + + for (i = 0;i < nr;i++) { + ctxt->nsNr--; + ctxt->nsTab[ctxt->nsNr] = NULL; + } + return(nr); +} +#endif + +static int +xmlCtxtGrowAttrs(xmlParserCtxtPtr ctxt, int nr) { + const xmlChar **atts; + int *attallocs; + int maxatts; + + if (ctxt->atts == NULL) { + maxatts = 55; /* allow for 10 attrs by default */ + atts = (const xmlChar **) + xmlMalloc(maxatts * sizeof(xmlChar *)); + if (atts == NULL) goto mem_error; + ctxt->atts = atts; + attallocs = (int *) xmlMalloc((maxatts / 5) * sizeof(int)); + if (attallocs == NULL) goto mem_error; + ctxt->attallocs = attallocs; + ctxt->maxatts = maxatts; + } else if (nr + 5 > ctxt->maxatts) { + maxatts = (nr + 5) * 2; + atts = (const xmlChar **) xmlRealloc((void *) ctxt->atts, + maxatts * sizeof(const xmlChar *)); + if (atts == NULL) goto mem_error; + ctxt->atts = atts; + attallocs = (int *) xmlRealloc((void *) ctxt->attallocs, + (maxatts / 5) * sizeof(int)); + if (attallocs == NULL) goto mem_error; + ctxt->attallocs = attallocs; + ctxt->maxatts = maxatts; + } + return(ctxt->maxatts); +mem_error: + xmlErrMemory(ctxt, NULL); + return(-1); +} + +/** + * inputPush: + * @ctxt: an XML parser context + * @value: the parser input + * + * Pushes a new parser input on top of the input stack + * + * Returns -1 in case of error, the index in the stack otherwise + */ +int +inputPush(xmlParserCtxtPtr ctxt, xmlParserInputPtr value) +{ + if ((ctxt == NULL) || (value == NULL)) + return(-1); + if (ctxt->inputNr >= ctxt->inputMax) { + ctxt->inputMax *= 2; + ctxt->inputTab = + (xmlParserInputPtr *) xmlRealloc(ctxt->inputTab, + ctxt->inputMax * + sizeof(ctxt->inputTab[0])); + if (ctxt->inputTab == NULL) { + xmlErrMemory(ctxt, NULL); + xmlFreeInputStream(value); + ctxt->inputMax /= 2; + value = NULL; + return (-1); + } + } + ctxt->inputTab[ctxt->inputNr] = value; + ctxt->input = value; + return (ctxt->inputNr++); +} +/** + * inputPop: + * @ctxt: an XML parser context + * + * Pops the top parser input from the input stack + * + * Returns the input just removed + */ +xmlParserInputPtr +inputPop(xmlParserCtxtPtr ctxt) +{ + xmlParserInputPtr ret; + + if (ctxt == NULL) + return(NULL); + if (ctxt->inputNr <= 0) + return (NULL); + ctxt->inputNr--; + if (ctxt->inputNr > 0) + ctxt->input = ctxt->inputTab[ctxt->inputNr - 1]; + else + ctxt->input = NULL; + ret = ctxt->inputTab[ctxt->inputNr]; + ctxt->inputTab[ctxt->inputNr] = NULL; + return (ret); +} +/** + * nodePush: + * @ctxt: an XML parser context + * @value: the element node + * + * Pushes a new element node on top of the node stack + * + * Returns -1 in case of error, the index in the stack otherwise + */ +int +nodePush(xmlParserCtxtPtr ctxt, xmlNodePtr value) +{ + if (ctxt == NULL) return(0); + if (ctxt->nodeNr >= ctxt->nodeMax) { + xmlNodePtr *tmp; + + tmp = (xmlNodePtr *) xmlRealloc(ctxt->nodeTab, + ctxt->nodeMax * 2 * + sizeof(ctxt->nodeTab[0])); + if (tmp == NULL) { + xmlErrMemory(ctxt, NULL); + return (-1); + } + ctxt->nodeTab = tmp; + ctxt->nodeMax *= 2; + } + if ((((unsigned int) ctxt->nodeNr) > xmlParserMaxDepth) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErrMsgInt(ctxt, XML_ERR_INTERNAL_ERROR, + "Excessive depth in document: %d use XML_PARSE_HUGE option\n", + xmlParserMaxDepth); + xmlHaltParser(ctxt); + return(-1); + } + ctxt->nodeTab[ctxt->nodeNr] = value; + ctxt->node = value; + return (ctxt->nodeNr++); +} + +/** + * nodePop: + * @ctxt: an XML parser context + * + * Pops the top element node from the node stack + * + * Returns the node just removed + */ +xmlNodePtr +nodePop(xmlParserCtxtPtr ctxt) +{ + xmlNodePtr ret; + + if (ctxt == NULL) return(NULL); + if (ctxt->nodeNr <= 0) + return (NULL); + ctxt->nodeNr--; + if (ctxt->nodeNr > 0) + ctxt->node = ctxt->nodeTab[ctxt->nodeNr - 1]; + else + ctxt->node = NULL; + ret = ctxt->nodeTab[ctxt->nodeNr]; + ctxt->nodeTab[ctxt->nodeNr] = NULL; + return (ret); +} + +#ifdef LIBXML_PUSH_ENABLED +/** + * nameNsPush: + * @ctxt: an XML parser context + * @value: the element name + * @prefix: the element prefix + * @URI: the element namespace name + * + * Pushes a new element name/prefix/URL on top of the name stack + * + * Returns -1 in case of error, the index in the stack otherwise + */ +static int +nameNsPush(xmlParserCtxtPtr ctxt, const xmlChar * value, + const xmlChar *prefix, const xmlChar *URI, int nsNr) +{ + if (ctxt->nameNr >= ctxt->nameMax) { + const xmlChar * *tmp; + void **tmp2; + ctxt->nameMax *= 2; + tmp = (const xmlChar * *) xmlRealloc((xmlChar * *)ctxt->nameTab, + ctxt->nameMax * + sizeof(ctxt->nameTab[0])); + if (tmp == NULL) { + ctxt->nameMax /= 2; + goto mem_error; + } + ctxt->nameTab = tmp; + tmp2 = (void **) xmlRealloc((void * *)ctxt->pushTab, + ctxt->nameMax * 3 * + sizeof(ctxt->pushTab[0])); + if (tmp2 == NULL) { + ctxt->nameMax /= 2; + goto mem_error; + } + ctxt->pushTab = tmp2; + } + ctxt->nameTab[ctxt->nameNr] = value; + ctxt->name = value; + ctxt->pushTab[ctxt->nameNr * 3] = (void *) prefix; + ctxt->pushTab[ctxt->nameNr * 3 + 1] = (void *) URI; + ctxt->pushTab[ctxt->nameNr * 3 + 2] = (void *) (ptrdiff_t) nsNr; + return (ctxt->nameNr++); +mem_error: + xmlErrMemory(ctxt, NULL); + return (-1); +} +/** + * nameNsPop: + * @ctxt: an XML parser context + * + * Pops the top element/prefix/URI name from the name stack + * + * Returns the name just removed + */ +static const xmlChar * +nameNsPop(xmlParserCtxtPtr ctxt) +{ + const xmlChar *ret; + + if (ctxt->nameNr <= 0) + return (NULL); + ctxt->nameNr--; + if (ctxt->nameNr > 0) + ctxt->name = ctxt->nameTab[ctxt->nameNr - 1]; + else + ctxt->name = NULL; + ret = ctxt->nameTab[ctxt->nameNr]; + ctxt->nameTab[ctxt->nameNr] = NULL; + return (ret); +} +#endif /* LIBXML_PUSH_ENABLED */ + +/** + * namePush: + * @ctxt: an XML parser context + * @value: the element name + * + * Pushes a new element name on top of the name stack + * + * Returns -1 in case of error, the index in the stack otherwise + */ +int +namePush(xmlParserCtxtPtr ctxt, const xmlChar * value) +{ + if (ctxt == NULL) return (-1); + + if (ctxt->nameNr >= ctxt->nameMax) { + const xmlChar * *tmp; + tmp = (const xmlChar * *) xmlRealloc((xmlChar * *)ctxt->nameTab, + ctxt->nameMax * 2 * + sizeof(ctxt->nameTab[0])); + if (tmp == NULL) { + goto mem_error; + } + ctxt->nameTab = tmp; + ctxt->nameMax *= 2; + } + ctxt->nameTab[ctxt->nameNr] = value; + ctxt->name = value; + return (ctxt->nameNr++); +mem_error: + xmlErrMemory(ctxt, NULL); + return (-1); +} +/** + * namePop: + * @ctxt: an XML parser context + * + * Pops the top element name from the name stack + * + * Returns the name just removed + */ +const xmlChar * +namePop(xmlParserCtxtPtr ctxt) +{ + const xmlChar *ret; + + if ((ctxt == NULL) || (ctxt->nameNr <= 0)) + return (NULL); + ctxt->nameNr--; + if (ctxt->nameNr > 0) + ctxt->name = ctxt->nameTab[ctxt->nameNr - 1]; + else + ctxt->name = NULL; + ret = ctxt->nameTab[ctxt->nameNr]; + ctxt->nameTab[ctxt->nameNr] = NULL; + return (ret); +} + +static int spacePush(xmlParserCtxtPtr ctxt, int val) { + if (ctxt->spaceNr >= ctxt->spaceMax) { + int *tmp; + + ctxt->spaceMax *= 2; + tmp = (int *) xmlRealloc(ctxt->spaceTab, + ctxt->spaceMax * sizeof(ctxt->spaceTab[0])); + if (tmp == NULL) { + xmlErrMemory(ctxt, NULL); + ctxt->spaceMax /=2; + return(-1); + } + ctxt->spaceTab = tmp; + } + ctxt->spaceTab[ctxt->spaceNr] = val; + ctxt->space = &ctxt->spaceTab[ctxt->spaceNr]; + return(ctxt->spaceNr++); +} + +static int spacePop(xmlParserCtxtPtr ctxt) { + int ret; + if (ctxt->spaceNr <= 0) return(0); + ctxt->spaceNr--; + if (ctxt->spaceNr > 0) + ctxt->space = &ctxt->spaceTab[ctxt->spaceNr - 1]; + else + ctxt->space = &ctxt->spaceTab[0]; + ret = ctxt->spaceTab[ctxt->spaceNr]; + ctxt->spaceTab[ctxt->spaceNr] = -1; + return(ret); +} + +/* + * Macros for accessing the content. Those should be used only by the parser, + * and not exported. + * + * Dirty macros, i.e. one often need to make assumption on the context to + * use them + * + * CUR_PTR return the current pointer to the xmlChar to be parsed. + * To be used with extreme caution since operations consuming + * characters may move the input buffer to a different location ! + * CUR returns the current xmlChar value, i.e. a 8 bit value if compiled + * This should be used internally by the parser + * only to compare to ASCII values otherwise it would break when + * running with UTF-8 encoding. + * RAW same as CUR but in the input buffer, bypass any token + * extraction that may have been done + * NXT(n) returns the n'th next xmlChar. Same as CUR is should be used only + * to compare on ASCII based substring. + * SKIP(n) Skip n xmlChar, and must also be used only to skip ASCII defined + * strings without newlines within the parser. + * NEXT1(l) Skip 1 xmlChar, and must also be used only to skip 1 non-newline ASCII + * defined char within the parser. + * Clean macros, not dependent of an ASCII context, expect UTF-8 encoding + * + * NEXT Skip to the next character, this does the proper decoding + * in UTF-8 mode. It also pop-up unfinished entities on the fly. + * NEXTL(l) Skip the current unicode character of l xmlChars long. + * CUR_CHAR(l) returns the current unicode character (int), set l + * to the number of xmlChars used for the encoding [0-5]. + * CUR_SCHAR same but operate on a string instead of the context + * COPY_BUF copy the current unicode char to the target buffer, increment + * the index + * GROW, SHRINK handling of input buffers + */ + +#define RAW (*ctxt->input->cur) +#define CUR (*ctxt->input->cur) +#define NXT(val) ctxt->input->cur[(val)] +#define CUR_PTR ctxt->input->cur +#define BASE_PTR ctxt->input->base + +#define CMP4( s, c1, c2, c3, c4 ) \ + ( ((unsigned char *) s)[ 0 ] == c1 && ((unsigned char *) s)[ 1 ] == c2 && \ + ((unsigned char *) s)[ 2 ] == c3 && ((unsigned char *) s)[ 3 ] == c4 ) +#define CMP5( s, c1, c2, c3, c4, c5 ) \ + ( CMP4( s, c1, c2, c3, c4 ) && ((unsigned char *) s)[ 4 ] == c5 ) +#define CMP6( s, c1, c2, c3, c4, c5, c6 ) \ + ( CMP5( s, c1, c2, c3, c4, c5 ) && ((unsigned char *) s)[ 5 ] == c6 ) +#define CMP7( s, c1, c2, c3, c4, c5, c6, c7 ) \ + ( CMP6( s, c1, c2, c3, c4, c5, c6 ) && ((unsigned char *) s)[ 6 ] == c7 ) +#define CMP8( s, c1, c2, c3, c4, c5, c6, c7, c8 ) \ + ( CMP7( s, c1, c2, c3, c4, c5, c6, c7 ) && ((unsigned char *) s)[ 7 ] == c8 ) +#define CMP9( s, c1, c2, c3, c4, c5, c6, c7, c8, c9 ) \ + ( CMP8( s, c1, c2, c3, c4, c5, c6, c7, c8 ) && \ + ((unsigned char *) s)[ 8 ] == c9 ) +#define CMP10( s, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 ) \ + ( CMP9( s, c1, c2, c3, c4, c5, c6, c7, c8, c9 ) && \ + ((unsigned char *) s)[ 9 ] == c10 ) + +#define SKIP(val) do { \ + ctxt->nbChars += (val),ctxt->input->cur += (val),ctxt->input->col+=(val); \ + if (*ctxt->input->cur == 0) \ + xmlParserInputGrow(ctxt->input, INPUT_CHUNK); \ + } while (0) + +#define SKIPL(val) do { \ + int skipl; \ + for(skipl=0; skiplinput->cur) == '\n') { \ + ctxt->input->line++; ctxt->input->col = 1; \ + } else ctxt->input->col++; \ + ctxt->nbChars++; \ + ctxt->input->cur++; \ + } \ + if (*ctxt->input->cur == 0) \ + xmlParserInputGrow(ctxt->input, INPUT_CHUNK); \ + } while (0) + +#define SHRINK if ((ctxt->progressive == 0) && \ + (ctxt->input->cur - ctxt->input->base > 2 * INPUT_CHUNK) && \ + (ctxt->input->end - ctxt->input->cur < 2 * INPUT_CHUNK)) \ + xmlSHRINK (ctxt); + +static void xmlSHRINK (xmlParserCtxtPtr ctxt) { + xmlParserInputShrink(ctxt->input); + if (*ctxt->input->cur == 0) + xmlParserInputGrow(ctxt->input, INPUT_CHUNK); +} + +#define GROW if ((ctxt->progressive == 0) && \ + (ctxt->input->end - ctxt->input->cur < INPUT_CHUNK)) \ + xmlGROW (ctxt); + +static void xmlGROW (xmlParserCtxtPtr ctxt) { + unsigned long curEnd = ctxt->input->end - ctxt->input->cur; + unsigned long curBase = ctxt->input->cur - ctxt->input->base; + + if (((curEnd > (unsigned long) XML_MAX_LOOKUP_LIMIT) || + (curBase > (unsigned long) XML_MAX_LOOKUP_LIMIT)) && + ((ctxt->input->buf) && (ctxt->input->buf->readcallback != (xmlInputReadCallback) xmlNop)) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_INTERNAL_ERROR, "Huge input lookup"); + xmlHaltParser(ctxt); + return; + } + xmlParserInputGrow(ctxt->input, INPUT_CHUNK); + if ((ctxt->input->cur > ctxt->input->end) || + (ctxt->input->cur < ctxt->input->base)) { + xmlHaltParser(ctxt); + xmlFatalErr(ctxt, XML_ERR_INTERNAL_ERROR, "cur index out of bound"); + return; + } + if ((ctxt->input->cur != NULL) && (*ctxt->input->cur == 0)) + xmlParserInputGrow(ctxt->input, INPUT_CHUNK); +} + +#define SKIP_BLANKS xmlSkipBlankChars(ctxt) + +#define NEXT xmlNextChar(ctxt) + +#define NEXT1 { \ + ctxt->input->col++; \ + ctxt->input->cur++; \ + ctxt->nbChars++; \ + if (*ctxt->input->cur == 0) \ + xmlParserInputGrow(ctxt->input, INPUT_CHUNK); \ + } + +#define NEXTL(l) do { \ + if (*(ctxt->input->cur) == '\n') { \ + ctxt->input->line++; ctxt->input->col = 1; \ + } else ctxt->input->col++; \ + ctxt->input->cur += l; \ + } while (0) + +#define CUR_CHAR(l) xmlCurrentChar(ctxt, &l) +#define CUR_SCHAR(s, l) xmlStringCurrentChar(ctxt, s, &l) + +#define COPY_BUF(l,b,i,v) \ + if (l == 1) b[i++] = (xmlChar) v; \ + else i += xmlCopyCharMultiByte(&b[i],v) + +/** + * xmlSkipBlankChars: + * @ctxt: the XML parser context + * + * skip all blanks character found at that point in the input streams. + * It pops up finished entities in the process if allowable at that point. + * + * Returns the number of space chars skipped + */ + +int +xmlSkipBlankChars(xmlParserCtxtPtr ctxt) { + int res = 0; + + /* + * It's Okay to use CUR/NEXT here since all the blanks are on + * the ASCII range. + */ + if ((ctxt->inputNr == 1) && (ctxt->instate != XML_PARSER_DTD)) { + const xmlChar *cur; + /* + * if we are in the document content, go really fast + */ + cur = ctxt->input->cur; + while (IS_BLANK_CH(*cur)) { + if (*cur == '\n') { + ctxt->input->line++; ctxt->input->col = 1; + } else { + ctxt->input->col++; + } + cur++; + res++; + if (*cur == 0) { + ctxt->input->cur = cur; + xmlParserInputGrow(ctxt->input, INPUT_CHUNK); + cur = ctxt->input->cur; + } + } + ctxt->input->cur = cur; + } else { + int expandPE = ((ctxt->external != 0) || (ctxt->inputNr != 1)); + + while (1) { + if (IS_BLANK_CH(CUR)) { /* CHECKED tstblanks.xml */ + NEXT; + } else if (CUR == '%') { + /* + * Need to handle support of entities branching here + */ + if ((expandPE == 0) || (IS_BLANK_CH(NXT(1))) || (NXT(1) == 0)) + break; + xmlParsePEReference(ctxt); + } else if (CUR == 0) { + if (ctxt->inputNr <= 1) + break; + xmlPopInput(ctxt); + } else { + break; + } + + /* + * Also increase the counter when entering or exiting a PERef. + * The spec says: "When a parameter-entity reference is recognized + * in the DTD and included, its replacement text MUST be enlarged + * by the attachment of one leading and one following space (#x20) + * character." + */ + res++; + } + } + return(res); +} + +/************************************************************************ + * * + * Commodity functions to handle entities * + * * + ************************************************************************/ + +/** + * xmlPopInput: + * @ctxt: an XML parser context + * + * xmlPopInput: the current input pointed by ctxt->input came to an end + * pop it and return the next char. + * + * Returns the current xmlChar in the parser context + */ +xmlChar +xmlPopInput(xmlParserCtxtPtr ctxt) { + if ((ctxt == NULL) || (ctxt->inputNr <= 1)) return(0); + if (xmlParserDebugEntities) + xmlGenericError(xmlGenericErrorContext, + "Popping input %d\n", ctxt->inputNr); + if ((ctxt->inputNr > 1) && (ctxt->inSubset == 0) && + (ctxt->instate != XML_PARSER_EOF)) + xmlFatalErr(ctxt, XML_ERR_INTERNAL_ERROR, + "Unfinished entity outside the DTD"); + xmlFreeInputStream(inputPop(ctxt)); + if (*ctxt->input->cur == 0) + xmlParserInputGrow(ctxt->input, INPUT_CHUNK); + return(CUR); +} + +/** + * xmlPushInput: + * @ctxt: an XML parser context + * @input: an XML parser input fragment (entity, XML fragment ...). + * + * xmlPushInput: switch to a new input stream which is stacked on top + * of the previous one(s). + * Returns -1 in case of error or the index in the input stack + */ +int +xmlPushInput(xmlParserCtxtPtr ctxt, xmlParserInputPtr input) { + int ret; + if (input == NULL) return(-1); + + if (xmlParserDebugEntities) { + if ((ctxt->input != NULL) && (ctxt->input->filename)) + xmlGenericError(xmlGenericErrorContext, + "%s(%d): ", ctxt->input->filename, + ctxt->input->line); + xmlGenericError(xmlGenericErrorContext, + "Pushing input %d : %.30s\n", ctxt->inputNr+1, input->cur); + } + if (((ctxt->inputNr > 40) && ((ctxt->options & XML_PARSE_HUGE) == 0)) || + (ctxt->inputNr > 1024)) { + xmlFatalErr(ctxt, XML_ERR_ENTITY_LOOP, NULL); + while (ctxt->inputNr > 1) + xmlFreeInputStream(inputPop(ctxt)); + return(-1); + } + ret = inputPush(ctxt, input); + if (ctxt->instate == XML_PARSER_EOF) + return(-1); + GROW; + return(ret); +} + +/** + * xmlParseCharRef: + * @ctxt: an XML parser context + * + * parse Reference declarations + * + * [66] CharRef ::= '&#' [0-9]+ ';' | + * '&#x' [0-9a-fA-F]+ ';' + * + * [ WFC: Legal Character ] + * Characters referred to using character references must match the + * production for Char. + * + * Returns the value parsed (as an int), 0 in case of error + */ +int +xmlParseCharRef(xmlParserCtxtPtr ctxt) { + unsigned int val = 0; + int count = 0; + unsigned int outofrange = 0; + + /* + * Using RAW/CUR/NEXT is okay since we are working on ASCII range here + */ + if ((RAW == '&') && (NXT(1) == '#') && + (NXT(2) == 'x')) { + SKIP(3); + GROW; + while (RAW != ';') { /* loop blocked by count */ + if (count++ > 20) { + count = 0; + GROW; + if (ctxt->instate == XML_PARSER_EOF) + return(0); + } + if ((RAW >= '0') && (RAW <= '9')) + val = val * 16 + (CUR - '0'); + else if ((RAW >= 'a') && (RAW <= 'f') && (count < 20)) + val = val * 16 + (CUR - 'a') + 10; + else if ((RAW >= 'A') && (RAW <= 'F') && (count < 20)) + val = val * 16 + (CUR - 'A') + 10; + else { + xmlFatalErr(ctxt, XML_ERR_INVALID_HEX_CHARREF, NULL); + val = 0; + break; + } + if (val > 0x10FFFF) + outofrange = val; + + NEXT; + count++; + } + if (RAW == ';') { + /* on purpose to avoid reentrancy problems with NEXT and SKIP */ + ctxt->input->col++; + ctxt->nbChars ++; + ctxt->input->cur++; + } + } else if ((RAW == '&') && (NXT(1) == '#')) { + SKIP(2); + GROW; + while (RAW != ';') { /* loop blocked by count */ + if (count++ > 20) { + count = 0; + GROW; + if (ctxt->instate == XML_PARSER_EOF) + return(0); + } + if ((RAW >= '0') && (RAW <= '9')) + val = val * 10 + (CUR - '0'); + else { + xmlFatalErr(ctxt, XML_ERR_INVALID_DEC_CHARREF, NULL); + val = 0; + break; + } + if (val > 0x10FFFF) + outofrange = val; + + NEXT; + count++; + } + if (RAW == ';') { + /* on purpose to avoid reentrancy problems with NEXT and SKIP */ + ctxt->input->col++; + ctxt->nbChars ++; + ctxt->input->cur++; + } + } else { + xmlFatalErr(ctxt, XML_ERR_INVALID_CHARREF, NULL); + } + + /* + * [ WFC: Legal Character ] + * Characters referred to using character references must match the + * production for Char. + */ + if ((IS_CHAR(val) && (outofrange == 0))) { + return(val); + } else { + xmlFatalErrMsgInt(ctxt, XML_ERR_INVALID_CHAR, + "xmlParseCharRef: invalid xmlChar value %d\n", + val); + } + return(0); +} + +/** + * xmlParseStringCharRef: + * @ctxt: an XML parser context + * @str: a pointer to an index in the string + * + * parse Reference declarations, variant parsing from a string rather + * than an an input flow. + * + * [66] CharRef ::= '&#' [0-9]+ ';' | + * '&#x' [0-9a-fA-F]+ ';' + * + * [ WFC: Legal Character ] + * Characters referred to using character references must match the + * production for Char. + * + * Returns the value parsed (as an int), 0 in case of error, str will be + * updated to the current value of the index + */ +static int +xmlParseStringCharRef(xmlParserCtxtPtr ctxt, const xmlChar **str) { + const xmlChar *ptr; + xmlChar cur; + unsigned int val = 0; + unsigned int outofrange = 0; + + if ((str == NULL) || (*str == NULL)) return(0); + ptr = *str; + cur = *ptr; + if ((cur == '&') && (ptr[1] == '#') && (ptr[2] == 'x')) { + ptr += 3; + cur = *ptr; + while (cur != ';') { /* Non input consuming loop */ + if ((cur >= '0') && (cur <= '9')) + val = val * 16 + (cur - '0'); + else if ((cur >= 'a') && (cur <= 'f')) + val = val * 16 + (cur - 'a') + 10; + else if ((cur >= 'A') && (cur <= 'F')) + val = val * 16 + (cur - 'A') + 10; + else { + xmlFatalErr(ctxt, XML_ERR_INVALID_HEX_CHARREF, NULL); + val = 0; + break; + } + if (val > 0x10FFFF) + outofrange = val; + + ptr++; + cur = *ptr; + } + if (cur == ';') + ptr++; + } else if ((cur == '&') && (ptr[1] == '#')){ + ptr += 2; + cur = *ptr; + while (cur != ';') { /* Non input consuming loops */ + if ((cur >= '0') && (cur <= '9')) + val = val * 10 + (cur - '0'); + else { + xmlFatalErr(ctxt, XML_ERR_INVALID_DEC_CHARREF, NULL); + val = 0; + break; + } + if (val > 0x10FFFF) + outofrange = val; + + ptr++; + cur = *ptr; + } + if (cur == ';') + ptr++; + } else { + xmlFatalErr(ctxt, XML_ERR_INVALID_CHARREF, NULL); + return(0); + } + *str = ptr; + + /* + * [ WFC: Legal Character ] + * Characters referred to using character references must match the + * production for Char. + */ + if ((IS_CHAR(val) && (outofrange == 0))) { + return(val); + } else { + xmlFatalErrMsgInt(ctxt, XML_ERR_INVALID_CHAR, + "xmlParseStringCharRef: invalid xmlChar value %d\n", + val); + } + return(0); +} + +/** + * xmlParserHandlePEReference: + * @ctxt: the parser context + * + * [69] PEReference ::= '%' Name ';' + * + * [ WFC: No Recursion ] + * A parsed entity must not contain a recursive + * reference to itself, either directly or indirectly. + * + * [ WFC: Entity Declared ] + * In a document without any DTD, a document with only an internal DTD + * subset which contains no parameter entity references, or a document + * with "standalone='yes'", ... ... The declaration of a parameter + * entity must precede any reference to it... + * + * [ VC: Entity Declared ] + * In a document with an external subset or external parameter entities + * with "standalone='no'", ... ... The declaration of a parameter entity + * must precede any reference to it... + * + * [ WFC: In DTD ] + * Parameter-entity references may only appear in the DTD. + * NOTE: misleading but this is handled. + * + * A PEReference may have been detected in the current input stream + * the handling is done accordingly to + * http://www.w3.org/TR/REC-xml#entproc + * i.e. + * - Included in literal in entity values + * - Included as Parameter Entity reference within DTDs + */ +void +xmlParserHandlePEReference(xmlParserCtxtPtr ctxt) { + switch(ctxt->instate) { + case XML_PARSER_CDATA_SECTION: + return; + case XML_PARSER_COMMENT: + return; + case XML_PARSER_START_TAG: + return; + case XML_PARSER_END_TAG: + return; + case XML_PARSER_EOF: + xmlFatalErr(ctxt, XML_ERR_PEREF_AT_EOF, NULL); + return; + case XML_PARSER_PROLOG: + case XML_PARSER_START: + case XML_PARSER_MISC: + xmlFatalErr(ctxt, XML_ERR_PEREF_IN_PROLOG, NULL); + return; + case XML_PARSER_ENTITY_DECL: + case XML_PARSER_CONTENT: + case XML_PARSER_ATTRIBUTE_VALUE: + case XML_PARSER_PI: + case XML_PARSER_SYSTEM_LITERAL: + case XML_PARSER_PUBLIC_LITERAL: + /* we just ignore it there */ + return; + case XML_PARSER_EPILOG: + xmlFatalErr(ctxt, XML_ERR_PEREF_IN_EPILOG, NULL); + return; + case XML_PARSER_ENTITY_VALUE: + /* + * NOTE: in the case of entity values, we don't do the + * substitution here since we need the literal + * entity value to be able to save the internal + * subset of the document. + * This will be handled by xmlStringDecodeEntities + */ + return; + case XML_PARSER_DTD: + /* + * [WFC: Well-Formedness Constraint: PEs in Internal Subset] + * In the internal DTD subset, parameter-entity references + * can occur only where markup declarations can occur, not + * within markup declarations. + * In that case this is handled in xmlParseMarkupDecl + */ + if ((ctxt->external == 0) && (ctxt->inputNr == 1)) + return; + if (IS_BLANK_CH(NXT(1)) || NXT(1) == 0) + return; + break; + case XML_PARSER_IGNORE: + return; + } + + xmlParsePEReference(ctxt); +} + +/* + * Macro used to grow the current buffer. + * buffer##_size is expected to be a size_t + * mem_error: is expected to handle memory allocation failures + */ +#define growBuffer(buffer, n) { \ + xmlChar *tmp; \ + size_t new_size = buffer##_size * 2 + n; \ + if (new_size < buffer##_size) goto mem_error; \ + tmp = (xmlChar *) xmlRealloc(buffer, new_size); \ + if (tmp == NULL) goto mem_error; \ + buffer = tmp; \ + buffer##_size = new_size; \ +} + +/** + * xmlStringLenDecodeEntities: + * @ctxt: the parser context + * @str: the input string + * @len: the string length + * @what: combination of XML_SUBSTITUTE_REF and XML_SUBSTITUTE_PEREF + * @end: an end marker xmlChar, 0 if none + * @end2: an end marker xmlChar, 0 if none + * @end3: an end marker xmlChar, 0 if none + * + * Takes a entity string content and process to do the adequate substitutions. + * + * [67] Reference ::= EntityRef | CharRef + * + * [69] PEReference ::= '%' Name ';' + * + * Returns A newly allocated string with the substitution done. The caller + * must deallocate it ! + */ +xmlChar * +xmlStringLenDecodeEntities(xmlParserCtxtPtr ctxt, const xmlChar *str, int len, + int what, xmlChar end, xmlChar end2, xmlChar end3) { + xmlChar *buffer = NULL; + size_t buffer_size = 0; + size_t nbchars = 0; + + xmlChar *current = NULL; + xmlChar *rep = NULL; + const xmlChar *last; + xmlEntityPtr ent; + int c,l; + + if ((ctxt == NULL) || (str == NULL) || (len < 0)) + return(NULL); + last = str + len; + + if (((ctxt->depth > 40) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) || + (ctxt->depth > 1024)) { + xmlFatalErr(ctxt, XML_ERR_ENTITY_LOOP, NULL); + return(NULL); + } + + /* + * allocate a translation buffer. + */ + buffer_size = XML_PARSER_BIG_BUFFER_SIZE; + buffer = (xmlChar *) xmlMallocAtomic(buffer_size); + if (buffer == NULL) goto mem_error; + + /* + * OK loop until we reach one of the ending char or a size limit. + * we are operating on already parsed values. + */ + if (str < last) + c = CUR_SCHAR(str, l); + else + c = 0; + while ((c != 0) && (c != end) && /* non input consuming loop */ + (c != end2) && (c != end3)) { + + if (c == 0) break; + if ((c == '&') && (str[1] == '#')) { + int val = xmlParseStringCharRef(ctxt, &str); + if (val == 0) + goto int_error; + COPY_BUF(0,buffer,nbchars,val); + if (nbchars + XML_PARSER_BUFFER_SIZE > buffer_size) { + growBuffer(buffer, XML_PARSER_BUFFER_SIZE); + } + } else if ((c == '&') && (what & XML_SUBSTITUTE_REF)) { + if (xmlParserDebugEntities) + xmlGenericError(xmlGenericErrorContext, + "String decoding Entity Reference: %.30s\n", + str); + ent = xmlParseStringEntityRef(ctxt, &str); + xmlParserEntityCheck(ctxt, 0, ent, 0); + if (ent != NULL) + ctxt->nbentities += ent->checked / 2; + if ((ent != NULL) && + (ent->etype == XML_INTERNAL_PREDEFINED_ENTITY)) { + if (ent->content != NULL) { + COPY_BUF(0,buffer,nbchars,ent->content[0]); + if (nbchars + XML_PARSER_BUFFER_SIZE > buffer_size) { + growBuffer(buffer, XML_PARSER_BUFFER_SIZE); + } + } else { + xmlFatalErrMsg(ctxt, XML_ERR_INTERNAL_ERROR, + "predefined entity has no content\n"); + goto int_error; + } + } else if ((ent != NULL) && (ent->content != NULL)) { + ctxt->depth++; + rep = xmlStringDecodeEntities(ctxt, ent->content, what, + 0, 0, 0); + ctxt->depth--; + if (rep == NULL) + goto int_error; + + current = rep; + while (*current != 0) { /* non input consuming loop */ + buffer[nbchars++] = *current++; + if (nbchars + XML_PARSER_BUFFER_SIZE > buffer_size) { + if (xmlParserEntityCheck(ctxt, nbchars, ent, 0)) + goto int_error; + growBuffer(buffer, XML_PARSER_BUFFER_SIZE); + } + } + xmlFree(rep); + rep = NULL; + } else if (ent != NULL) { + int i = xmlStrlen(ent->name); + const xmlChar *cur = ent->name; + + buffer[nbchars++] = '&'; + if (nbchars + i + XML_PARSER_BUFFER_SIZE > buffer_size) { + growBuffer(buffer, i + XML_PARSER_BUFFER_SIZE); + } + for (;i > 0;i--) + buffer[nbchars++] = *cur++; + buffer[nbchars++] = ';'; + } + } else if (c == '%' && (what & XML_SUBSTITUTE_PEREF)) { + if (xmlParserDebugEntities) + xmlGenericError(xmlGenericErrorContext, + "String decoding PE Reference: %.30s\n", str); + ent = xmlParseStringPEReference(ctxt, &str); + xmlParserEntityCheck(ctxt, 0, ent, 0); + if (ent != NULL) + ctxt->nbentities += ent->checked / 2; + if (ent != NULL) { + if (ent->content == NULL) { + /* + * Note: external parsed entities will not be loaded, + * it is not required for a non-validating parser to + * complete external PEreferences coming from the + * internal subset + */ + if (((ctxt->options & XML_PARSE_NOENT) != 0) || + ((ctxt->options & XML_PARSE_DTDVALID) != 0) || + (ctxt->validate != 0)) { + xmlLoadEntityContent(ctxt, ent); + } else { + xmlWarningMsg(ctxt, XML_ERR_ENTITY_PROCESSING, + "not validating will not read content for PE entity %s\n", + ent->name, NULL); + } + } + ctxt->depth++; + rep = xmlStringDecodeEntities(ctxt, ent->content, what, + 0, 0, 0); + ctxt->depth--; + if (rep == NULL) + goto int_error; + current = rep; + while (*current != 0) { /* non input consuming loop */ + buffer[nbchars++] = *current++; + if (nbchars + XML_PARSER_BUFFER_SIZE > buffer_size) { + if (xmlParserEntityCheck(ctxt, nbchars, ent, 0)) + goto int_error; + growBuffer(buffer, XML_PARSER_BUFFER_SIZE); + } + } + xmlFree(rep); + rep = NULL; + } + } else { + COPY_BUF(l,buffer,nbchars,c); + str += l; + if (nbchars + XML_PARSER_BUFFER_SIZE > buffer_size) { + growBuffer(buffer, XML_PARSER_BUFFER_SIZE); + } + } + if (str < last) + c = CUR_SCHAR(str, l); + else + c = 0; + } + buffer[nbchars] = 0; + return(buffer); + +mem_error: + xmlErrMemory(ctxt, NULL); +int_error: + if (rep != NULL) + xmlFree(rep); + if (buffer != NULL) + xmlFree(buffer); + return(NULL); +} + +/** + * xmlStringDecodeEntities: + * @ctxt: the parser context + * @str: the input string + * @what: combination of XML_SUBSTITUTE_REF and XML_SUBSTITUTE_PEREF + * @end: an end marker xmlChar, 0 if none + * @end2: an end marker xmlChar, 0 if none + * @end3: an end marker xmlChar, 0 if none + * + * Takes a entity string content and process to do the adequate substitutions. + * + * [67] Reference ::= EntityRef | CharRef + * + * [69] PEReference ::= '%' Name ';' + * + * Returns A newly allocated string with the substitution done. The caller + * must deallocate it ! + */ +xmlChar * +xmlStringDecodeEntities(xmlParserCtxtPtr ctxt, const xmlChar *str, int what, + xmlChar end, xmlChar end2, xmlChar end3) { + if ((ctxt == NULL) || (str == NULL)) return(NULL); + return(xmlStringLenDecodeEntities(ctxt, str, xmlStrlen(str), what, + end, end2, end3)); +} + +/************************************************************************ + * * + * Commodity functions, cleanup needed ? * + * * + ************************************************************************/ + +/** + * areBlanks: + * @ctxt: an XML parser context + * @str: a xmlChar * + * @len: the size of @str + * @blank_chars: we know the chars are blanks + * + * Is this a sequence of blank chars that one can ignore ? + * + * Returns 1 if ignorable 0 otherwise. + */ + +static int areBlanks(xmlParserCtxtPtr ctxt, const xmlChar *str, int len, + int blank_chars) { + int i, ret; + xmlNodePtr lastChild; + + /* + * Don't spend time trying to differentiate them, the same callback is + * used ! + */ + if (ctxt->sax->ignorableWhitespace == ctxt->sax->characters) + return(0); + + /* + * Check for xml:space value. + */ + if ((ctxt->space == NULL) || (*(ctxt->space) == 1) || + (*(ctxt->space) == -2)) + return(0); + + /* + * Check that the string is made of blanks + */ + if (blank_chars == 0) { + for (i = 0;i < len;i++) + if (!(IS_BLANK_CH(str[i]))) return(0); + } + + /* + * Look if the element is mixed content in the DTD if available + */ + if (ctxt->node == NULL) return(0); + if (ctxt->myDoc != NULL) { + ret = xmlIsMixedElement(ctxt->myDoc, ctxt->node->name); + if (ret == 0) return(1); + if (ret == 1) return(0); + } + + /* + * Otherwise, heuristic :-\ + */ + if ((RAW != '<') && (RAW != 0xD)) return(0); + if ((ctxt->node->children == NULL) && + (RAW == '<') && (NXT(1) == '/')) return(0); + + lastChild = xmlGetLastChild(ctxt->node); + if (lastChild == NULL) { + if ((ctxt->node->type != XML_ELEMENT_NODE) && + (ctxt->node->content != NULL)) return(0); + } else if (xmlNodeIsText(lastChild)) + return(0); + else if ((ctxt->node->children != NULL) && + (xmlNodeIsText(ctxt->node->children))) + return(0); + return(1); +} + +/************************************************************************ + * * + * Extra stuff for namespace support * + * Relates to http://www.w3.org/TR/WD-xml-names * + * * + ************************************************************************/ + +/** + * xmlSplitQName: + * @ctxt: an XML parser context + * @name: an XML parser context + * @prefix: a xmlChar ** + * + * parse an UTF8 encoded XML qualified name string + * + * [NS 5] QName ::= (Prefix ':')? LocalPart + * + * [NS 6] Prefix ::= NCName + * + * [NS 7] LocalPart ::= NCName + * + * Returns the local part, and prefix is updated + * to get the Prefix if any. + */ + +xmlChar * +xmlSplitQName(xmlParserCtxtPtr ctxt, const xmlChar *name, xmlChar **prefix) { + xmlChar buf[XML_MAX_NAMELEN + 5]; + xmlChar *buffer = NULL; + int len = 0; + int max = XML_MAX_NAMELEN; + xmlChar *ret = NULL; + const xmlChar *cur = name; + int c; + + if (prefix == NULL) return(NULL); + *prefix = NULL; + + if (cur == NULL) return(NULL); + +#ifndef XML_XML_NAMESPACE + /* xml: prefix is not really a namespace */ + if ((cur[0] == 'x') && (cur[1] == 'm') && + (cur[2] == 'l') && (cur[3] == ':')) + return(xmlStrdup(name)); +#endif + + /* nasty but well=formed */ + if (cur[0] == ':') + return(xmlStrdup(name)); + + c = *cur++; + while ((c != 0) && (c != ':') && (len < max)) { /* tested bigname.xml */ + buf[len++] = c; + c = *cur++; + } + if (len >= max) { + /* + * Okay someone managed to make a huge name, so he's ready to pay + * for the processing speed. + */ + max = len * 2; + + buffer = (xmlChar *) xmlMallocAtomic(max * sizeof(xmlChar)); + if (buffer == NULL) { + xmlErrMemory(ctxt, NULL); + return(NULL); + } + memcpy(buffer, buf, len); + while ((c != 0) && (c != ':')) { /* tested bigname.xml */ + if (len + 10 > max) { + xmlChar *tmp; + + max *= 2; + tmp = (xmlChar *) xmlRealloc(buffer, + max * sizeof(xmlChar)); + if (tmp == NULL) { + xmlFree(buffer); + xmlErrMemory(ctxt, NULL); + return(NULL); + } + buffer = tmp; + } + buffer[len++] = c; + c = *cur++; + } + buffer[len] = 0; + } + + if ((c == ':') && (*cur == 0)) { + if (buffer != NULL) + xmlFree(buffer); + *prefix = NULL; + return(xmlStrdup(name)); + } + + if (buffer == NULL) + ret = xmlStrndup(buf, len); + else { + ret = buffer; + buffer = NULL; + max = XML_MAX_NAMELEN; + } + + + if (c == ':') { + c = *cur; + *prefix = ret; + if (c == 0) { + return(xmlStrndup(BAD_CAST "", 0)); + } + len = 0; + + /* + * Check that the first character is proper to start + * a new name + */ + if (!(((c >= 0x61) && (c <= 0x7A)) || + ((c >= 0x41) && (c <= 0x5A)) || + (c == '_') || (c == ':'))) { + int l; + int first = CUR_SCHAR(cur, l); + + if (!IS_LETTER(first) && (first != '_')) { + xmlFatalErrMsgStr(ctxt, XML_NS_ERR_QNAME, + "Name %s is not XML Namespace compliant\n", + name); + } + } + cur++; + + while ((c != 0) && (len < max)) { /* tested bigname2.xml */ + buf[len++] = c; + c = *cur++; + } + if (len >= max) { + /* + * Okay someone managed to make a huge name, so he's ready to pay + * for the processing speed. + */ + max = len * 2; + + buffer = (xmlChar *) xmlMallocAtomic(max * sizeof(xmlChar)); + if (buffer == NULL) { + xmlErrMemory(ctxt, NULL); + return(NULL); + } + memcpy(buffer, buf, len); + while (c != 0) { /* tested bigname2.xml */ + if (len + 10 > max) { + xmlChar *tmp; + + max *= 2; + tmp = (xmlChar *) xmlRealloc(buffer, + max * sizeof(xmlChar)); + if (tmp == NULL) { + xmlErrMemory(ctxt, NULL); + xmlFree(buffer); + return(NULL); + } + buffer = tmp; + } + buffer[len++] = c; + c = *cur++; + } + buffer[len] = 0; + } + + if (buffer == NULL) + ret = xmlStrndup(buf, len); + else { + ret = buffer; + } + } + + return(ret); +} + +/************************************************************************ + * * + * The parser itself * + * Relates to http://www.w3.org/TR/REC-xml * + * * + ************************************************************************/ + +/************************************************************************ + * * + * Routines to parse Name, NCName and NmToken * + * * + ************************************************************************/ +#ifdef DEBUG +static unsigned long nbParseName = 0; +static unsigned long nbParseNmToken = 0; +static unsigned long nbParseNCName = 0; +static unsigned long nbParseNCNameComplex = 0; +static unsigned long nbParseNameComplex = 0; +static unsigned long nbParseStringName = 0; +#endif + +/* + * The two following functions are related to the change of accepted + * characters for Name and NmToken in the Revision 5 of XML-1.0 + * They correspond to the modified production [4] and the new production [4a] + * changes in that revision. Also note that the macros used for the + * productions Letter, Digit, CombiningChar and Extender are not needed + * anymore. + * We still keep compatibility to pre-revision5 parsing semantic if the + * new XML_PARSE_OLD10 option is given to the parser. + */ +static int +xmlIsNameStartChar(xmlParserCtxtPtr ctxt, int c) { + if ((ctxt->options & XML_PARSE_OLD10) == 0) { + /* + * Use the new checks of production [4] [4a] amd [5] of the + * Update 5 of XML-1.0 + */ + if ((c != ' ') && (c != '>') && (c != '/') && /* accelerators */ + (((c >= 'a') && (c <= 'z')) || + ((c >= 'A') && (c <= 'Z')) || + (c == '_') || (c == ':') || + ((c >= 0xC0) && (c <= 0xD6)) || + ((c >= 0xD8) && (c <= 0xF6)) || + ((c >= 0xF8) && (c <= 0x2FF)) || + ((c >= 0x370) && (c <= 0x37D)) || + ((c >= 0x37F) && (c <= 0x1FFF)) || + ((c >= 0x200C) && (c <= 0x200D)) || + ((c >= 0x2070) && (c <= 0x218F)) || + ((c >= 0x2C00) && (c <= 0x2FEF)) || + ((c >= 0x3001) && (c <= 0xD7FF)) || + ((c >= 0xF900) && (c <= 0xFDCF)) || + ((c >= 0xFDF0) && (c <= 0xFFFD)) || + ((c >= 0x10000) && (c <= 0xEFFFF)))) + return(1); + } else { + if (IS_LETTER(c) || (c == '_') || (c == ':')) + return(1); + } + return(0); +} + +static int +xmlIsNameChar(xmlParserCtxtPtr ctxt, int c) { + if ((ctxt->options & XML_PARSE_OLD10) == 0) { + /* + * Use the new checks of production [4] [4a] amd [5] of the + * Update 5 of XML-1.0 + */ + if ((c != ' ') && (c != '>') && (c != '/') && /* accelerators */ + (((c >= 'a') && (c <= 'z')) || + ((c >= 'A') && (c <= 'Z')) || + ((c >= '0') && (c <= '9')) || /* !start */ + (c == '_') || (c == ':') || + (c == '-') || (c == '.') || (c == 0xB7) || /* !start */ + ((c >= 0xC0) && (c <= 0xD6)) || + ((c >= 0xD8) && (c <= 0xF6)) || + ((c >= 0xF8) && (c <= 0x2FF)) || + ((c >= 0x300) && (c <= 0x36F)) || /* !start */ + ((c >= 0x370) && (c <= 0x37D)) || + ((c >= 0x37F) && (c <= 0x1FFF)) || + ((c >= 0x200C) && (c <= 0x200D)) || + ((c >= 0x203F) && (c <= 0x2040)) || /* !start */ + ((c >= 0x2070) && (c <= 0x218F)) || + ((c >= 0x2C00) && (c <= 0x2FEF)) || + ((c >= 0x3001) && (c <= 0xD7FF)) || + ((c >= 0xF900) && (c <= 0xFDCF)) || + ((c >= 0xFDF0) && (c <= 0xFFFD)) || + ((c >= 0x10000) && (c <= 0xEFFFF)))) + return(1); + } else { + if ((IS_LETTER(c)) || (IS_DIGIT(c)) || + (c == '.') || (c == '-') || + (c == '_') || (c == ':') || + (IS_COMBINING(c)) || + (IS_EXTENDER(c))) + return(1); + } + return(0); +} + +static xmlChar * xmlParseAttValueInternal(xmlParserCtxtPtr ctxt, + int *len, int *alloc, int normalize); + +static const xmlChar * +xmlParseNameComplex(xmlParserCtxtPtr ctxt) { + int len = 0, l; + int c; + int count = 0; + +#ifdef DEBUG + nbParseNameComplex++; +#endif + + /* + * Handler for more complex cases + */ + GROW; + if (ctxt->instate == XML_PARSER_EOF) + return(NULL); + c = CUR_CHAR(l); + if ((ctxt->options & XML_PARSE_OLD10) == 0) { + /* + * Use the new checks of production [4] [4a] amd [5] of the + * Update 5 of XML-1.0 + */ + if ((c == ' ') || (c == '>') || (c == '/') || /* accelerators */ + (!(((c >= 'a') && (c <= 'z')) || + ((c >= 'A') && (c <= 'Z')) || + (c == '_') || (c == ':') || + ((c >= 0xC0) && (c <= 0xD6)) || + ((c >= 0xD8) && (c <= 0xF6)) || + ((c >= 0xF8) && (c <= 0x2FF)) || + ((c >= 0x370) && (c <= 0x37D)) || + ((c >= 0x37F) && (c <= 0x1FFF)) || + ((c >= 0x200C) && (c <= 0x200D)) || + ((c >= 0x2070) && (c <= 0x218F)) || + ((c >= 0x2C00) && (c <= 0x2FEF)) || + ((c >= 0x3001) && (c <= 0xD7FF)) || + ((c >= 0xF900) && (c <= 0xFDCF)) || + ((c >= 0xFDF0) && (c <= 0xFFFD)) || + ((c >= 0x10000) && (c <= 0xEFFFF))))) { + return(NULL); + } + len += l; + NEXTL(l); + c = CUR_CHAR(l); + while ((c != ' ') && (c != '>') && (c != '/') && /* accelerators */ + (((c >= 'a') && (c <= 'z')) || + ((c >= 'A') && (c <= 'Z')) || + ((c >= '0') && (c <= '9')) || /* !start */ + (c == '_') || (c == ':') || + (c == '-') || (c == '.') || (c == 0xB7) || /* !start */ + ((c >= 0xC0) && (c <= 0xD6)) || + ((c >= 0xD8) && (c <= 0xF6)) || + ((c >= 0xF8) && (c <= 0x2FF)) || + ((c >= 0x300) && (c <= 0x36F)) || /* !start */ + ((c >= 0x370) && (c <= 0x37D)) || + ((c >= 0x37F) && (c <= 0x1FFF)) || + ((c >= 0x200C) && (c <= 0x200D)) || + ((c >= 0x203F) && (c <= 0x2040)) || /* !start */ + ((c >= 0x2070) && (c <= 0x218F)) || + ((c >= 0x2C00) && (c <= 0x2FEF)) || + ((c >= 0x3001) && (c <= 0xD7FF)) || + ((c >= 0xF900) && (c <= 0xFDCF)) || + ((c >= 0xFDF0) && (c <= 0xFFFD)) || + ((c >= 0x10000) && (c <= 0xEFFFF)) + )) { + if (count++ > XML_PARSER_CHUNK_SIZE) { + count = 0; + GROW; + if (ctxt->instate == XML_PARSER_EOF) + return(NULL); + } + len += l; + NEXTL(l); + c = CUR_CHAR(l); + } + } else { + if ((c == ' ') || (c == '>') || (c == '/') || /* accelerators */ + (!IS_LETTER(c) && (c != '_') && + (c != ':'))) { + return(NULL); + } + len += l; + NEXTL(l); + c = CUR_CHAR(l); + + while ((c != ' ') && (c != '>') && (c != '/') && /* test bigname.xml */ + ((IS_LETTER(c)) || (IS_DIGIT(c)) || + (c == '.') || (c == '-') || + (c == '_') || (c == ':') || + (IS_COMBINING(c)) || + (IS_EXTENDER(c)))) { + if (count++ > XML_PARSER_CHUNK_SIZE) { + count = 0; + GROW; + if (ctxt->instate == XML_PARSER_EOF) + return(NULL); + } + len += l; + NEXTL(l); + c = CUR_CHAR(l); + } + } + if ((len > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "Name"); + return(NULL); + } + if (ctxt->input->cur - ctxt->input->base < len) { + /* + * There were a couple of bugs where PERefs lead to to a change + * of the buffer. Check the buffer size to avoid passing an invalid + * pointer to xmlDictLookup. + */ + xmlFatalErr(ctxt, XML_ERR_INTERNAL_ERROR, + "unexpected change of input buffer"); + return (NULL); + } + if ((*ctxt->input->cur == '\n') && (ctxt->input->cur[-1] == '\r')) + return(xmlDictLookup(ctxt->dict, ctxt->input->cur - (len + 1), len)); + return(xmlDictLookup(ctxt->dict, ctxt->input->cur - len, len)); +} + +/** + * xmlParseName: + * @ctxt: an XML parser context + * + * parse an XML name. + * + * [4] NameChar ::= Letter | Digit | '.' | '-' | '_' | ':' | + * CombiningChar | Extender + * + * [5] Name ::= (Letter | '_' | ':') (NameChar)* + * + * [6] Names ::= Name (#x20 Name)* + * + * Returns the Name parsed or NULL + */ + +const xmlChar * +xmlParseName(xmlParserCtxtPtr ctxt) { + const xmlChar *in; + const xmlChar *ret; + int count = 0; + + GROW; + +#ifdef DEBUG + nbParseName++; +#endif + + /* + * Accelerator for simple ASCII names + */ + in = ctxt->input->cur; + if (((*in >= 0x61) && (*in <= 0x7A)) || + ((*in >= 0x41) && (*in <= 0x5A)) || + (*in == '_') || (*in == ':')) { + in++; + while (((*in >= 0x61) && (*in <= 0x7A)) || + ((*in >= 0x41) && (*in <= 0x5A)) || + ((*in >= 0x30) && (*in <= 0x39)) || + (*in == '_') || (*in == '-') || + (*in == ':') || (*in == '.')) + in++; + if ((*in > 0) && (*in < 0x80)) { + count = in - ctxt->input->cur; + if ((count > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "Name"); + return(NULL); + } + ret = xmlDictLookup(ctxt->dict, ctxt->input->cur, count); + ctxt->input->cur = in; + ctxt->nbChars += count; + ctxt->input->col += count; + if (ret == NULL) + xmlErrMemory(ctxt, NULL); + return(ret); + } + } + /* accelerator for special cases */ + return(xmlParseNameComplex(ctxt)); +} + +static const xmlChar * +xmlParseNCNameComplex(xmlParserCtxtPtr ctxt) { + int len = 0, l; + int c; + int count = 0; + size_t startPosition = 0; + +#ifdef DEBUG + nbParseNCNameComplex++; +#endif + + /* + * Handler for more complex cases + */ + GROW; + startPosition = CUR_PTR - BASE_PTR; + c = CUR_CHAR(l); + if ((c == ' ') || (c == '>') || (c == '/') || /* accelerators */ + (!xmlIsNameStartChar(ctxt, c) || (c == ':'))) { + return(NULL); + } + + while ((c != ' ') && (c != '>') && (c != '/') && /* test bigname.xml */ + (xmlIsNameChar(ctxt, c) && (c != ':'))) { + if (count++ > XML_PARSER_CHUNK_SIZE) { + if ((len > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "NCName"); + return(NULL); + } + count = 0; + GROW; + if (ctxt->instate == XML_PARSER_EOF) + return(NULL); + } + len += l; + NEXTL(l); + c = CUR_CHAR(l); + if (c == 0) { + count = 0; + /* + * when shrinking to extend the buffer we really need to preserve + * the part of the name we already parsed. Hence rolling back + * by current lenght. + */ + ctxt->input->cur -= l; + GROW; + ctxt->input->cur += l; + if (ctxt->instate == XML_PARSER_EOF) + return(NULL); + c = CUR_CHAR(l); + } + } + if ((len > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "NCName"); + return(NULL); + } + return(xmlDictLookup(ctxt->dict, (BASE_PTR + startPosition), len)); +} + +/** + * xmlParseNCName: + * @ctxt: an XML parser context + * @len: length of the string parsed + * + * parse an XML name. + * + * [4NS] NCNameChar ::= Letter | Digit | '.' | '-' | '_' | + * CombiningChar | Extender + * + * [5NS] NCName ::= (Letter | '_') (NCNameChar)* + * + * Returns the Name parsed or NULL + */ + +static const xmlChar * +xmlParseNCName(xmlParserCtxtPtr ctxt) { + const xmlChar *in, *e; + const xmlChar *ret; + int count = 0; + +#ifdef DEBUG + nbParseNCName++; +#endif + + /* + * Accelerator for simple ASCII names + */ + in = ctxt->input->cur; + e = ctxt->input->end; + if ((((*in >= 0x61) && (*in <= 0x7A)) || + ((*in >= 0x41) && (*in <= 0x5A)) || + (*in == '_')) && (in < e)) { + in++; + while ((((*in >= 0x61) && (*in <= 0x7A)) || + ((*in >= 0x41) && (*in <= 0x5A)) || + ((*in >= 0x30) && (*in <= 0x39)) || + (*in == '_') || (*in == '-') || + (*in == '.')) && (in < e)) + in++; + if (in >= e) + goto complex; + if ((*in > 0) && (*in < 0x80)) { + count = in - ctxt->input->cur; + if ((count > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "NCName"); + return(NULL); + } + ret = xmlDictLookup(ctxt->dict, ctxt->input->cur, count); + ctxt->input->cur = in; + ctxt->nbChars += count; + ctxt->input->col += count; + if (ret == NULL) { + xmlErrMemory(ctxt, NULL); + } + return(ret); + } + } +complex: + return(xmlParseNCNameComplex(ctxt)); +} + +/** + * xmlParseNameAndCompare: + * @ctxt: an XML parser context + * + * parse an XML name and compares for match + * (specialized for endtag parsing) + * + * Returns NULL for an illegal name, (xmlChar*) 1 for success + * and the name for mismatch + */ + +static const xmlChar * +xmlParseNameAndCompare(xmlParserCtxtPtr ctxt, xmlChar const *other) { + register const xmlChar *cmp = other; + register const xmlChar *in; + const xmlChar *ret; + + GROW; + if (ctxt->instate == XML_PARSER_EOF) + return(NULL); + + in = ctxt->input->cur; + while (*in != 0 && *in == *cmp) { + ++in; + ++cmp; + ctxt->input->col++; + } + if (*cmp == 0 && (*in == '>' || IS_BLANK_CH (*in))) { + /* success */ + ctxt->input->cur = in; + return (const xmlChar*) 1; + } + /* failure (or end of input buffer), check with full function */ + ret = xmlParseName (ctxt); + /* strings coming from the dictionary direct compare possible */ + if (ret == other) { + return (const xmlChar*) 1; + } + return ret; +} + +/** + * xmlParseStringName: + * @ctxt: an XML parser context + * @str: a pointer to the string pointer (IN/OUT) + * + * parse an XML name. + * + * [4] NameChar ::= Letter | Digit | '.' | '-' | '_' | ':' | + * CombiningChar | Extender + * + * [5] Name ::= (Letter | '_' | ':') (NameChar)* + * + * [6] Names ::= Name (#x20 Name)* + * + * Returns the Name parsed or NULL. The @str pointer + * is updated to the current location in the string. + */ + +static xmlChar * +xmlParseStringName(xmlParserCtxtPtr ctxt, const xmlChar** str) { + xmlChar buf[XML_MAX_NAMELEN + 5]; + const xmlChar *cur = *str; + int len = 0, l; + int c; + +#ifdef DEBUG + nbParseStringName++; +#endif + + c = CUR_SCHAR(cur, l); + if (!xmlIsNameStartChar(ctxt, c)) { + return(NULL); + } + + COPY_BUF(l,buf,len,c); + cur += l; + c = CUR_SCHAR(cur, l); + while (xmlIsNameChar(ctxt, c)) { + COPY_BUF(l,buf,len,c); + cur += l; + c = CUR_SCHAR(cur, l); + if (len >= XML_MAX_NAMELEN) { /* test bigentname.xml */ + /* + * Okay someone managed to make a huge name, so he's ready to pay + * for the processing speed. + */ + xmlChar *buffer; + int max = len * 2; + + buffer = (xmlChar *) xmlMallocAtomic(max * sizeof(xmlChar)); + if (buffer == NULL) { + xmlErrMemory(ctxt, NULL); + return(NULL); + } + memcpy(buffer, buf, len); + while (xmlIsNameChar(ctxt, c)) { + if (len + 10 > max) { + xmlChar *tmp; + + if ((len > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "NCName"); + xmlFree(buffer); + return(NULL); + } + max *= 2; + tmp = (xmlChar *) xmlRealloc(buffer, + max * sizeof(xmlChar)); + if (tmp == NULL) { + xmlErrMemory(ctxt, NULL); + xmlFree(buffer); + return(NULL); + } + buffer = tmp; + } + COPY_BUF(l,buffer,len,c); + cur += l; + c = CUR_SCHAR(cur, l); + } + buffer[len] = 0; + *str = cur; + return(buffer); + } + } + if ((len > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "NCName"); + return(NULL); + } + *str = cur; + return(xmlStrndup(buf, len)); +} + +/** + * xmlParseNmtoken: + * @ctxt: an XML parser context + * + * parse an XML Nmtoken. + * + * [7] Nmtoken ::= (NameChar)+ + * + * [8] Nmtokens ::= Nmtoken (#x20 Nmtoken)* + * + * Returns the Nmtoken parsed or NULL + */ + +xmlChar * +xmlParseNmtoken(xmlParserCtxtPtr ctxt) { + xmlChar buf[XML_MAX_NAMELEN + 5]; + int len = 0, l; + int c; + int count = 0; + +#ifdef DEBUG + nbParseNmToken++; +#endif + + GROW; + if (ctxt->instate == XML_PARSER_EOF) + return(NULL); + c = CUR_CHAR(l); + + while (xmlIsNameChar(ctxt, c)) { + if (count++ > XML_PARSER_CHUNK_SIZE) { + count = 0; + GROW; + } + COPY_BUF(l,buf,len,c); + NEXTL(l); + c = CUR_CHAR(l); + if (c == 0) { + count = 0; + GROW; + if (ctxt->instate == XML_PARSER_EOF) + return(NULL); + c = CUR_CHAR(l); + } + if (len >= XML_MAX_NAMELEN) { + /* + * Okay someone managed to make a huge token, so he's ready to pay + * for the processing speed. + */ + xmlChar *buffer; + int max = len * 2; + + buffer = (xmlChar *) xmlMallocAtomic(max * sizeof(xmlChar)); + if (buffer == NULL) { + xmlErrMemory(ctxt, NULL); + return(NULL); + } + memcpy(buffer, buf, len); + while (xmlIsNameChar(ctxt, c)) { + if (count++ > XML_PARSER_CHUNK_SIZE) { + count = 0; + GROW; + if (ctxt->instate == XML_PARSER_EOF) { + xmlFree(buffer); + return(NULL); + } + } + if (len + 10 > max) { + xmlChar *tmp; + + if ((max > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "NmToken"); + xmlFree(buffer); + return(NULL); + } + max *= 2; + tmp = (xmlChar *) xmlRealloc(buffer, + max * sizeof(xmlChar)); + if (tmp == NULL) { + xmlErrMemory(ctxt, NULL); + xmlFree(buffer); + return(NULL); + } + buffer = tmp; + } + COPY_BUF(l,buffer,len,c); + NEXTL(l); + c = CUR_CHAR(l); + } + buffer[len] = 0; + return(buffer); + } + } + if (len == 0) + return(NULL); + if ((len > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "NmToken"); + return(NULL); + } + return(xmlStrndup(buf, len)); +} + +/** + * xmlParseEntityValue: + * @ctxt: an XML parser context + * @orig: if non-NULL store a copy of the original entity value + * + * parse a value for ENTITY declarations + * + * [9] EntityValue ::= '"' ([^%&"] | PEReference | Reference)* '"' | + * "'" ([^%&'] | PEReference | Reference)* "'" + * + * Returns the EntityValue parsed with reference substituted or NULL + */ + +xmlChar * +xmlParseEntityValue(xmlParserCtxtPtr ctxt, xmlChar **orig) { + xmlChar *buf = NULL; + int len = 0; + int size = XML_PARSER_BUFFER_SIZE; + int c, l; + xmlChar stop; + xmlChar *ret = NULL; + const xmlChar *cur = NULL; + xmlParserInputPtr input; + + if (RAW == '"') stop = '"'; + else if (RAW == '\'') stop = '\''; + else { + xmlFatalErr(ctxt, XML_ERR_ENTITY_NOT_STARTED, NULL); + return(NULL); + } + buf = (xmlChar *) xmlMallocAtomic(size * sizeof(xmlChar)); + if (buf == NULL) { + xmlErrMemory(ctxt, NULL); + return(NULL); + } + + /* + * The content of the entity definition is copied in a buffer. + */ + + ctxt->instate = XML_PARSER_ENTITY_VALUE; + input = ctxt->input; + GROW; + if (ctxt->instate == XML_PARSER_EOF) + goto error; + NEXT; + c = CUR_CHAR(l); + /* + * NOTE: 4.4.5 Included in Literal + * When a parameter entity reference appears in a literal entity + * value, ... a single or double quote character in the replacement + * text is always treated as a normal data character and will not + * terminate the literal. + * In practice it means we stop the loop only when back at parsing + * the initial entity and the quote is found + */ + while (((IS_CHAR(c)) && ((c != stop) || /* checked */ + (ctxt->input != input))) && (ctxt->instate != XML_PARSER_EOF)) { + if (len + 5 >= size) { + xmlChar *tmp; + + size *= 2; + tmp = (xmlChar *) xmlRealloc(buf, size * sizeof(xmlChar)); + if (tmp == NULL) { + xmlErrMemory(ctxt, NULL); + goto error; + } + buf = tmp; + } + COPY_BUF(l,buf,len,c); + NEXTL(l); + + GROW; + c = CUR_CHAR(l); + if (c == 0) { + GROW; + c = CUR_CHAR(l); + } + } + buf[len] = 0; + if (ctxt->instate == XML_PARSER_EOF) + goto error; + if (c != stop) { + xmlFatalErr(ctxt, XML_ERR_ENTITY_NOT_FINISHED, NULL); + goto error; + } + NEXT; + + /* + * Raise problem w.r.t. '&' and '%' being used in non-entities + * reference constructs. Note Charref will be handled in + * xmlStringDecodeEntities() + */ + cur = buf; + while (*cur != 0) { /* non input consuming */ + if ((*cur == '%') || ((*cur == '&') && (cur[1] != '#'))) { + xmlChar *name; + xmlChar tmp = *cur; + int nameOk = 0; + + cur++; + name = xmlParseStringName(ctxt, &cur); + if (name != NULL) { + nameOk = 1; + xmlFree(name); + } + if ((nameOk == 0) || (*cur != ';')) { + xmlFatalErrMsgInt(ctxt, XML_ERR_ENTITY_CHAR_ERROR, + "EntityValue: '%c' forbidden except for entities references\n", + tmp); + goto error; + } + if ((tmp == '%') && (ctxt->inSubset == 1) && + (ctxt->inputNr == 1)) { + xmlFatalErr(ctxt, XML_ERR_ENTITY_PE_INTERNAL, NULL); + goto error; + } + if (*cur == 0) + break; + } + cur++; + } + + /* + * Then PEReference entities are substituted. + * + * NOTE: 4.4.7 Bypassed + * When a general entity reference appears in the EntityValue in + * an entity declaration, it is bypassed and left as is. + * so XML_SUBSTITUTE_REF is not set here. + */ + ++ctxt->depth; + ret = xmlStringDecodeEntities(ctxt, buf, XML_SUBSTITUTE_PEREF, + 0, 0, 0); + --ctxt->depth; + if (orig != NULL) { + *orig = buf; + buf = NULL; + } + +error: + if (buf != NULL) + xmlFree(buf); + return(ret); +} + +/** + * xmlParseAttValueComplex: + * @ctxt: an XML parser context + * @len: the resulting attribute len + * @normalize: wether to apply the inner normalization + * + * parse a value for an attribute, this is the fallback function + * of xmlParseAttValue() when the attribute parsing requires handling + * of non-ASCII characters, or normalization compaction. + * + * Returns the AttValue parsed or NULL. The value has to be freed by the caller. + */ +static xmlChar * +xmlParseAttValueComplex(xmlParserCtxtPtr ctxt, int *attlen, int normalize) { + xmlChar limit = 0; + xmlChar *buf = NULL; + xmlChar *rep = NULL; + size_t len = 0; + size_t buf_size = 0; + int c, l, in_space = 0; + xmlChar *current = NULL; + xmlEntityPtr ent; + + if (NXT(0) == '"') { + ctxt->instate = XML_PARSER_ATTRIBUTE_VALUE; + limit = '"'; + NEXT; + } else if (NXT(0) == '\'') { + limit = '\''; + ctxt->instate = XML_PARSER_ATTRIBUTE_VALUE; + NEXT; + } else { + xmlFatalErr(ctxt, XML_ERR_ATTRIBUTE_NOT_STARTED, NULL); + return(NULL); + } + + /* + * allocate a translation buffer. + */ + buf_size = XML_PARSER_BUFFER_SIZE; + buf = (xmlChar *) xmlMallocAtomic(buf_size); + if (buf == NULL) goto mem_error; + + /* + * OK loop until we reach one of the ending char or a size limit. + */ + c = CUR_CHAR(l); + while (((NXT(0) != limit) && /* checked */ + (IS_CHAR(c)) && (c != '<')) && + (ctxt->instate != XML_PARSER_EOF)) { + /* + * Impose a reasonable limit on attribute size, unless XML_PARSE_HUGE + * special option is given + */ + if ((len > XML_MAX_TEXT_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErrMsg(ctxt, XML_ERR_ATTRIBUTE_NOT_FINISHED, + "AttValue length too long\n"); + goto mem_error; + } + if (c == 0) break; + if (c == '&') { + in_space = 0; + if (NXT(1) == '#') { + int val = xmlParseCharRef(ctxt); + + if (val == '&') { + if (ctxt->replaceEntities) { + if (len + 10 > buf_size) { + growBuffer(buf, 10); + } + buf[len++] = '&'; + } else { + /* + * The reparsing will be done in xmlStringGetNodeList() + * called by the attribute() function in SAX.c + */ + if (len + 10 > buf_size) { + growBuffer(buf, 10); + } + buf[len++] = '&'; + buf[len++] = '#'; + buf[len++] = '3'; + buf[len++] = '8'; + buf[len++] = ';'; + } + } else if (val != 0) { + if (len + 10 > buf_size) { + growBuffer(buf, 10); + } + len += xmlCopyChar(0, &buf[len], val); + } + } else { + ent = xmlParseEntityRef(ctxt); + ctxt->nbentities++; + if (ent != NULL) + ctxt->nbentities += ent->owner; + if ((ent != NULL) && + (ent->etype == XML_INTERNAL_PREDEFINED_ENTITY)) { + if (len + 10 > buf_size) { + growBuffer(buf, 10); + } + if ((ctxt->replaceEntities == 0) && + (ent->content[0] == '&')) { + buf[len++] = '&'; + buf[len++] = '#'; + buf[len++] = '3'; + buf[len++] = '8'; + buf[len++] = ';'; + } else { + buf[len++] = ent->content[0]; + } + } else if ((ent != NULL) && + (ctxt->replaceEntities != 0)) { + if (ent->etype != XML_INTERNAL_PREDEFINED_ENTITY) { + ++ctxt->depth; + rep = xmlStringDecodeEntities(ctxt, ent->content, + XML_SUBSTITUTE_REF, + 0, 0, 0); + --ctxt->depth; + if (rep != NULL) { + current = rep; + while (*current != 0) { /* non input consuming */ + if ((*current == 0xD) || (*current == 0xA) || + (*current == 0x9)) { + buf[len++] = 0x20; + current++; + } else + buf[len++] = *current++; + if (len + 10 > buf_size) { + growBuffer(buf, 10); + } + } + xmlFree(rep); + rep = NULL; + } + } else { + if (len + 10 > buf_size) { + growBuffer(buf, 10); + } + if (ent->content != NULL) + buf[len++] = ent->content[0]; + } + } else if (ent != NULL) { + int i = xmlStrlen(ent->name); + const xmlChar *cur = ent->name; + + /* + * This may look absurd but is needed to detect + * entities problems + */ + if ((ent->etype != XML_INTERNAL_PREDEFINED_ENTITY) && + (ent->content != NULL) && (ent->checked == 0)) { + unsigned long oldnbent = ctxt->nbentities; + + ++ctxt->depth; + rep = xmlStringDecodeEntities(ctxt, ent->content, + XML_SUBSTITUTE_REF, 0, 0, 0); + --ctxt->depth; + + ent->checked = (ctxt->nbentities - oldnbent + 1) * 2; + if (rep != NULL) { + if (xmlStrchr(rep, '<')) + ent->checked |= 1; + xmlFree(rep); + rep = NULL; + } else { + ent->content[0] = 0; + } + } + + /* + * Just output the reference + */ + buf[len++] = '&'; + while (len + i + 10 > buf_size) { + growBuffer(buf, i + 10); + } + for (;i > 0;i--) + buf[len++] = *cur++; + buf[len++] = ';'; + } + } + } else { + if ((c == 0x20) || (c == 0xD) || (c == 0xA) || (c == 0x9)) { + if ((len != 0) || (!normalize)) { + if ((!normalize) || (!in_space)) { + COPY_BUF(l,buf,len,0x20); + while (len + 10 > buf_size) { + growBuffer(buf, 10); + } + } + in_space = 1; + } + } else { + in_space = 0; + COPY_BUF(l,buf,len,c); + if (len + 10 > buf_size) { + growBuffer(buf, 10); + } + } + NEXTL(l); + } + GROW; + c = CUR_CHAR(l); + } + if (ctxt->instate == XML_PARSER_EOF) + goto error; + + if ((in_space) && (normalize)) { + while ((len > 0) && (buf[len - 1] == 0x20)) len--; + } + buf[len] = 0; + if (RAW == '<') { + xmlFatalErr(ctxt, XML_ERR_LT_IN_ATTRIBUTE, NULL); + } else if (RAW != limit) { + if ((c != 0) && (!IS_CHAR(c))) { + xmlFatalErrMsg(ctxt, XML_ERR_INVALID_CHAR, + "invalid character in attribute value\n"); + } else { + xmlFatalErrMsg(ctxt, XML_ERR_ATTRIBUTE_NOT_FINISHED, + "AttValue: ' expected\n"); + } + } else + NEXT; + + /* + * There we potentially risk an overflow, don't allow attribute value of + * length more than INT_MAX it is a very reasonnable assumption ! + */ + if (len >= INT_MAX) { + xmlFatalErrMsg(ctxt, XML_ERR_ATTRIBUTE_NOT_FINISHED, + "AttValue length too long\n"); + goto mem_error; + } + + if (attlen != NULL) *attlen = (int) len; + return(buf); + +mem_error: + xmlErrMemory(ctxt, NULL); +error: + if (buf != NULL) + xmlFree(buf); + if (rep != NULL) + xmlFree(rep); + return(NULL); +} + +/** + * xmlParseAttValue: + * @ctxt: an XML parser context + * + * parse a value for an attribute + * Note: the parser won't do substitution of entities here, this + * will be handled later in xmlStringGetNodeList + * + * [10] AttValue ::= '"' ([^<&"] | Reference)* '"' | + * "'" ([^<&'] | Reference)* "'" + * + * 3.3.3 Attribute-Value Normalization: + * Before the value of an attribute is passed to the application or + * checked for validity, the XML processor must normalize it as follows: + * - a character reference is processed by appending the referenced + * character to the attribute value + * - an entity reference is processed by recursively processing the + * replacement text of the entity + * - a whitespace character (#x20, #xD, #xA, #x9) is processed by + * appending #x20 to the normalized value, except that only a single + * #x20 is appended for a "#xD#xA" sequence that is part of an external + * parsed entity or the literal entity value of an internal parsed entity + * - other characters are processed by appending them to the normalized value + * If the declared value is not CDATA, then the XML processor must further + * process the normalized attribute value by discarding any leading and + * trailing space (#x20) characters, and by replacing sequences of space + * (#x20) characters by a single space (#x20) character. + * All attributes for which no declaration has been read should be treated + * by a non-validating parser as if declared CDATA. + * + * Returns the AttValue parsed or NULL. The value has to be freed by the caller. + */ + + +xmlChar * +xmlParseAttValue(xmlParserCtxtPtr ctxt) { + if ((ctxt == NULL) || (ctxt->input == NULL)) return(NULL); + return(xmlParseAttValueInternal(ctxt, NULL, NULL, 0)); +} + +/** + * xmlParseSystemLiteral: + * @ctxt: an XML parser context + * + * parse an XML Literal + * + * [11] SystemLiteral ::= ('"' [^"]* '"') | ("'" [^']* "'") + * + * Returns the SystemLiteral parsed or NULL + */ + +xmlChar * +xmlParseSystemLiteral(xmlParserCtxtPtr ctxt) { + xmlChar *buf = NULL; + int len = 0; + int size = XML_PARSER_BUFFER_SIZE; + int cur, l; + xmlChar stop; + int state = ctxt->instate; + int count = 0; + + SHRINK; + if (RAW == '"') { + NEXT; + stop = '"'; + } else if (RAW == '\'') { + NEXT; + stop = '\''; + } else { + xmlFatalErr(ctxt, XML_ERR_LITERAL_NOT_STARTED, NULL); + return(NULL); + } + + buf = (xmlChar *) xmlMallocAtomic(size * sizeof(xmlChar)); + if (buf == NULL) { + xmlErrMemory(ctxt, NULL); + return(NULL); + } + ctxt->instate = XML_PARSER_SYSTEM_LITERAL; + cur = CUR_CHAR(l); + while ((IS_CHAR(cur)) && (cur != stop)) { /* checked */ + if (len + 5 >= size) { + xmlChar *tmp; + + if ((size > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "SystemLiteral"); + xmlFree(buf); + ctxt->instate = (xmlParserInputState) state; + return(NULL); + } + size *= 2; + tmp = (xmlChar *) xmlRealloc(buf, size * sizeof(xmlChar)); + if (tmp == NULL) { + xmlFree(buf); + xmlErrMemory(ctxt, NULL); + ctxt->instate = (xmlParserInputState) state; + return(NULL); + } + buf = tmp; + } + count++; + if (count > 50) { + GROW; + count = 0; + if (ctxt->instate == XML_PARSER_EOF) { + xmlFree(buf); + return(NULL); + } + } + COPY_BUF(l,buf,len,cur); + NEXTL(l); + cur = CUR_CHAR(l); + if (cur == 0) { + GROW; + SHRINK; + cur = CUR_CHAR(l); + } + } + buf[len] = 0; + ctxt->instate = (xmlParserInputState) state; + if (!IS_CHAR(cur)) { + xmlFatalErr(ctxt, XML_ERR_LITERAL_NOT_FINISHED, NULL); + } else { + NEXT; + } + return(buf); +} + +/** + * xmlParsePubidLiteral: + * @ctxt: an XML parser context + * + * parse an XML public literal + * + * [12] PubidLiteral ::= '"' PubidChar* '"' | "'" (PubidChar - "'")* "'" + * + * Returns the PubidLiteral parsed or NULL. + */ + +xmlChar * +xmlParsePubidLiteral(xmlParserCtxtPtr ctxt) { + xmlChar *buf = NULL; + int len = 0; + int size = XML_PARSER_BUFFER_SIZE; + xmlChar cur; + xmlChar stop; + int count = 0; + xmlParserInputState oldstate = ctxt->instate; + + SHRINK; + if (RAW == '"') { + NEXT; + stop = '"'; + } else if (RAW == '\'') { + NEXT; + stop = '\''; + } else { + xmlFatalErr(ctxt, XML_ERR_LITERAL_NOT_STARTED, NULL); + return(NULL); + } + buf = (xmlChar *) xmlMallocAtomic(size * sizeof(xmlChar)); + if (buf == NULL) { + xmlErrMemory(ctxt, NULL); + return(NULL); + } + ctxt->instate = XML_PARSER_PUBLIC_LITERAL; + cur = CUR; + while ((IS_PUBIDCHAR_CH(cur)) && (cur != stop)) { /* checked */ + if (len + 1 >= size) { + xmlChar *tmp; + + if ((size > XML_MAX_NAME_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "Public ID"); + xmlFree(buf); + return(NULL); + } + size *= 2; + tmp = (xmlChar *) xmlRealloc(buf, size * sizeof(xmlChar)); + if (tmp == NULL) { + xmlErrMemory(ctxt, NULL); + xmlFree(buf); + return(NULL); + } + buf = tmp; + } + buf[len++] = cur; + count++; + if (count > 50) { + GROW; + count = 0; + if (ctxt->instate == XML_PARSER_EOF) { + xmlFree(buf); + return(NULL); + } + } + NEXT; + cur = CUR; + if (cur == 0) { + GROW; + SHRINK; + cur = CUR; + } + } + buf[len] = 0; + if (cur != stop) { + xmlFatalErr(ctxt, XML_ERR_LITERAL_NOT_FINISHED, NULL); + } else { + NEXT; + } + ctxt->instate = oldstate; + return(buf); +} + +static void xmlParseCharDataComplex(xmlParserCtxtPtr ctxt, int cdata); + +/* + * used for the test in the inner loop of the char data testing + */ +static const unsigned char test_char_data[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9, CR/LF separated */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x00, 0x27, /* & */ + 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3A, 0x3B, 0x00, 0x3D, 0x3E, 0x3F, /* < */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x00, 0x5E, 0x5F, /* ] */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* non-ascii */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/** + * xmlParseCharData: + * @ctxt: an XML parser context + * @cdata: int indicating whether we are within a CDATA section + * + * parse a CharData section. + * if we are within a CDATA section ']]>' marks an end of section. + * + * The right angle bracket (>) may be represented using the string ">", + * and must, for compatibility, be escaped using ">" or a character + * reference when it appears in the string "]]>" in content, when that + * string is not marking the end of a CDATA section. + * + * [14] CharData ::= [^<&]* - ([^<&]* ']]>' [^<&]*) + */ + +void +xmlParseCharData(xmlParserCtxtPtr ctxt, int cdata) { + const xmlChar *in; + int nbchar = 0; + int line = ctxt->input->line; + int col = ctxt->input->col; + int ccol; + + SHRINK; + GROW; + /* + * Accelerated common case where input don't need to be + * modified before passing it to the handler. + */ + if (!cdata) { + in = ctxt->input->cur; + do { +get_more_space: + while (*in == 0x20) { in++; ctxt->input->col++; } + if (*in == 0xA) { + do { + ctxt->input->line++; ctxt->input->col = 1; + in++; + } while (*in == 0xA); + goto get_more_space; + } + if (*in == '<') { + nbchar = in - ctxt->input->cur; + if (nbchar > 0) { + const xmlChar *tmp = ctxt->input->cur; + ctxt->input->cur = in; + + if ((ctxt->sax != NULL) && + (ctxt->sax->ignorableWhitespace != + ctxt->sax->characters)) { + if (areBlanks(ctxt, tmp, nbchar, 1)) { + if (ctxt->sax->ignorableWhitespace != NULL) + ctxt->sax->ignorableWhitespace(ctxt->userData, + tmp, nbchar); + } else { + if (ctxt->sax->characters != NULL) + ctxt->sax->characters(ctxt->userData, + tmp, nbchar); + if (*ctxt->space == -1) + *ctxt->space = -2; + } + } else if ((ctxt->sax != NULL) && + (ctxt->sax->characters != NULL)) { + ctxt->sax->characters(ctxt->userData, + tmp, nbchar); + } + } + return; + } + +get_more: + ccol = ctxt->input->col; + while (test_char_data[*in]) { + in++; + ccol++; + } + ctxt->input->col = ccol; + if (*in == 0xA) { + do { + ctxt->input->line++; ctxt->input->col = 1; + in++; + } while (*in == 0xA); + goto get_more; + } + if (*in == ']') { + if ((in[1] == ']') && (in[2] == '>')) { + xmlFatalErr(ctxt, XML_ERR_MISPLACED_CDATA_END, NULL); + ctxt->input->cur = in + 1; + return; + } + in++; + ctxt->input->col++; + goto get_more; + } + nbchar = in - ctxt->input->cur; + if (nbchar > 0) { + if ((ctxt->sax != NULL) && + (ctxt->sax->ignorableWhitespace != + ctxt->sax->characters) && + (IS_BLANK_CH(*ctxt->input->cur))) { + const xmlChar *tmp = ctxt->input->cur; + ctxt->input->cur = in; + + if (areBlanks(ctxt, tmp, nbchar, 0)) { + if (ctxt->sax->ignorableWhitespace != NULL) + ctxt->sax->ignorableWhitespace(ctxt->userData, + tmp, nbchar); + } else { + if (ctxt->sax->characters != NULL) + ctxt->sax->characters(ctxt->userData, + tmp, nbchar); + if (*ctxt->space == -1) + *ctxt->space = -2; + } + line = ctxt->input->line; + col = ctxt->input->col; + } else if (ctxt->sax != NULL) { + if (ctxt->sax->characters != NULL) + ctxt->sax->characters(ctxt->userData, + ctxt->input->cur, nbchar); + line = ctxt->input->line; + col = ctxt->input->col; + } + /* something really bad happened in the SAX callback */ + if (ctxt->instate != XML_PARSER_CONTENT) + return; + } + ctxt->input->cur = in; + if (*in == 0xD) { + in++; + if (*in == 0xA) { + ctxt->input->cur = in; + in++; + ctxt->input->line++; ctxt->input->col = 1; + continue; /* while */ + } + in--; + } + if (*in == '<') { + return; + } + if (*in == '&') { + return; + } + SHRINK; + GROW; + if (ctxt->instate == XML_PARSER_EOF) + return; + in = ctxt->input->cur; + } while (((*in >= 0x20) && (*in <= 0x7F)) || (*in == 0x09)); + nbchar = 0; + } + ctxt->input->line = line; + ctxt->input->col = col; + xmlParseCharDataComplex(ctxt, cdata); +} + +/** + * xmlParseCharDataComplex: + * @ctxt: an XML parser context + * @cdata: int indicating whether we are within a CDATA section + * + * parse a CharData section.this is the fallback function + * of xmlParseCharData() when the parsing requires handling + * of non-ASCII characters. + */ +static void +xmlParseCharDataComplex(xmlParserCtxtPtr ctxt, int cdata) { + xmlChar buf[XML_PARSER_BIG_BUFFER_SIZE + 5]; + int nbchar = 0; + int cur, l; + int count = 0; + + SHRINK; + GROW; + cur = CUR_CHAR(l); + while ((cur != '<') && /* checked */ + (cur != '&') && + (IS_CHAR(cur))) /* test also done in xmlCurrentChar() */ { + if ((cur == ']') && (NXT(1) == ']') && + (NXT(2) == '>')) { + if (cdata) break; + else { + xmlFatalErr(ctxt, XML_ERR_MISPLACED_CDATA_END, NULL); + } + } + COPY_BUF(l,buf,nbchar,cur); + if (nbchar >= XML_PARSER_BIG_BUFFER_SIZE) { + buf[nbchar] = 0; + + /* + * OK the segment is to be consumed as chars. + */ + if ((ctxt->sax != NULL) && (!ctxt->disableSAX)) { + if (areBlanks(ctxt, buf, nbchar, 0)) { + if (ctxt->sax->ignorableWhitespace != NULL) + ctxt->sax->ignorableWhitespace(ctxt->userData, + buf, nbchar); + } else { + if (ctxt->sax->characters != NULL) + ctxt->sax->characters(ctxt->userData, buf, nbchar); + if ((ctxt->sax->characters != + ctxt->sax->ignorableWhitespace) && + (*ctxt->space == -1)) + *ctxt->space = -2; + } + } + nbchar = 0; + /* something really bad happened in the SAX callback */ + if (ctxt->instate != XML_PARSER_CONTENT) + return; + } + count++; + if (count > 50) { + GROW; + count = 0; + if (ctxt->instate == XML_PARSER_EOF) + return; + } + NEXTL(l); + cur = CUR_CHAR(l); + } + if (nbchar != 0) { + buf[nbchar] = 0; + /* + * OK the segment is to be consumed as chars. + */ + if ((ctxt->sax != NULL) && (!ctxt->disableSAX)) { + if (areBlanks(ctxt, buf, nbchar, 0)) { + if (ctxt->sax->ignorableWhitespace != NULL) + ctxt->sax->ignorableWhitespace(ctxt->userData, buf, nbchar); + } else { + if (ctxt->sax->characters != NULL) + ctxt->sax->characters(ctxt->userData, buf, nbchar); + if ((ctxt->sax->characters != ctxt->sax->ignorableWhitespace) && + (*ctxt->space == -1)) + *ctxt->space = -2; + } + } + } + if ((cur != 0) && (!IS_CHAR(cur))) { + /* Generate the error and skip the offending character */ + xmlFatalErrMsgInt(ctxt, XML_ERR_INVALID_CHAR, + "PCDATA invalid Char value %d\n", + cur); + NEXTL(l); + } +} + +/** + * xmlParseExternalID: + * @ctxt: an XML parser context + * @publicID: a xmlChar** receiving PubidLiteral + * @strict: indicate whether we should restrict parsing to only + * production [75], see NOTE below + * + * Parse an External ID or a Public ID + * + * NOTE: Productions [75] and [83] interact badly since [75] can generate + * 'PUBLIC' S PubidLiteral S SystemLiteral + * + * [75] ExternalID ::= 'SYSTEM' S SystemLiteral + * | 'PUBLIC' S PubidLiteral S SystemLiteral + * + * [83] PublicID ::= 'PUBLIC' S PubidLiteral + * + * Returns the function returns SystemLiteral and in the second + * case publicID receives PubidLiteral, is strict is off + * it is possible to return NULL and have publicID set. + */ + +xmlChar * +xmlParseExternalID(xmlParserCtxtPtr ctxt, xmlChar **publicID, int strict) { + xmlChar *URI = NULL; + + SHRINK; + + *publicID = NULL; + if (CMP6(CUR_PTR, 'S', 'Y', 'S', 'T', 'E', 'M')) { + SKIP(6); + if (SKIP_BLANKS == 0) { + xmlFatalErrMsg(ctxt, XML_ERR_SPACE_REQUIRED, + "Space required after 'SYSTEM'\n"); + } + URI = xmlParseSystemLiteral(ctxt); + if (URI == NULL) { + xmlFatalErr(ctxt, XML_ERR_URI_REQUIRED, NULL); + } + } else if (CMP6(CUR_PTR, 'P', 'U', 'B', 'L', 'I', 'C')) { + SKIP(6); + if (SKIP_BLANKS == 0) { + xmlFatalErrMsg(ctxt, XML_ERR_SPACE_REQUIRED, + "Space required after 'PUBLIC'\n"); + } + *publicID = xmlParsePubidLiteral(ctxt); + if (*publicID == NULL) { + xmlFatalErr(ctxt, XML_ERR_PUBID_REQUIRED, NULL); + } + if (strict) { + /* + * We don't handle [83] so "S SystemLiteral" is required. + */ + if (SKIP_BLANKS == 0) { + xmlFatalErrMsg(ctxt, XML_ERR_SPACE_REQUIRED, + "Space required after the Public Identifier\n"); + } + } else { + /* + * We handle [83] so we return immediately, if + * "S SystemLiteral" is not detected. We skip blanks if no + * system literal was found, but this is harmless since we must + * be at the end of a NotationDecl. + */ + if (SKIP_BLANKS == 0) return(NULL); + if ((CUR != '\'') && (CUR != '"')) return(NULL); + } + URI = xmlParseSystemLiteral(ctxt); + if (URI == NULL) { + xmlFatalErr(ctxt, XML_ERR_URI_REQUIRED, NULL); + } + } + return(URI); +} + +/** + * xmlParseCommentComplex: + * @ctxt: an XML parser context + * @buf: the already parsed part of the buffer + * @len: number of bytes filles in the buffer + * @size: allocated size of the buffer + * + * Skip an XML (SGML) comment + * The spec says that "For compatibility, the string "--" (double-hyphen) + * must not occur within comments. " + * This is the slow routine in case the accelerator for ascii didn't work + * + * [15] Comment ::= '' + */ +static void +xmlParseCommentComplex(xmlParserCtxtPtr ctxt, xmlChar *buf, + size_t len, size_t size) { + int q, ql; + int r, rl; + int cur, l; + size_t count = 0; + int inputid; + + inputid = ctxt->input->id; + + if (buf == NULL) { + len = 0; + size = XML_PARSER_BUFFER_SIZE; + buf = (xmlChar *) xmlMallocAtomic(size * sizeof(xmlChar)); + if (buf == NULL) { + xmlErrMemory(ctxt, NULL); + return; + } + } + GROW; /* Assure there's enough input data */ + q = CUR_CHAR(ql); + if (q == 0) + goto not_terminated; + if (!IS_CHAR(q)) { + xmlFatalErrMsgInt(ctxt, XML_ERR_INVALID_CHAR, + "xmlParseComment: invalid xmlChar value %d\n", + q); + xmlFree (buf); + return; + } + NEXTL(ql); + r = CUR_CHAR(rl); + if (r == 0) + goto not_terminated; + if (!IS_CHAR(r)) { + xmlFatalErrMsgInt(ctxt, XML_ERR_INVALID_CHAR, + "xmlParseComment: invalid xmlChar value %d\n", + q); + xmlFree (buf); + return; + } + NEXTL(rl); + cur = CUR_CHAR(l); + if (cur == 0) + goto not_terminated; + while (IS_CHAR(cur) && /* checked */ + ((cur != '>') || + (r != '-') || (q != '-'))) { + if ((r == '-') && (q == '-')) { + xmlFatalErr(ctxt, XML_ERR_HYPHEN_IN_COMMENT, NULL); + } + if ((len > XML_MAX_TEXT_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErrMsgStr(ctxt, XML_ERR_COMMENT_NOT_FINISHED, + "Comment too big found", NULL); + xmlFree (buf); + return; + } + if (len + 5 >= size) { + xmlChar *new_buf; + size_t new_size; + + new_size = size * 2; + new_buf = (xmlChar *) xmlRealloc(buf, new_size); + if (new_buf == NULL) { + xmlFree (buf); + xmlErrMemory(ctxt, NULL); + return; + } + buf = new_buf; + size = new_size; + } + COPY_BUF(ql,buf,len,q); + q = r; + ql = rl; + r = cur; + rl = l; + + count++; + if (count > 50) { + GROW; + count = 0; + if (ctxt->instate == XML_PARSER_EOF) { + xmlFree(buf); + return; + } + } + NEXTL(l); + cur = CUR_CHAR(l); + if (cur == 0) { + SHRINK; + GROW; + cur = CUR_CHAR(l); + } + } + buf[len] = 0; + if (cur == 0) { + xmlFatalErrMsgStr(ctxt, XML_ERR_COMMENT_NOT_FINISHED, + "Comment not terminated \n + * The spec says that "For compatibility, the string "--" (double-hyphen) + * must not occur within comments. " + * + * [15] Comment ::= '' + */ +void +xmlParseComment(xmlParserCtxtPtr ctxt) { + xmlChar *buf = NULL; + size_t size = XML_PARSER_BUFFER_SIZE; + size_t len = 0; + xmlParserInputState state; + const xmlChar *in; + size_t nbchar = 0; + int ccol; + int inputid; + + /* + * Check that there is a comment right here. + */ + if ((RAW != '<') || (NXT(1) != '!') || + (NXT(2) != '-') || (NXT(3) != '-')) return; + state = ctxt->instate; + ctxt->instate = XML_PARSER_COMMENT; + inputid = ctxt->input->id; + SKIP(4); + SHRINK; + GROW; + + /* + * Accelerated common case where input don't need to be + * modified before passing it to the handler. + */ + in = ctxt->input->cur; + do { + if (*in == 0xA) { + do { + ctxt->input->line++; ctxt->input->col = 1; + in++; + } while (*in == 0xA); + } +get_more: + ccol = ctxt->input->col; + while (((*in > '-') && (*in <= 0x7F)) || + ((*in >= 0x20) && (*in < '-')) || + (*in == 0x09)) { + in++; + ccol++; + } + ctxt->input->col = ccol; + if (*in == 0xA) { + do { + ctxt->input->line++; ctxt->input->col = 1; + in++; + } while (*in == 0xA); + goto get_more; + } + nbchar = in - ctxt->input->cur; + /* + * save current set of data + */ + if (nbchar > 0) { + if ((ctxt->sax != NULL) && + (ctxt->sax->comment != NULL)) { + if (buf == NULL) { + if ((*in == '-') && (in[1] == '-')) + size = nbchar + 1; + else + size = XML_PARSER_BUFFER_SIZE + nbchar; + buf = (xmlChar *) xmlMallocAtomic(size * sizeof(xmlChar)); + if (buf == NULL) { + xmlErrMemory(ctxt, NULL); + ctxt->instate = state; + return; + } + len = 0; + } else if (len + nbchar + 1 >= size) { + xmlChar *new_buf; + size += len + nbchar + XML_PARSER_BUFFER_SIZE; + new_buf = (xmlChar *) xmlRealloc(buf, + size * sizeof(xmlChar)); + if (new_buf == NULL) { + xmlFree (buf); + xmlErrMemory(ctxt, NULL); + ctxt->instate = state; + return; + } + buf = new_buf; + } + memcpy(&buf[len], ctxt->input->cur, nbchar); + len += nbchar; + buf[len] = 0; + } + } + if ((len > XML_MAX_TEXT_LENGTH) && + ((ctxt->options & XML_PARSE_HUGE) == 0)) { + xmlFatalErrMsgStr(ctxt, XML_ERR_COMMENT_NOT_FINISHED, + "Comment too big found", NULL); + xmlFree (buf); + return; + } + ctxt->input->cur = in; + if (*in == 0xA) { + in++; + ctxt->input->line++; ctxt->input->col = 1; + } + if (*in == 0xD) { + in++; + if (*in == 0xA) { + ctxt->input->cur = in; + in++; + ctxt->input->line++; ctxt->input->col = 1; + continue; /* while */ + } + in--; + } + SHRINK; + GROW; + if (ctxt->instate == XML_PARSER_EOF) { + xmlFree(buf); + return; + } + in = ctxt->input->cur; + if (*in == '-') { + if (in[1] == '-') { + if (in[2] == '>') { + if (ctxt->input->id != inputid) { + xmlFatalErrMsg(ctxt, XML_ERR_ENTITY_BOUNDARY, + "comment doesn't start and stop in the" + " same entity\n"); + } + SKIP(3); + if ((ctxt->sax != NULL) && (ctxt->sax->comment != NULL) && + (!ctxt->disableSAX)) { + if (buf != NULL) + ctxt->sax->comment(ctxt->userData, buf); + else + ctxt->sax->comment(ctxt->userData, BAD_CAST ""); + } + if (buf != NULL) + xmlFree(buf); + if (ctxt->instate != XML_PARSER_EOF) + ctxt->instate = state; + return; + } + if (buf != NULL) { + xmlFatalErrMsgStr(ctxt, XML_ERR_HYPHEN_IN_COMMENT, + "Double hyphen within comment: " + " \n, Aborting " + f"pipeline.... %s", {func.__name__}, + {func.__module__}, exc, extra=MULTI_LINE_MESSAGE_TRUE) + raise + + return catch_errors + + +def catch_tool_errors(tool_name): + def catch_tool_errors_impl(func): + @functools.wraps(func) + async def catch_errors(self, *args, **kwargs): + try: + logger.debug(f"Before running tool {tool_name} in file {func.__module__}") + if args == () and not kwargs == {}: + return await func(self, **kwargs) + elif kwargs == {} and not args == (): + return await func(self, *args) + else: + return await func(self, *args, **kwargs) + + except Exception as exc: + format_error_message = f"Intercepted an error from tool {tool_name}, in function=" \ + f"{func.__name__}, on file=" \ + f"{func.__module__}" + raise ToolRaisedException(message=format_error_message, tool_name=tool_name, original_exception=exc) + return catch_errors + + return catch_tool_errors_impl diff --git a/src/vuln_analysis/utils/full_text_search.py b/src/vuln_analysis/utils/full_text_search.py index 58b6fcb21..452e5393a 100644 --- a/src/vuln_analysis/utils/full_text_search.py +++ b/src/vuln_analysis/utils/full_text_search.py @@ -13,9 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +import json import os import re +import threading +from collections import OrderedDict from pathlib import Path from typing import Iterable @@ -28,15 +30,14 @@ from tantivy import SchemaBuilder from tqdm import tqdm -logger = logging.getLogger(__name__) +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) variable_pattern = re.compile(r"([A-Z][a-z]+|[a-z]+|[A-Z]+(?=[A-Z]|$))") def tokenize_code(code: str) -> str: - """ - Tokenize code into simple readable document format - """ + """Normalize source code identifiers for search: split camelCase/snake_case, lowercase, filter noise.""" matches = re.finditer(r"\b\w{2,}\b", code) tokens = [] for m in matches: @@ -46,6 +47,7 @@ def tokenize_code(code: str) -> str: for part in variable_pattern.findall(section): if len(part) < 2: continue + # Exclude hex-like strings and repetitive patterns if (sum(1 for c in part if "a" <= c <= "z" or "A" <= c <= "Z" or "0" <= c <= "9") > len(part) // 2 and len(part) / len(set(part)) < 4): tokens.append(part.lower()) @@ -54,8 +56,9 @@ def tokenize_code(code: str) -> str: def clean_query(input_query) -> str: - """ - Parse a query string with OR/AND operations and fix quotes issues using regex. + """Parse a query string with OR/AND operations and fix quote issues. + + Sanitizes LLM-generated queries for Tantivy — unmatched quotes crash its parser. """ # Remove anything after newline characters if '\n' in input_query: @@ -84,9 +87,21 @@ def replace_quoted(match): return input_query +from vuln_analysis.utils.source_classification import is_dependency_path, filter_by_source_scope class FullTextSearch: + """Tantivy-based full text search index for source code and documentation. + + Wraps a Tantivy index with document management, query sanitization, and + result separation into app code vs. dependency code. Instances are cached + via get_instance() with LRU eviction to avoid holding too many open indexes. + """ INDEX_TYPE = "tantivy" + # LRU cache of FullTextSearch instances keyed by cache_path. + # Prevents unbounded growth of open Tantivy index handles. + _instances: "OrderedDict[str, FullTextSearch]" = None + _INSTANCE_CACHE_MAX = 10 + _instances_lock = threading.Lock() def __init__(self, cache_path: str = None, tokenizer=False): if cache_path: @@ -96,13 +111,37 @@ def __init__(self, cache_path: str = None, tokenizer=False): self.index.reload() self.tokenizer = tokenizer + @classmethod + def get_instance(cls, cache_path: str, tokenizer=False) -> "FullTextSearch": + """Return a cached instance for the given path, with LRU eviction. + + On cache hit, moves the entry to the end (most recently used). + On cache miss, creates a new instance and evicts the least recently used + if the cache exceeds _INSTANCE_CACHE_MAX. + """ + with cls._instances_lock: + if cls._instances is None: + cls._instances = OrderedDict() + if cache_path in cls._instances: + cls._instances.move_to_end(cache_path) + return cls._instances[cache_path] + instance = cls(cache_path=cache_path, tokenizer=tokenizer) + with cls._instances_lock: + if cache_path in cls._instances: + return cls._instances[cache_path] + cls._instances[cache_path] = instance + if len(cls._instances) > cls._INSTANCE_CACHE_MAX: + evicted = cls._instances.popitem(last=False) + logger.debug("Evicted LRU FullTextSearch instance: %s", evicted[0]) + return instance + @classmethod def get_index_directory(cls, base_path: str, hash_value: str) -> Path: """Returns the path where the index should be stored""" return Path(base_path) / cls.INDEX_TYPE / hash_value def _build_schema(self): - """Build schema for the code index""" + """Build Tantivy schema: file_path + content (both stored for retrieval) + doc_id.""" schema_builder = SchemaBuilder() schema_builder.add_text_field("file_path", stored=True) schema_builder.add_text_field("content", stored=True) @@ -111,17 +150,28 @@ def _build_schema(self): return schema def add_documents(self, documents: Iterable): - + """Index an iterable of (file_path, content) tuples into Tantivy.""" writer = self.index.writer() for doc_id, (title, text) in enumerate(documents): writer.add_document(TantivyDocument(file_path=title, content=text, doc_id=doc_id)) writer.commit() def is_empty(self): + """Check if the index contains any documents.""" return self.index.searcher().num_docs == 0 - def search_index(self, query: str, top_k: int = 10) -> list[dict] | str: + def search_index(self, query: str, top_k: int = 10, source_scope: list[str] | None = None) -> str: + """Search the index using Tantivy's ranked query engine. + + Returns results separated into application code and dependency code sections. + When source_scope is provided, dependency results are filtered to only + include files matching the scope (e.g. specific package names). + Args: + query: Search query string (sanitized internally) + top_k: Maximum total results to return (app results prioritized) + source_scope: Optional list of path substrings to filter dependency results + """ self.index.reload() try: if self.tokenizer: @@ -129,17 +179,75 @@ def search_index(self, query: str, top_k: int = 10) -> list[dict] | str: query = clean_query(query) query, _ = self.index.parse_query_lenient(query) searcher = self.index.searcher() - results = searcher.search(query, limit=top_k).hits - return [{ - "source": searcher.doc(doc_id)["file_path"][0], "content": searcher.doc(doc_id)["content"][0] - } for _, doc_id in results] + results = searcher.search(query, limit=50).hits + app_docs = [] + dep_docs = [] + + for _, doc_id in results: + raw = searcher.doc(doc_id) + source_path = raw["file_path"][0] + # Skip .git/ internal directory (but not .github/, .gitignore, etc.) + if source_path == ".git" or source_path.startswith(".git/") or "/.git/" in source_path: + continue + doc = {"source": source_path, "content": raw["content"][0]} + if is_dependency_path(doc["source"]): + dep_docs.append(doc) + else: + app_docs.append(doc) + + if not app_docs and dep_docs: + seen_sources = {d["source"] for d in dep_docs} + wider = searcher.search(query, limit=500).hits + for _, doc_id in wider: + raw = searcher.doc(doc_id) + src = raw["file_path"][0] + # Skip .git/ internal directory (but not .github/, .gitignore, etc.) + if src == ".git" or src.startswith(".git/") or "/.git/" in src: + continue + if src in seen_sources: + continue + doc = {"source": src, "content": raw["content"][0]} + if is_dependency_path(src): + dep_docs.append(doc) + else: + app_docs.append(doc) + if len(app_docs) >= top_k: + break + + pre_filter_count = len(dep_docs) + dep_docs = filter_by_source_scope(dep_docs, source_scope, lambda d: d["source"]) + if pre_filter_count != len(dep_docs): + logger.debug("Source scope %s filtered dep results from %d to %d", + source_scope, pre_filter_count, len(dep_docs)) + + total_app = len(app_docs) + total_dep = len(dep_docs) + + if total_app == 0 and total_dep == 0: + return "[]" + + app_docs = app_docs[:top_k] + remaining = top_k - len(app_docs) + dep_docs = dep_docs[:max(remaining, 0)] + + app_header = f"Main application ({len(app_docs)} of {total_app} results)" + dep_header = f"Application library dependencies ({len(dep_docs)} of {total_dep} results)" + + parts = [app_header] + if app_docs: + parts.append(json.dumps(app_docs)) + parts.append(dep_header) + if dep_docs: + parts.append(json.dumps(dep_docs)) + + return "\n".join(parts) except Exception as e: - logger.exception(e) + logger.exception("Search failed for query") return "There was an error searching for the query." def add_documents_from_langchain_chunks(self, documents: list[Document]): - """Create an index from langchain chunked documents""" + """Index pre-chunked LangChain documents (used by document_embedding.py for VDB build).""" try: documents = [(doc.metadata['source'], doc.page_content) for doc in documents] @@ -149,23 +257,30 @@ def add_documents_from_langchain_chunks(self, documents: list[Document]): self.add_documents(tqdm(documents, total=len(documents), desc="Indexing")) except Exception as e: - logger.warning(f"Unable to add documents to the index {e}") + logger.warning("Unable to add documents to the index: %s", e) def add_documents_from_code_path(self, code_path: str, include_extensions: list[str], use_langparser=True, splitter=True): - """Create an index from raw files.""" + """Index raw source files from a directory, optionally using LangChain's LanguageParser. + + When use_langparser=True, files are parsed with language-aware chunking. + When False, files are read as raw text (optionally tokenized for search). + """ doc_content = [] if use_langparser: + # Exclude internal/metadata directories from indexing + exclude_patterns = [".git/**", ".git", "**/.git/**", "**/__pycache__/**"] loader = GenericLoader.from_filesystem( code_path, glob="**/*", suffixes=include_extensions, + exclude=exclude_patterns, parser=LanguageParser(), ) docs = loader.load() @@ -176,10 +291,15 @@ def add_documents_from_code_path(self, doc_content = [(doc.metadata["source"], doc.page_content) for doc in docs] else: + # Directories to skip during traversal (internal/metadata dirs) + skip_dirs = {".git", "__pycache__", ".hg", ".svn"} + + for root, dirs, files in os.walk(code_path): + # Prune directories in-place to skip internal/metadata directories + dirs[:] = [d for d in dirs if d not in skip_dirs] - for root, _, files in os.walk(code_path): for file in files: - if any(file.endswith(ext) for ext in include_extensions): + if any(file.endswith(ext) for ext in include_extensions) or file in no_extension: file_path = os.path.join(root, file) try: with open(file_path, "r") as f: @@ -188,6 +308,6 @@ def add_documents_from_code_path(self, content = tokenize_code(content) doc_content.append((file_path, content)) except Exception as e: - logger.warning(f"Error reading {file_path}: {e}") + logger.warning("Error reading %s: %s", file_path, e) self.add_documents(doc_content) diff --git a/src/vuln_analysis/utils/function_name_extractor.py b/src/vuln_analysis/utils/function_name_extractor.py new file mode 100644 index 000000000..b3d19e467 --- /dev/null +++ b/src/vuln_analysis/utils/function_name_extractor.py @@ -0,0 +1,155 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +from exploit_iq_commons.utils.chain_of_calls_retriever import ChainOfCallsRetrieverBase +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + + +logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") + + +def traverse_all_parameters(function_ending_index_end, function_prefix_index_end, function_string): + current_idx = function_prefix_index_end + function_builder = function_string[:function_prefix_index_end] + if current_idx == function_ending_index_end: + function_builder += ")" + while current_idx < function_ending_index_end: + end_of_arg_ind = function_string[current_idx:].find(",") + if end_of_arg_ind > -1: + value = handle_argument(function_string[current_idx: current_idx + end_of_arg_ind].strip()) + function_builder += f"\\s?{value}," + current_idx = current_idx + end_of_arg_ind + 1 + else: + # last argument + value = handle_argument(function_string[current_idx:function_ending_index_end].strip()) + function_builder += f"\\s?{value}\\s?)" + current_idx = function_ending_index_end + + return function_builder + + +def infer_if_short_package_name_match(package: str, package_names: list[str], final_package: set[str]) -> bool: + package_with_version_suffix = r"[./][vV][1-9]+$" + match = re.search(package_with_version_suffix, package) + if match: + parts = package.split(match.group()) + short_package_name = parts[-2].split("/")[-1] + else: + parts = package.split("/") + short_package_name = parts[-1].split("/")[-1] + + for current_package in package_names: + if short_package_name in current_package: + final_package.add(current_package) + return True + + return False + + +def handle_argument(param: str) -> str: + """ + This function get argument string.if it's a literal, returns it as is,otherwise, returns wildcard to match anything. + as literals are intended to be searched as specified, and variables/identifiers names can be any other name. + :param param: str + :return: returns the right matcher. + """ + if (param.startswith("'") and param.endswith("'")) or (param.startswith('"') and param.endswith('"')): + return param + elif param.__contains__("(") and param.__contains__(")"): + function_prefix_index_end = param.find("(") + 1 + function_ending_index_end = param.rfind(")") + return traverse_all_parameters(function_ending_index_end, function_prefix_index_end, param) + else: + return ".*" + + +class FunctionNameExtractor: + + def __init__(self, coc_retriever: ChainOfCallsRetrieverBase): + self.lang_parser = coc_retriever.language_parser + self.docs = coc_retriever.documents + self.ecosystem = coc_retriever.ecosystem + self.coc_retriever = coc_retriever + + def fetch_list(self, query: str) -> list[str]: + """ +use this tool to search for real specific usages of standard library functions/methods in a given package. +It searches for exact string matching, except variables placed in arguments, which can be anyname in essence, +and not necessarily the ones that are in the search string. + +returns list of calling functions in the package, e.g - ['packageName,functionName','packageName,functionName2'] + """ + + query = query.splitlines()[0].replace('"', '').replace("'", "").strip() + parts_input = query.split(",", 1) + + # Validate input format + if len(parts_input) < 2: + error_msg = ( + f"ERROR: Invalid input format. " + f"received: '{query}'. " + f"Please provide input in the correct format with a comma separating package name and function name. " + f"Example: 'github.com/golang-jwt/jwt,errors.New(\"value1\",\"value2\")'" + ) + logger.error(error_msg) + return [error_msg] # Return error message that LLM can see + + package = parts_input[0] + function = parts_input[1] + final_package = set() + package_functions = [doc for doc in self.docs if + self.lang_parser.get_package_name(doc, package) + or + infer_if_short_package_name_match(package, self.lang_parser.get_package_names(doc), + final_package)] + the_final_package = list(final_package) + # If short package name was inferred , use it also in searching all the way + if len(the_final_package) > 0 and the_final_package[0] != package: + package = the_final_package[0] + + if function.__contains__(".") and function.__contains__("(") and function.__contains__(")"): + function_string = str(function) + function_prefix_index_end = function_string.find("(") + 1 + function_ending_index_end = function_string.rfind(")") + function_builder = traverse_all_parameters(function_ending_index_end, function_prefix_index_end + , function_string) + else: + return [] + + containing_functions = list() + comment_line_character = self.lang_parser.get_comment_line_notation() + for func in package_functions: + + if match := re.search(rf"{function_builder}", func.page_content, flags=re.MULTILINE): + current_offset = match.start() - 1 + while func.page_content[current_offset] != os.linesep: + current_offset -= 1 + + # Make sure to filter out matches that are code comments + if func.page_content[current_offset: match.start()].strip().startswith(comment_line_character): + pass + else: + try: + new_function_name = self.lang_parser.get_function_name(func) + except ValueError: + continue + if not new_function_name: + continue + containing_functions.append(f"{package},{new_function_name}") + + return containing_functions diff --git a/src/vuln_analysis/utils/function_name_locator.py b/src/vuln_analysis/utils/function_name_locator.py new file mode 100644 index 000000000..4530ec24a --- /dev/null +++ b/src/vuln_analysis/utils/function_name_locator.py @@ -0,0 +1,484 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import difflib + + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.chain_of_calls_retriever_base import ChainOfCallsRetrieverBase +from exploit_iq_commons.utils.dep_tree import Ecosystem +from exploit_iq_commons.utils.standard_library_cache import StandardLibraryCache +from vuln_analysis.utils.serp_api_wrapper import MorpheusSerpAPIWrapper + +from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager +from vuln_analysis.utils.prompt_factory import FL_EXAMPLES + +logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}") + + +class FunctionNameLocator: + """ + A class dedicated to locating function names using fuzzy matching. + Initially designed for C/C++ but can be extended for other languages. + """ + + def __init__(self, coc_retriever: ChainOfCallsRetrieverBase): + self.lang_parser = coc_retriever.language_parser + self.coc_retriever = coc_retriever + self.is_std_package = False + self.is_package_valid = False + self.stdlib_cache = StandardLibraryCache.get_instance() + self.short_go_package_name = {} + if self.coc_retriever.ecosystem.value == Ecosystem.GO.value: + self.short_go_package_name = self.build_short_go_package_name() + self.sbom_dict = self.build_sbom_dict() + + def build_sbom_dict(self) -> dict: + sbom_dict = {} + for package in RPMDependencyManager.get_instance().sbom: + sbom_dict[package.name] = package.version + return sbom_dict + def check_package_in_sbom(self, package: str) -> tuple[bool, str]: + if package in self.sbom_dict: + return True, self.sbom_dict[package] + else: + return False, None + + def check_fuzzy_match_in_sbom(self, package: str) -> tuple[bool, str]: + fuzzy_matches = difflib.get_close_matches(package, self.sbom_dict.keys(), n=3, cutoff=0.6) + if fuzzy_matches: + return True, ", ".join(fuzzy_matches) + else: + return False, None + + def build_short_go_package_name(self) -> dict: + short_go_package_name = {} + for package in self.coc_retriever.supported_packages: + short_name = package.split("/")[-1] + short_go_package_name[short_name] = package + return short_go_package_name + + def _resolve_go_fqdn_to_module(self, package: str) -> str | None: + """Find the longest supported_package that is a prefix of the input package.""" + best_match = None + for supported_package in self.coc_retriever.supported_packages: + if package.lower().startswith(supported_package.lower() + "/") or \ + package.lower() == supported_package.lower(): + if best_match is None or len(supported_package) > len(best_match): + best_match = supported_package + return best_match + + @staticmethod + def _is_java_gav_version_mismatch(requested: str, match: str) -> bool: + """Check if requested and match are the same Java GAV but with different versions.""" + req_parts = requested.split(":") + match_parts = match.split(":") + if len(req_parts) == 3 and len(match_parts) == 3: + return req_parts[0] == match_parts[0] and req_parts[1] == match_parts[1] and req_parts[2] != match_parts[2] + return False + + @staticmethod + def _find_java_gav_auto_resolve(requested: str, matches: list[str]) -> str | None: + """If requested is group:artifact (no version), find a match with the same group:artifact. + Returns the full GAV if exactly one match found, None otherwise.""" + req_parts = requested.split(":") + if len(req_parts) != 2: + return None + resolved = [m for m in matches if m.startswith(requested + ":") and len(m.split(":")) == 3] + if len(resolved) == 1: + return resolved[0] + return None + + @staticmethod + def _find_go_common_root_auto_resolve(package: str, matches: list[str]) -> str | None: + """For Go, find the longest common module root (at / boundaries) between + the input package and any close match. Returns the common root, not the + sibling itself, so downstream get_package_name filtering works correctly. + Requires at least 2 common path segments to avoid false matches.""" + best_root = None + best_common = 0 + for m in matches: + pkg_parts = package.lower().split("/") + m_parts = m.lower().split("/") + common = 0 + for p, mp in zip(pkg_parts, m_parts): + if p == mp: + common += 1 + else: + break + if common >= 2 and common > best_common: + best_root = "/".join(m.split("/")[:common]) + best_common = common + return best_root + + def _build_java_package_error(self, package: str, close_matches: list[str]) -> str: + """Build error message for Java when package is not found (after auto-resolve already failed). + Detects version mismatches and returns a CONCLUSION or generic error.""" + version_mismatches = [m for m in close_matches if self._is_java_gav_version_mismatch(package, m)] + if version_mismatches: + installed_versions = ', '.join(version_mismatches) + return ( + f"CONCLUSION: The requested version '{package}' is NOT installed. " + f"The installed version is: {installed_versions}. " + f"Since the vulnerable version is not present, the application is NOT affected by this CVE. " + f"Do NOT search with the installed version — report that the vulnerable version is absent." + ) + return ( + f"ERROR: Package '{package}' not found in available packages. " + f"Close matches are: {', '.join(close_matches)}. " + f"Please use one of the suggested package names. " + f"Available ecosystem: JAVA" + ) + + def handle_package_not_in_supported_packages(self, package: str) -> list[str]: + logger.info("package %s is not in supported packages", package) + if self.coc_retriever.supported_packages is None: + return [] + pkg_close_matches = difflib.get_close_matches(package, self.coc_retriever.supported_packages, n=10, cutoff=0.5) + std_close_matches = self.stdlib_cache.get_close_match_list(package, self.coc_retriever.ecosystem) + if std_close_matches: + logger.info("package %s is not in supported packages, but close matches are: %s", package, std_close_matches) + return std_close_matches + if pkg_close_matches: + logger.info("package %s is not in supported packages, but close matches are: %s", package, pkg_close_matches) + return pkg_close_matches + else: + return [] + + def python_flow_control(self, input_function: str, package_docs) -> list[str]: + """ + Python flow control + Args: + input_function: The function name to search for + package_docs: The package documents to search in + Returns: + List of function names in format 'package,function_name' that match the search term, + or error message with package suggestions if package is not found + """ + count_of_dots = min(input_function.count('.'), 2) + + list_of_matching_combinations = set() + for doc in package_docs: + if self.lang_parser: + function_name = self.lang_parser.get_function_name(doc) + module_name = doc.metadata.get('source').split('/')[-1].split('.')[0] + class_name = self.lang_parser.get_class_name_from_class_function(doc) + if count_of_dots == 0: + if function_name: + list_of_matching_combinations.add(f"{function_name}") + if class_name: + list_of_matching_combinations.add(f"{class_name}") + if count_of_dots == 1: + if module_name and class_name: + list_of_matching_combinations.add(f"{module_name}.{class_name}") + if module_name and function_name: + list_of_matching_combinations.add(f"{module_name}.{function_name}") + if class_name and function_name: + list_of_matching_combinations.add(f"{class_name}.{function_name}") + if count_of_dots == 2: + if module_name and class_name and function_name: + list_of_matching_combinations.add(f"{module_name}.{class_name}.{function_name}") + + close_matches = difflib.get_close_matches(input_function, list_of_matching_combinations, n=10, cutoff=0.6) + if len(close_matches) == 0: + close_matches = difflib.get_close_matches(input_function, list_of_matching_combinations, n=10, cutoff=0.3) + return close_matches + + def java_flow_control(self, input_function: str, package_docs) -> list[str]: + """ + Java flow control - uses fuzzy matching on ClassName.methodName candidates. + Handles class queries, method queries, and full ClassName.method queries uniformly. + """ + candidates = [] + for doc in package_docs: + if self.lang_parser: + cn = self.lang_parser.get_class_name_from_class_function(doc) + fn = self.lang_parser.get_function_name(doc) + if cn and fn: + simple_cn = cn.rsplit(".", 1)[-1] + candidates.append(f"{simple_cn}.{fn}") + candidates = list(dict.fromkeys(candidates)) + + matches = difflib.get_close_matches(input_function, candidates, n=10, cutoff=0.6) + if not matches: + matches = difflib.get_close_matches(input_function, candidates, n=10, cutoff=0.3) + if matches: + matches.append( + "INFO: For Call Chain Analyzer, use ClassName.methodName format " + "(e.g. PropertyUtilsBean.getProperty), not just the class name." + ) + return matches + + def common_flow_control(self,input_function: str, package_docs) -> list[str]: + """ + C/Go flow control have the same logic for finding close matches for function names in the package + Args: + input_function: The function name to search for + package_docs: The package documents to search in + Returns: + List of function names in format 'package,function_name' that match the search term, + or error message with package suggestions if package is not found + """ + package_functions = [] + for doc in package_docs: + if self.lang_parser: + function_name = self.lang_parser.get_function_name(doc) + if function_name == "": + continue + package_functions.append(f"{function_name}") + # Use fuzzy matching to find close matches + close_matches = difflib.get_close_matches(input_function, package_functions, n=10, cutoff=0.6) + if len(close_matches) == 0: + close_matches = difflib.get_close_matches(input_function, package_functions, n=10, cutoff=0.3) + logger.info("Close matches found: %s", close_matches) + return close_matches + + + def search_in_third_party_packages(self, package: str) -> bool: + """ + Search in third party packages + Args: + package: The package to search in + Returns: + True if package is found in third party packages, False otherwise + """ + for supported_package in self.coc_retriever.supported_packages: + if self.lang_parser.is_same_package(package, supported_package): + return True + + if package in (self.short_go_package_name or {}): + return True + if self.coc_retriever.ecosystem.value == Ecosystem.GO.value: + fqdn_match = self._resolve_go_fqdn_to_module(package) + if fqdn_match: + return True + return False + + async def locate_functions(self, query: str) -> list[str]: + """ + Locate function names in a package using fuzzy matching and validate package names. + Expected format: 'package_name,function_name' + + Args: + query: Input string in format 'package_name,function_name' + + Returns: + List of function names in format 'package,function_name' that match the search term, + or error message with package suggestions if package is not found + """ + query = query.splitlines()[0].replace('"', '').replace("'", "").replace("`", "").strip() + parts_input = query.split(",", 1) + self.is_std_package = False + self.is_package_valid = False + # Validate input format + if len(parts_input) < 2: + eco_key = self.coc_retriever.ecosystem.value if self.coc_retriever.ecosystem else "" + example = FL_EXAMPLES.get(eco_key, "'package_name,function_name'") + error_msg = ( + f"ERROR: Invalid input format. Expected format: 'package_name,function_name' " + f"but received: '{query}'. " + f"Please provide input in the correct format with a comma separating package name and function name. " + f"Example: {example}" + ) + logger.error(error_msg) + return [error_msg] # Return error message that LLM can see + + package = parts_input[0] + function = parts_input[1] + + try: + is_error = False + logger.info("Package and Function Locator: searching for (%s,%s)", package, function) + + # First validate package name exists in supported packages + if (not self.search_in_third_party_packages(package)): + logger.info("Package '%s' not found in supported packages", package) + + in_sbom, sbom_version = self.check_package_in_sbom(package) + if in_sbom: + self.is_package_valid = True + logger.info("Package '%s' (version %s) found in SBOM but no source code available", package, sbom_version) + return [ + ( + f"INFO: Package '{package}' (version {sbom_version}) is present in the container SBOM, " + f"however no source code is available so package code content and function names cannot be checked." + ) + ] + + fuzzy_in_sbom, fuzzy_matches = self.check_fuzzy_match_in_sbom(package) + if fuzzy_in_sbom: + logger.info("Package '%s' not found exactly in SBOM, but fuzzy matches found: %s", package, fuzzy_matches) + return [ + ( + f"WARNING: Package '{package}' was NOT found in the container. " + f"Similar SBOM packages exist: {fuzzy_matches}. The function name could NOT be verified. Do NOT treat this as a confirmed match." + + ) + ] + + is_standard_lib = self.stdlib_cache.is_standard_library(package, self.coc_retriever.ecosystem) + if is_standard_lib: + self.is_std_package = True + self.is_package_valid = True + return [ + ( + f"INFO: Package '{package}' is a standard library package. " + f"make call with the Transitive code search tool" + ) + ] + else: + if (self.coc_retriever.ecosystem and + self.coc_retriever.ecosystem.value != Ecosystem.JAVA.value): + is_standard_lib_api,is_error = await quick_standard_lib_check(package, self.coc_retriever.ecosystem) + if is_standard_lib_api: + self.is_std_package = True + self.is_package_valid = True + self.stdlib_cache.add_to_cache(package, self.coc_retriever.ecosystem) + return [ + ( + f"INFO: Package '{package}' is a standard library package. " + f"make call with the Transitive code search tool" + )] + close_package_matches = self.handle_package_not_in_supported_packages(package) + error_msg = "" + if close_package_matches: + is_java = (self.coc_retriever.ecosystem and + self.coc_retriever.ecosystem.value == Ecosystem.JAVA.value) + if is_java: + resolved_package = self._find_java_gav_auto_resolve(package, close_package_matches) + if resolved_package: + logger.info("Auto-resolved '%s' to '%s'", package, resolved_package) + package = resolved_package + self.is_package_valid = True + else: + error_msg = self._build_java_package_error(package, close_package_matches) + else: + is_go = (self.coc_retriever.ecosystem and + self.coc_retriever.ecosystem.value == Ecosystem.GO.value) + if is_go: + resolved_root = self._find_go_common_root_auto_resolve(package, close_package_matches) + if resolved_root: + logger.info("Go auto-resolved '%s' to common root '%s'", package, resolved_root) + package = resolved_root + self.is_package_valid = True + if not self.is_package_valid: + error_msg = ( + f"ERROR: Package '{package}' not found in available packages. " + f"Close matches are: {', '.join(close_package_matches)}. " + f"Please use one of the suggested package names. " + f"Available ecosystem: " + f"{self.coc_retriever.ecosystem.name if self.coc_retriever.ecosystem else 'Unknown'}" + ) + else: + if not is_error: + error_msg = ( + f"ERROR: Package '{package}' not found in available packages. " + f"No close matches found. " + f"Available ecosystem: " + f"{self.coc_retriever.ecosystem.name if self.coc_retriever.ecosystem else 'Unknown'}" + ) + else: + error_msg = "UNKNOWN could not determine if package is standard library" + if not self.is_package_valid: + logger.error(error_msg) + return [error_msg] + + # Package is valid, proceed with function search + self.is_package_valid = True + package_docs = [] + if (self.coc_retriever.ecosystem and + self.coc_retriever.ecosystem.value == Ecosystem.C_CPP.value): + package_docs = self.coc_retriever.sort_docs[package] + else: + # package_docs = self.coc_retriever.documents_of_functions + if package in self.short_go_package_name: + package = self.short_go_package_name[package] + elif self.coc_retriever.ecosystem.value == Ecosystem.GO.value: + resolved = self._resolve_go_fqdn_to_module(package) + if resolved: + package = resolved + package_docs = [ + doc + for doc in (self.coc_retriever.documents_of_functions or []) + if ( + self.lang_parser + and self.lang_parser.get_package_name(doc, package) + ) + ] + + if not package_docs: + logger.warning("Package '%s' is valid but no source code is indexed", package) + return [ + f"INFO: Package '{package}' is in the dependency tree but no source code is indexed for it. " + f"Function names cannot be verified. Proceed directly with Call Chain Analyzer using " + f"the function name from the CVE description." + ] + + match self.coc_retriever.ecosystem.value: + case Ecosystem.PYTHON.value: + return self.python_flow_control(function, package_docs) + case Ecosystem.C_CPP.value: + return self.common_flow_control(function, package_docs) + case Ecosystem.GO.value: + return self.common_flow_control(function, package_docs) + case Ecosystem.JAVA.value: + return self.java_flow_control(function, package_docs) + case _: + return self.common_flow_control(function, package_docs) + + + except KeyError: + logger.warning("Package '%s' not found in available packages", package) + close_package_matches = self.handle_package_not_in_supported_packages(package) + if close_package_matches: + error_msg = ( + f"ERROR: Package '{package}' not found in available packages. " + f"Close matches are: {', '.join(close_package_matches)}. " + f"Please use one of the suggested package names." + ) + else: + error_msg = ( + f"ERROR: Package '{package}' not found in available packages. " + f"No close matches found." + ) + return [error_msg] + except Exception as e: + logger.error("Error locating functions in package '%s': %s", package, e) + return [] + +async def quick_standard_lib_check(package_name: str, ecosystem: Ecosystem) -> tuple[bool, bool]: + """Quick check if package is standard library + Args: + package_name: The package name to check + ecosystem: The ecosystem to check + Returns: + True if package is standard library, False otherwise + """ + try: + search = MorpheusSerpAPIWrapper(max_retries=2) + result = await search.arun(f"Is '{package_name}' part of the {ecosystem.value} standard library?") + logger.info("quick_standard_lib_check Standard library check result: %s", result) + text = str(result).lower() + + # Basic positive signals and avoidance of negative phrasing + if "part of the standard library" in text or \ + "is a standard library" in text or \ + ("standard library" in text and "not" not in text): + return True, False + + return False, False + except Exception as e: + logger.error("Error checking if package is standard library: %s", e) + return False, True diff --git a/src/vuln_analysis/utils/gerrit_client.py b/src/vuln_analysis/utils/gerrit_client.py new file mode 100644 index 000000000..7a32489dd --- /dev/null +++ b/src/vuln_analysis/utils/gerrit_client.py @@ -0,0 +1,362 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Gerrit API client for Chromium bug resolution. + +This module provides functions to: +- Search Chromium Gerrit for CLs associated with a bug ID +- Filter for MERGED CLs only +- Select the correct CL when multiple exist (via LLM) +- Get commit SHA from a CL for Gitiles patch fetching +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import aiohttp +from pydantic import BaseModel + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.utils.async_http_utils import request_with_retry + +if TYPE_CHECKING: + from langchain_core.language_models import BaseChatModel + +logger = LoggingFactory.get_agent_logger(__name__) + +# Chromium Gerrit API base URL +GERRIT_BASE_URL = "https://chromium-review.googlesource.com" +GERRIT_TIMEOUT_SECONDS = 30 + + +# --------------------------------------------------------------------------- +# Data Models +# --------------------------------------------------------------------------- + +class GerritChangeCandidate(BaseModel): + """Represents a candidate Gerrit CL for LLM selection.""" + submission_id: int # _number field from Gerrit API + project: str # e.g., "angle/angle", "chromium/src" + subject: str # Commit subject line + + +class GerritChangeSelection(BaseModel): + """LLM response for selecting the correct Gerrit CL.""" + submission_id: int | None # None = no matching fix found + reason: str # Explanation for the selection + + +# --------------------------------------------------------------------------- +# Response Parsing +# --------------------------------------------------------------------------- + +def parse_gerrit_response(text: str) -> list[dict] | dict: + """Parse Gerrit API response, stripping the XSSI prevention prefix. + + Gerrit API responses start with ")]}'" to prevent XSSI attacks. + + Args: + text: Raw response text from Gerrit API + + Returns: + Parsed JSON (list for search results, dict for single item endpoints) + """ + # Strip the XSSI prefix if present + if text.startswith(")]}'"): + text = text[4:].lstrip() + + return json.loads(text) + + +def list_merged_changes(raw_changes: list[dict]) -> list[GerritChangeCandidate]: + """Filter Gerrit changes to only MERGED status. + + Args: + raw_changes: List of change dicts from Gerrit API + + Returns: + List of GerritChangeCandidate for MERGED CLs only + """ + candidates = [] + has_seen = set() + dates = [] + for change in raw_changes: + if change.get("status") == "MERGED": + if change.get("_number") not in has_seen: + + candidates.append(GerritChangeCandidate( + submission_id=change.get("_number", 0), + project=change.get("project", ""), + subject=change.get("subject", ""), + )) + has_seen.add(change.get("_number")) + dates.append(change.get("updated")) + return candidates + + +# --------------------------------------------------------------------------- +# Gerrit API Functions +# --------------------------------------------------------------------------- + +async def search_changes_by_bug( + session: aiohttp.ClientSession, + bug_id: str, + n: int = 25, +) -> list[dict]: + """Search Gerrit for CLs associated with a Chromium bug ID. + + Args: + session: aiohttp session + bug_id: Chromium bug ID (digits only) + n: Maximum number of results to return + + Returns: + List of change dicts from Gerrit API + """ + url = f"{GERRIT_BASE_URL}/changes/" + params = { + "q": f"bug:{bug_id}", + "n": str(n), + } + + timeout = aiohttp.ClientTimeout(total=GERRIT_TIMEOUT_SECONDS) + try: + async with request_with_retry( + session=session, + request_kwargs={ + "method": "GET", + "url": url, + "params": params, + "timeout": timeout, + }, + max_retries=3, + sleep_time=0.5, + log_on_error=False, + ) as response: + text = await response.text() + data = parse_gerrit_response(text) + # Search endpoint returns a list + return data if isinstance(data, list) else [] + except aiohttp.ClientResponseError as e: + logger.warning("Gerrit search failed for bug %s: %s", bug_id, e) + return [] + except Exception as e: + logger.warning("Gerrit search error for bug %s: %s", bug_id, e) + return [] + + +async def get_current_commit_sha( + session: aiohttp.ClientSession, + change_number: int, +) -> str | None: + """Get the commit SHA for the current revision of a Gerrit CL. + + Args: + session: aiohttp session + change_number: Gerrit change number (_number field) + + Returns: + Commit SHA string, or None if not found + """ + url = f"{GERRIT_BASE_URL}/changes/{change_number}/revisions/current/commit" + + timeout = aiohttp.ClientTimeout(total=GERRIT_TIMEOUT_SECONDS) + try: + async with request_with_retry( + session=session, + request_kwargs={ + "method": "GET", + "url": url, + "timeout": timeout, + }, + max_retries=3, + sleep_time=0.5, + log_on_error=False, + ) as response: + text = await response.text() + data = parse_gerrit_response(text) + if isinstance(data, dict): + return data.get("commit") + return None + except aiohttp.ClientResponseError as e: + logger.warning("Failed to get commit SHA for CL %d: %s", change_number, e) + return None + except Exception as e: + logger.warning("Error getting commit SHA for CL %d: %s", change_number, e) + return None + + +# --------------------------------------------------------------------------- +# Gitiles URL Building +# --------------------------------------------------------------------------- + +def project_to_gitiles_repo_url(project: str) -> str: + """Convert Gerrit project path to Gitiles repository URL. + + Args: + project: Gerrit project path (e.g., "angle/angle", "chromium/src") + + Returns: + Full Gitiles repository URL + """ + return f"https://chromium.googlesource.com/{project}" + + +def build_gitiles_patch_url(repo_url: str, sha: str) -> str: + """Build a Gitiles URL for fetching a patch in TEXT format. + + The ^! suffix means "this commit only" (not including parents). + format=TEXT returns base64-encoded patch content. + + Args: + repo_url: Gitiles repository URL + sha: Commit SHA + + Returns: + Gitiles patch URL + """ + # URL-encode the ^! as %5E%21 + return f"{repo_url}/+/{sha}%5E%21?format=TEXT" + + +# --------------------------------------------------------------------------- +# LLM Selection +# --------------------------------------------------------------------------- + +def build_gerrit_cl_select_prompt( + candidates: list[GerritChangeCandidate], + cve_id: str, + cve_description: str, +) -> str: + """Build prompt for LLM to select the correct Gerrit CL. + + Args: + candidates: List of MERGED CL candidates + cve_id: CVE identifier + cve_description: CVE description for context + + Returns: + Formatted prompt string + """ + candidate_list = "\n".join( + f"- submission_id: {c.submission_id}, project: {c.project}, subject: {c.subject}" + for c in candidates + ) + + return f"""You are a security researcher analyzing Chromium CLs to find the fix for a CVE. + +CVE: {cve_id} +Description: {cve_description} + +The following MERGED Chromium CLs are associated with the bug tracker issue. +Select the one that implements the security fix. + +Candidates: +{candidate_list} + +Guidelines: +1. REJECT CLs with subjects starting with "Roll" - these are dependency updates, not fixes +2. REJECT CLs from "chromium/src" that only update DEPS - these are roll commits +3. PREFER CLs from upstream component projects (angle/angle, WebKit, etc.) + when the CVE mentions ANGLE, WebKit, etc. +4. Look for CLs with subjects mentioning the fix (bounds checks, memory handling, etc.) +5. If none of the CLs appear to be the actual security fix, return submission_id: null + +Return ONLY the submission_id of the best matching CL, or null if none are appropriate.""" + + +async def select_gerrit_change( + candidates: list[GerritChangeCandidate], + cve_id: str, + cve_description: str, + llm: "BaseChatModel | None" = None, +) -> int | None: + """Select the correct Gerrit CL from multiple MERGED candidates. + + Args: + candidates: List of MERGED CL candidates + cve_id: CVE identifier + cve_description: CVE description for context + llm: Optional LangChain LLM (base model, not pre-configured with structured output) + + Returns: + submission_id of selected CL, or None if no valid selection + """ + if not candidates: + return None + + if len(candidates) == 1: + # Only one candidate - return it directly + return candidates[0].submission_id + + if llm is None: + # No LLM available - apply heuristics + # Filter out Roll commits and chromium/src DEPS updates + non_roll = [ + c for c in candidates + if not c.subject.lower().startswith("roll ") + and not (c.project == "chromium/src" and "roll" in c.subject.lower()) + ] + + if len(non_roll) == 1: + return non_roll[0].submission_id + + # Prefer upstream component projects over chromium/src + upstream = [c for c in non_roll if c.project != "chromium/src"] + if len(upstream) == 1: + return upstream[0].submission_id + + # Can't decide without LLM - return None to fail safely + logger.warning( + "Multiple MERGED CLs for %s, no LLM available for selection: %s", + cve_id, + [c.submission_id for c in candidates] + ) + return None + + # Use LLM for selection with structured output + prompt = build_gerrit_cl_select_prompt(candidates, cve_id, cve_description) + + try: + # Configure LLM with structured output for GerritChangeSelection + gerrit_llm = llm.with_structured_output(GerritChangeSelection) + result = await gerrit_llm.ainvoke(prompt) + # with_structured_output guarantees the response type matches the schema + selection = result if isinstance(result, GerritChangeSelection) else GerritChangeSelection.model_validate(result) + + # Validate selection is in candidate list + if selection.submission_id is None: + logger.info("LLM returned no selection for %s: %s", cve_id, selection.reason) + return None + + valid_ids = {c.submission_id for c in candidates} + if selection.submission_id not in valid_ids: + logger.warning( + "LLM selected invalid submission_id %d for %s (valid: %s)", + selection.submission_id, cve_id, valid_ids + ) + return None + + logger.info( + "LLM selected CL %d for %s: %s", + selection.submission_id, cve_id, selection.reason + ) + return selection.submission_id + + except Exception as e: + logger.warning("LLM selection failed for %s: %s", cve_id, e) + return None diff --git a/src/vuln_analysis/utils/git_commit_searcher.py b/src/vuln_analysis/utils/git_commit_searcher.py new file mode 100644 index 000000000..994f6727b --- /dev/null +++ b/src/vuln_analysis/utils/git_commit_searcher.py @@ -0,0 +1,952 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Git Commit Searcher for CVE fix discovery. + +Searches git repositories for CVE fix commits using multiple strategies: +1. Revision search (SVN/git revision lookup) - highest confidence +2. Function search (function name in commit messages or code) + +Results are ranked by confidence and security keyword matching. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import TYPE_CHECKING + +import aiohttp + +from langchain_core.messages import SystemMessage +from pydantic import BaseModel, Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.functions.code_agent_graph_defs import ( + CONFIDENCE_FUNCTION_MESSAGE, + CONFIDENCE_FUNCTION_PICKAXE, + CONFIDENCE_REVISION_BRANCH, + CONFIDENCE_REVISION_DIRECT, + CONFIDENCE_REVISION_FALLBACK, + SEARCH_METHOD_FUNCTION_MESSAGE, + SEARCH_METHOD_FUNCTION_PICKAXE, + SEARCH_METHOD_REVISION, + CommitSearchResult, + GitSearchReport, + ParsedPatch, + ReferenceHints, +) +from vuln_analysis.utils.web_patch_fetcher import ( + WebPatchFetcher, + _parse_patch_content, + build_patch_url_from_repo, +) +from vuln_analysis.utils.git_repo_manager import ( + GIT_HASH_LENGTH_SHORT, + GitCommandError, + GitRepoManager, +) + +if TYPE_CHECKING: + from langchain_core.language_models import BaseChatModel + +logger = LoggingFactory.get_agent_logger(__name__) + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +MAX_RESULTS_PER_STRATEGY = 5 +DATE_RANGE_MONTHS = 12 + +GIT_HASH_FULL_PATTERN = re.compile(r"^[a-f0-9]{40}$") +GIT_HASH_SHORT_PATTERN = re.compile(r"^[a-f0-9]{7,12}$") +SVN_REVISION_PATTERN = re.compile(r"^r?(\d{5,8})$") + +GIT_LOG_FORMAT = "%H\x1f%h\x1f%s\x1f%an <%ae>\x1f%aI" +GIT_LOG_SEPARATOR = "\x1f" # ASCII Unit Separator - won't appear in commit messages + + +class _NoOpSpan: + """No-op context manager when tracer is not available.""" + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + def set_output(self, data): + pass + + +SECURITY_KEYWORDS = [ + ("fix", 0.05), + ("security", 0.10), + ("vulnerab", 0.10), + ("overflow", 0.08), + ("injection", 0.08), + ("sanitiz", 0.05), + ("validat", 0.05), + ("CVE", 0.05), + ("crash", 0.03), + ("denial", 0.05), + ("remote", 0.05), + ("arbitrary", 0.08), +] + +MERGE_COMMIT_PENALTY = 0.10 +MAX_CONFIDENCE = 1.0 + + +# --------------------------------------------------------------------------- +# Branch Selection (LLM-assisted) +# --------------------------------------------------------------------------- + +BRANCH_SELECTION_PROMPT = """You are selecting which git branch(es) to search for a CVE fix commit. + +## Context + +Package: {package_name} +CVE Description: {cve_description} +Fixed Version (from advisory): {version_hints} + +## Available Branches in Repository + +{available_branches} + +## Task + +Select the branch(es) most likely to contain the fix commit for this CVE. + +Consider: +1. Version hints - "2.4.68" likely means branch "2.4.x" or "release/2.4" +2. Stable/release branches are more likely than development branches +3. If multiple versions affected, select the primary stable branch + +## Output + +Return JSON with selected_branches (at most 2) and reasoning. +""" + + +class BranchSelection(BaseModel): + """LLM output for branch selection.""" + + selected_branches: list[str] = Field( + description="Branch names to search (at most 2)", + ) + reasoning: str = Field( + description="Why these branches were selected", + ) + + +# --------------------------------------------------------------------------- +# GitCommitSearcher +# --------------------------------------------------------------------------- + + +class GitCommitSearcher: + """Search git repositories for CVE fix commits. + + Implements multiple search strategies ordered by confidence: + 1. Revision search (direct hash or SVN revision grep) + 2. Function search (commit message or pickaxe) + """ + + def __init__( + self, + repo_manager: GitRepoManager, + max_results_per_strategy: int = MAX_RESULTS_PER_STRATEGY, + tracer=None, + ): + self.repo_manager = repo_manager + self.max_results = max_results_per_strategy + self._tracer = tracer + + def _trace(self, name: str, input_data=None): + """Create a tracing span if tracer is available, otherwise return a no-op context manager.""" + if self._tracer: + return self._tracer.push_active_function(name, input_data=input_data or {}) + return _NoOpSpan() + + async def search( + self, + repo_path: Path, + repo_url: str, + hints: ReferenceHints | None, + cve_date: str | None = None, + branches: list[str] | None = None, + llm: "BaseChatModel | None" = None, + package_name: str | None = None, + cve_description: str | None = None, + ) -> GitSearchReport: + """Execute all search strategies and return best results. + + Args: + repo_path: Path to cloned repository + repo_url: Original repository URL + hints: Extracted hints from reference mining + cve_date: CVE disclosure date for date filtering + branches: Specific branches to search (from hints) + llm: LLM for branch resolution (optional) + package_name: Package name for branch resolution context + cve_description: CVE description for branch resolution context + + Returns: + GitSearchReport with ranked results + """ + report = GitSearchReport( + repo_url=repo_url, + repo_path=str(repo_path), + ) + + if not hints: + logger.warning("search: No hints provided, cannot search") + report.error_messages.append("No hints provided for search") + return report + + resolved_branches = await self._resolve_branches( + repo_path=repo_path, + repo_url=repo_url, + hints=hints, + branches=branches, + llm=llm, + package_name=package_name, + cve_description=cve_description, + ) + report.branches_searched = resolved_branches + + all_results: list[CommitSearchResult] = [] + + if hints.revision_hint: + report.strategies_tried.append(SEARCH_METHOD_REVISION) + result = await self._search_by_revision( + repo_path=repo_path, + revision_hint=hints.revision_hint, + branches=resolved_branches, + repo_url=repo_url, + ) + if result: + report.strategies_succeeded.append(SEARCH_METHOD_REVISION) + all_results.append(result) + + if hints.function_hints: + report.strategies_tried.append(SEARCH_METHOD_FUNCTION_MESSAGE) + report.strategies_tried.append(SEARCH_METHOD_FUNCTION_PICKAXE) + + results = await self._search_by_functions( + repo_path=repo_path, + function_hints=hints.function_hints, + cve_date=cve_date, + ) + if results: + for r in results: + if r.search_method == SEARCH_METHOD_FUNCTION_MESSAGE: + if SEARCH_METHOD_FUNCTION_MESSAGE not in report.strategies_succeeded: + report.strategies_succeeded.append(SEARCH_METHOD_FUNCTION_MESSAGE) + elif r.search_method == SEARCH_METHOD_FUNCTION_PICKAXE: + if SEARCH_METHOD_FUNCTION_PICKAXE not in report.strategies_succeeded: + report.strategies_succeeded.append(SEARCH_METHOD_FUNCTION_PICKAXE) + all_results.extend(results) + + ranked_results = self._rank_results( + results=all_results, + file_hints=hints.file_hints, + function_hints=hints.function_hints, + ) + report.all_results = ranked_results[: self.max_results] + + if ranked_results: + report.best_result = ranked_results[0] + report.search_success = True + logger.info( + "search: Found %d results, best: %s (confidence=%.2f, method=%s)", + len(ranked_results), + report.best_result.commit_hash_short, + report.best_result.confidence, + report.best_result.search_method, + ) + else: + logger.info("search: No commits found") + + return report + + async def _resolve_branches( + self, + repo_path: Path, + repo_url: str, + hints: ReferenceHints, + branches: list[str] | None, + llm: "BaseChatModel | None", + package_name: str | None, + cve_description: str | None, + ) -> list[str]: + """Resolve which branches to search. + + Priority: + 1. Explicit branches parameter + 2. branch_hints from Reference Mining + 3. LLM-selected branches (if LLM provided) + 4. Default branches (main, master, trunk) + """ + with self._trace("resolve_branches", { + "has_explicit_branches": bool(branches), + "has_branch_hints": bool(hints.branch_hints), + "has_version_hints": bool(hints.version_hints), + "has_llm": bool(llm), + }) as span: + if branches: + logger.debug("Using explicit branches: %s", branches) + span.set_output({"resolution": "explicit", "branches": branches}) + return branches + + if llm and (hints.version_hints or hints.branch_hints): + selected = await self._llm_select_branches( + repo_url=repo_url, + version_hints=hints.version_hints or [], + llm=llm, + package_name=package_name, + cve_description=cve_description, + branch_hints=hints.branch_hints, + ) + if selected: + span.set_output({"resolution": "llm_or_exact_match", "branches": selected}) + return selected + + default_branch = await self.repo_manager.get_default_branch(repo_path) + defaults = [default_branch] + if default_branch not in ("main", "master"): + defaults.extend(["main", "master"]) + + logger.debug("Using default branches: %s", defaults) + span.set_output({"resolution": "default", "branches": defaults}) + return defaults + + async def _llm_select_branches( + self, + repo_url: str, + version_hints: list[str], + llm: "BaseChatModel", + package_name: str | None, + cve_description: str | None, + branch_hints: list[str] | None = None, + ) -> list[str]: + """Use LLM to select best branches from available options. + + Fast path: If branch_hints exactly match available branches, return + them directly without LLM invocation. + """ + with self._trace("select_branches", { + "branch_hints": branch_hints, + "version_hints": version_hints, + }) as span: + try: + available_branches = await self.repo_manager.list_remote_branches(repo_url) + if not available_branches: + logger.debug("No remote branches found for branch selection") + span.set_output({"result": "no_branches_available"}) + return [] + + available_set = set(available_branches) + + if branch_hints: + exact_matches = [b for b in branch_hints if b in available_set] + if exact_matches: + logger.info( + "Branch hints exactly matched available branches: %s", + exact_matches, + ) + span.set_output({ + "method": "exact_match", + "branches": exact_matches[:2], + "available_count": len(available_branches), + }) + return exact_matches[:2] + logger.debug( + "Branch hints %s did not match available branches, falling back to LLM", + branch_hints, + ) + + branches_text = "\n".join(f"- {b}" for b in available_branches[:50]) + + prompt = BRANCH_SELECTION_PROMPT.format( + package_name=package_name or "unknown", + cve_description=(cve_description or "")[:500], + version_hints=", ".join(version_hints), + available_branches=branches_text, + ) + + branch_llm = llm.with_structured_output(BranchSelection) + result = await branch_llm.ainvoke([SystemMessage(content=prompt)]) + if not isinstance(result, BranchSelection): + logger.warning("LLM returned unexpected type: %s", type(result)) + span.set_output({"result": "llm_invalid_response", "type": str(type(result))}) + return [] + + valid_branches = [b for b in result.selected_branches if b in available_branches] + if valid_branches: + logger.info( + "LLM selected branches: %s (reason: %s)", + valid_branches, + result.reasoning[:100], + ) + span.set_output({ + "method": "llm", + "branches": valid_branches[:2], + "reasoning": result.reasoning[:200], + "available_count": len(available_branches), + }) + return valid_branches[:2] + + span.set_output({"result": "llm_no_valid_branches"}) + + except Exception as e: + logger.warning("LLM branch selection failed: %s", e) + span.set_output({"error": str(e)}) + + return [] + + async def _search_by_revision( + self, + repo_path: Path, + revision_hint: str, + branches: list[str], + repo_url: str | None = None, + ) -> CommitSearchResult | None: + """Search for a single SVN/Git revision. + + Strategy: + 1. If looks like git hash → git show + 2. If looks like SVN revision → git log --grep on branches + + Args: + repo_path: Path to the git repository. + revision_hint: Single revision string (e.g., "r1934982" or "abc123"). + branches: Branches to search on for SVN revisions. + repo_url: Repository URL for building commit URL in trace output. + + Returns: + CommitSearchResult if found, None otherwise. + """ + hint_clean = revision_hint.strip() + + with self._trace("search_by_revision", { + "revision_hint": hint_clean, + "branches": branches, + }) as span: + result: CommitSearchResult | None = None + hint_type: str | None = None + + if GIT_HASH_FULL_PATTERN.match(hint_clean) or GIT_HASH_SHORT_PATTERN.match(hint_clean): + hint_type = "git_hash" + result = await self._search_direct_hash(repo_path, hint_clean) + + elif SVN_REVISION_PATTERN.match(hint_clean): + hint_type = "svn_revision" + svn_match = SVN_REVISION_PATTERN.match(hint_clean) + if svn_match: + revision_num = svn_match.group(1) + branch_results = await self._search_revision_on_branches( + repo_path, revision_num, branches + ) + if branch_results: + result = branch_results[0] + + commit_url = None + if result and repo_url: + patch_url = build_patch_url_from_repo(repo_url, result.commit_hash) + if patch_url: + commit_url = patch_url.removesuffix(".patch") + + span.set_output({ + "revision_hint": hint_clean, + "hint_type": hint_type, + "found": result is not None, + "commit": result.commit_hash_short if result else None, + "commit_url": commit_url, + }) + + return result + + async def _search_direct_hash( + self, + repo_path: Path, + commit_hash: str, + ) -> CommitSearchResult | None: + """Direct lookup of a git commit hash.""" + try: + output = await self.repo_manager.run_git_command( + ["show", commit_hash, f"--format={GIT_LOG_FORMAT}", "-s"], + cwd=repo_path, + ) + + lines = output.strip().split("\n") + if not lines or not lines[0]: + return None + + parts = lines[0].split(GIT_LOG_SEPARATOR) + if len(parts) < 5: + return None + + files = await self._get_commit_files(repo_path, parts[0]) + + return CommitSearchResult( + commit_hash=parts[0], + commit_hash_short=parts[1], + commit_message=parts[2], + author=parts[3], + date=parts[4], + files_changed=files, + search_method=SEARCH_METHOD_REVISION, + confidence=CONFIDENCE_REVISION_DIRECT, + match_details=f"Direct hash lookup: {commit_hash}", + ) + + except GitCommandError as e: + logger.debug("Direct hash lookup failed for %s: %s", commit_hash, e.stderr) + return None + + async def _search_revision_on_branches( + self, + repo_path: Path, + revision_num: str, + branches: list[str], + ) -> list[CommitSearchResult]: + """Search commit messages for references to an SVN revision number. + + When projects migrate from SVN to Git, fix commits often reference + the original SVN revision in their message (e.g., "Backport fix from + r1234567"). Uses `git log --grep` to find such references. + """ + results: list[CommitSearchResult] = [] + + for branch in branches: + fetched = await self.repo_manager.fetch_branch(repo_path, branch) + if not fetched: + continue + + try: + output = await self.repo_manager.run_git_command( + [ + "log", "FETCH_HEAD", + f"--grep={revision_num}", + f"--format={GIT_LOG_FORMAT}", + "-n", str(self.max_results), + ], + cwd=repo_path, + ) + + for line in output.strip().split("\n"): + if not line: + continue + parts = line.split(GIT_LOG_SEPARATOR) + if len(parts) < 5: + continue + + files = await self._get_commit_files(repo_path, parts[0]) + + results.append(CommitSearchResult( + commit_hash=parts[0], + commit_hash_short=parts[1], + commit_message=parts[2], + author=parts[3], + date=parts[4], + files_changed=files, + search_method=SEARCH_METHOD_REVISION, + confidence=CONFIDENCE_REVISION_BRANCH, + match_details=f"grep matched r{revision_num} on branch {branch}", + )) + + if results: + logger.info("Found %d commits for r%s on branch %s", len(results), revision_num, branch) + return results + + except GitCommandError as e: + logger.debug("Revision search failed on branch %s: %s", branch, e.stderr) + + if not results: + results = await self._search_revision_all_branches(repo_path, revision_num) + + return results + + async def _search_revision_all_branches( + self, + repo_path: Path, + revision_num: str, + ) -> list[CommitSearchResult]: + """Fallback: search all branches for revision.""" + try: + output = await self.repo_manager.run_git_command( + [ + "log", "--all", + f"--grep={revision_num}", + f"--format={GIT_LOG_FORMAT}", + "-n", str(self.max_results), + ], + cwd=repo_path, + ) + + results = [] + for line in output.strip().split("\n"): + if not line: + continue + parts = line.split(GIT_LOG_SEPARATOR) + if len(parts) < 5: + continue + + files = await self._get_commit_files(repo_path, parts[0]) + + results.append(CommitSearchResult( + commit_hash=parts[0], + commit_hash_short=parts[1], + commit_message=parts[2], + author=parts[3], + date=parts[4], + files_changed=files, + search_method=SEARCH_METHOD_REVISION, + confidence=CONFIDENCE_REVISION_FALLBACK, + match_details=f"grep matched r{revision_num} on --all branches (fallback)", + )) + + return results + + except GitCommandError as e: + logger.debug("Revision search --all failed: %s", e.stderr) + return [] + + async def _search_by_functions( + self, + repo_path: Path, + function_hints: list[str], + cve_date: str | None, + ) -> list[CommitSearchResult]: + """Search by function name in commit messages or code changes.""" + with self._trace("search_by_functions", { + "function_hints": function_hints, + "cve_date": cve_date, + }) as span: + results: list[CommitSearchResult] = [] + hints_processed = [] + + for func_name in function_hints: + hint_info = {"function": func_name, "method": None, "found": False} + + message_results = await self._search_function_in_messages( + repo_path, func_name, cve_date + ) + if message_results: + hint_info["method"] = "message" + hint_info["found"] = True + hint_info["commits"] = [r.commit_hash_short for r in message_results] + results.extend(message_results) + else: + pickaxe_results = await self._search_function_pickaxe( + repo_path, func_name, cve_date + ) + if pickaxe_results: + hint_info["method"] = "pickaxe" + hint_info["found"] = True + hint_info["commits"] = [r.commit_hash_short for r in pickaxe_results] + results.extend(pickaxe_results) + + hints_processed.append(hint_info) + + span.set_output({ + "hints_processed": hints_processed, + "total_results": len(results), + }) + return results + + async def _search_function_in_messages( + self, + repo_path: Path, + function_name: str, + cve_date: str | None, + ) -> list[CommitSearchResult]: + """Search for function name in commit messages.""" + try: + args = [ + "log", + f"--grep={function_name}", + f"--format={GIT_LOG_FORMAT}", + "-n", str(self.max_results), + ] + + if cve_date: + args.extend(["--after", f"{cve_date[:10]} - {DATE_RANGE_MONTHS} months"]) + args.extend(["--before", f"{cve_date[:10]} + 3 months"]) + + output = await self.repo_manager.run_git_command(args, cwd=repo_path) + + results = [] + for line in output.strip().split("\n"): + if not line: + continue + parts = line.split(GIT_LOG_SEPARATOR) + if len(parts) < 5: + continue + + files = await self._get_commit_files(repo_path, parts[0]) + + results.append(CommitSearchResult( + commit_hash=parts[0], + commit_hash_short=parts[1], + commit_message=parts[2], + author=parts[3], + date=parts[4], + files_changed=files, + search_method=SEARCH_METHOD_FUNCTION_MESSAGE, + confidence=CONFIDENCE_FUNCTION_MESSAGE, + match_details=f"Message grep matched: {function_name}", + )) + + return results + + except GitCommandError as e: + logger.debug("Function message search failed for %s: %s", function_name, e.stderr) + return [] + + async def _search_function_pickaxe( + self, + repo_path: Path, + function_name: str, + cve_date: str | None, + ) -> list[CommitSearchResult]: + """Search for function name in code changes (pickaxe).""" + try: + args = [ + "log", + f"-S{function_name}", + f"--format={GIT_LOG_FORMAT}", + "-n", str(self.max_results), + ] + + if cve_date: + args.extend(["--after", f"{cve_date[:10]} - {DATE_RANGE_MONTHS} months"]) + args.extend(["--before", f"{cve_date[:10]} + 3 months"]) + + output = await self.repo_manager.run_git_command(args, cwd=repo_path) + + results = [] + for line in output.strip().split("\n"): + if not line: + continue + parts = line.split(GIT_LOG_SEPARATOR) + if len(parts) < 5: + continue + + files = await self._get_commit_files(repo_path, parts[0]) + + results.append(CommitSearchResult( + commit_hash=parts[0], + commit_hash_short=parts[1], + commit_message=parts[2], + author=parts[3], + date=parts[4], + files_changed=files, + search_method=SEARCH_METHOD_FUNCTION_PICKAXE, + confidence=CONFIDENCE_FUNCTION_PICKAXE, + match_details=f"Pickaxe matched: {function_name}", + )) + + return results + + except GitCommandError as e: + logger.debug("Function pickaxe search failed for %s: %s", function_name, e.stderr) + return [] + + async def _get_commit_files( + self, + repo_path: Path, + commit_hash: str, + ) -> list[str]: + """Get list of files changed in a commit.""" + try: + output = await self.repo_manager.run_git_command( + ["show", commit_hash, "--format=", "--name-only"], + cwd=repo_path, + ) + return [f for f in output.strip().split("\n") if f] + + except GitCommandError: + return [] + + def _rank_results( + self, + results: list[CommitSearchResult], + file_hints: list[str] | None, + function_hints: list[str] | None, + ) -> list[CommitSearchResult]: + """Rank results by confidence with boosting for matches.""" + if not results: + return [] + + seen_hashes: set[str] = set() + unique_results: list[CommitSearchResult] = [] + for r in results: + if r.commit_hash not in seen_hashes: + seen_hashes.add(r.commit_hash) + unique_results.append(r) + + for result in unique_results: + result.confidence = self._compute_boosted_confidence( + result, file_hints, function_hints + ) + + unique_results.sort(key=lambda r: r.confidence, reverse=True) + + return unique_results + + def _compute_boosted_confidence( + self, + result: CommitSearchResult, + file_hints: list[str] | None, + function_hints: list[str] | None, + ) -> float: + """Compute confidence with security keyword and hint boosting.""" + score = result.confidence + message_lower = result.commit_message.lower() + + for keyword, boost in SECURITY_KEYWORDS: + if keyword.lower() in message_lower: + score += boost + + if file_hints: + matched_files = set(result.files_changed) & set(file_hints) + score += 0.05 * len(matched_files) + + if function_hints: + for func in function_hints: + if func.lower() in message_lower: + score += 0.10 + + if message_lower.startswith("merge"): + score -= MERGE_COMMIT_PENALTY + + return min(score, MAX_CONFIDENCE) + + async def get_commit_diff( + self, + repo_path: Path, + commit_hash: str, + max_diff_size: int = 100000, + ) -> str: + """Get the full diff for a commit, truncated if too large.""" + try: + output = await self.repo_manager.run_git_command( + ["show", commit_hash, "--format=", "-p"], + cwd=repo_path, + ) + + if len(output) > max_diff_size: + return output[:max_diff_size] + "\n... (truncated)" + + return output + + except GitCommandError as e: + logger.warning("Failed to get diff for %s: %s", commit_hash, e.stderr) + return "" + + async def commit_to_parsed_patch( + self, + repo_path: Path, + commit_hash: str, + max_diff_size: int = 50000, + repo_url: str | None = None, + ) -> ParsedPatch | None: + """Convert a git commit to ParsedPatch format. + + This makes the discovered commit compatible with existing + L1 agent code that expects ParsedPatch from web_patch_fetcher. + + Strategy: + 1. Try HTTP fetch first (GitHub, etc.) - gets complete patch + 2. Fall back to git show (shallow clone may have incomplete diff) + + Args: + repo_path: Path to local repository + commit_hash: Full commit hash + max_diff_size: Maximum diff size in characters + repo_url: Repository URL (for HTTP fetch) + + Returns: + ParsedPatch or None if parsing fails + """ + patch_filename = f"git_search_{commit_hash[:GIT_HASH_LENGTH_SHORT]}.patch" + diff_text = None + + if repo_url: + diff_text = await self._fetch_patch_via_http(repo_url, commit_hash, max_diff_size) + if diff_text: + logger.info( + "Fetched patch for %s via HTTP (%d chars)", + commit_hash[:GIT_HASH_LENGTH_SHORT], + len(diff_text), + ) + + if not diff_text: + logger.debug("HTTP fetch unavailable, falling back to git show") + diff_text = await self.get_commit_diff(repo_path, commit_hash, max_diff_size) + + if not diff_text: + return None + + parsed = _parse_patch_content(diff_text, patch_filename) + + if parsed: + logger.info( + "Parsed commit %s into %d files", + commit_hash[:GIT_HASH_LENGTH_SHORT], + len(parsed.files), + ) + + return parsed + + async def _fetch_patch_via_http( + self, + repo_url: str, + commit_hash: str, + max_size: int, + ) -> str | None: + """Fetch patch content via HTTP from GitHub/GitLab. + + This bypasses shallow clone limitations by fetching directly from the web. + Uses WebPatchFetcher for robust fetching with retries and rate limiting. + """ + patch_url = build_patch_url_from_repo(repo_url, commit_hash) + if not patch_url: + logger.debug("Cannot build patch URL for %s", repo_url) + return None + + try: + timeout = aiohttp.ClientTimeout(total=30) + async with aiohttp.ClientSession(timeout=timeout) as session: + fetcher = WebPatchFetcher(session) + result = await fetcher.fetch_from_url( + url=patch_url, + cve_id="git_search", + source="git_search", + ) + if result and result.patch_content: + content = result.patch_content + if len(content) > max_size: + content = content[:max_size] + "\n... (truncated)" + return content + + except Exception as e: + logger.debug("HTTP fetch error for %s: %s", patch_url, e) + + return None diff --git a/src/vuln_analysis/utils/git_repo_manager.py b/src/vuln_analysis/utils/git_repo_manager.py new file mode 100644 index 000000000..ac06748fd --- /dev/null +++ b/src/vuln_analysis/utils/git_repo_manager.py @@ -0,0 +1,424 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Git Repository Manager for commit discovery. + +Manages git repository cloning, updating, and caching for CVE fix commit search. +Handles shallow cloning for performance, local caching to avoid re-cloning, +branch fetching for specific branches, and concurrent access with locking. +""" + +from __future__ import annotations + +import asyncio +import shutil +import subprocess +import uuid +from pathlib import Path +from urllib.parse import unquote, urlparse + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + +logger = LoggingFactory.get_agent_logger(__name__) + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +DEFAULT_CACHE_DIR = ".cache/am_cache/checker/upstream_repos" +DEFAULT_CLONE_DEPTH = 1000 +DEFAULT_FETCH_TIMEOUT_SECONDS = 120 +DEFAULT_CLONE_TIMEOUT_SECONDS = 180 + +GIT_HASH_LENGTH_SHORT = 7 +GIT_HASH_LENGTH_FULL = 40 +GIT_SUFFIX = ".git" +UNSAFE_PATH_SEGMENTS = frozenset({".", ".."}) + + +class GitCommandError(Exception): + """Raised when a git command fails.""" + + def __init__(self, command: str, returncode: int, stderr: str): + self.command = command + self.returncode = returncode + self.stderr = stderr + super().__init__(f"Git command failed: {command} (exit {returncode}): {stderr}") + + +class GitRepoManager: + """Manage git repository cloning, updating, and caching. + + Features: + - Shallow cloning for performance + - Local caching to avoid re-cloning + - Branch fetching for specific branches + - Concurrent access handling with asyncio locks + """ + + def __init__( + self, + cache_dir: str | Path = DEFAULT_CACHE_DIR, + default_depth: int = DEFAULT_CLONE_DEPTH, + fetch_timeout_seconds: int = DEFAULT_FETCH_TIMEOUT_SECONDS, + clone_timeout_seconds: int = DEFAULT_CLONE_TIMEOUT_SECONDS, + ): + self.cache_dir = Path(cache_dir) + self.default_depth = default_depth + self.fetch_timeout = fetch_timeout_seconds + self.clone_timeout = clone_timeout_seconds + self.cache_dir.mkdir(parents=True, exist_ok=True) + + self._repo_locks: dict[str, asyncio.Lock] = {} + self._locks_lock = asyncio.Lock() + + async def _get_repo_lock(self, repo_url: str) -> asyncio.Lock: + """Get or create a lock for a specific repo URL.""" + async with self._locks_lock: + if repo_url not in self._repo_locks: + self._repo_locks[repo_url] = asyncio.Lock() + return self._repo_locks[repo_url] + + def get_repo_cache_path(self, repo_url: str) -> Path: + """Get deterministic local path for a repo URL. + + Example: + https://github.com/apache/httpd + → .cache/am_cache/checker/upstream_repos/github.com/apache/httpd + """ + parsed = urlparse(repo_url) + host = (parsed.hostname or parsed.netloc or "unknown").lower() + if not host or "/" in host or "\\" in host: + raise ValueError(f"Invalid repository host in URL: {repo_url!r}") + + path_parts = [ + segment + for segment in unquote(parsed.path).strip("/").split("/") + if segment + ] + if any(segment in UNSAFE_PATH_SEGMENTS for segment in path_parts): + raise ValueError(f"Repository URL contains unsafe path segment: {repo_url!r}") + if not path_parts: + raise ValueError(f"Repository URL has no path: {repo_url!r}") + + if path_parts[-1].endswith(GIT_SUFFIX): + path_parts[-1] = path_parts[-1][: -len(GIT_SUFFIX)] + if not path_parts[-1]: + raise ValueError(f"Repository URL has empty repo name: {repo_url!r}") + + cache_path = self.cache_dir / host + for segment in path_parts: + cache_path /= segment + + cache_root = self.cache_dir.resolve() + if not cache_path.resolve(strict=False).is_relative_to(cache_root): + raise ValueError(f"Repository cache path escapes cache root: {repo_url!r}") + + return cache_path + + def _is_valid_repo(self, repo_path: Path) -> bool: + """Check if path contains a valid git repository.""" + git_dir = repo_path / ".git" + return git_dir.is_dir() and (git_dir / "HEAD").exists() + + async def run_git_command( + self, + args: list[str], + cwd: Path | None = None, + timeout: int | None = None, + ) -> str: + """Run a git command asynchronously. + + Uses subprocess.run with asyncio.to_thread to avoid Python 3.12/uvloop + child watcher incompatibility with asyncio.create_subprocess_exec. + + Args: + args: Git command arguments (without 'git' prefix) + cwd: Working directory for the command + timeout: Command timeout in seconds + + Returns: + Command stdout as string + + Raises: + GitCommandError: If command fails + asyncio.TimeoutError: If command times out + """ + cmd = ["git"] + args + cmd_str = " ".join(cmd) + effective_timeout = timeout or self.fetch_timeout + + logger.debug("Running git command: %s (cwd=%s)", cmd_str, cwd) + + def _run_sync() -> tuple[str, str, int]: + """Execute git command synchronously.""" + result = subprocess.run( + cmd, + cwd=cwd, + capture_output=True, + text=True, + errors="replace", + timeout=effective_timeout, + ) + return (result.stdout, result.stderr, result.returncode) + + try: + stdout, stderr, returncode = await asyncio.to_thread(_run_sync) + + if returncode != 0: + raise GitCommandError( + command=cmd_str, + returncode=returncode, + stderr=stderr, + ) + + return stdout + + except subprocess.TimeoutExpired: + logger.error("Git command timed out after %ds: %s", effective_timeout, cmd_str) + raise asyncio.TimeoutError(f"Git command timed out after {effective_timeout}s: {cmd_str}") + except Exception as e: + logger.error("Git command failed unexpectedly: %s", cmd_str, exc_info=True) + raise GitCommandError( + command=cmd_str, + returncode=-1, + stderr=str(e), + ) + + async def clone_or_update( + self, + repo_url: str, + depth: int | None = None, + ) -> tuple[Path, bool]: + """Clone a repo or update if it exists. + + Args: + repo_url: Git repository URL + depth: Clone depth (None for full clone) + + Returns: + (repo_path, is_fresh_clone) + """ + lock = await self._get_repo_lock(repo_url) + + async with lock: + cache_path = self.get_repo_cache_path(repo_url) + + if cache_path.exists() and self._is_valid_repo(cache_path): + logger.info("Repo exists in cache, fetching updates: %s", repo_url) + await self._git_fetch(cache_path, depth=depth) + return (cache_path, False) + + logger.info("Cloning repository: %s", repo_url) + temp_dir = self.cache_dir / ".tmp" / str(uuid.uuid4()) + + try: + await self._git_clone(repo_url, temp_dir, depth=depth) + cache_path.parent.mkdir(parents=True, exist_ok=True) + + if cache_path.exists(): + shutil.rmtree(cache_path, ignore_errors=True) + + temp_dir.rename(cache_path) + logger.info("Successfully cloned to: %s", cache_path) + return (cache_path, True) + + except Exception: + shutil.rmtree(temp_dir, ignore_errors=True) + raise + + async def _git_clone( + self, + repo_url: str, + target_dir: Path, + depth: int | None = None, + ) -> None: + """Clone a repository with optional shallow depth.""" + effective_depth = depth or self.default_depth + target_dir.parent.mkdir(parents=True, exist_ok=True) + + args = [ + "clone", + "--depth", str(effective_depth), + "--single-branch", + repo_url, + str(target_dir), + ] + + await self.run_git_command(args, timeout=self.clone_timeout) + + async def _git_fetch( + self, + repo_path: Path, + depth: int | None = None, + ) -> None: + """Fetch updates for an existing repository.""" + effective_depth = depth or self.default_depth + + args = ["fetch", "origin", f"--depth={effective_depth}"] + await self.run_git_command(args, cwd=repo_path) + + async def fetch_branch( + self, + repo_path: Path, + branch: str, + depth: int | None = None, + ) -> bool: + """Fetch a specific branch with depth limit. + + Critical for SVN revision search on stable branches. + + Example: + git fetch origin 2.4.x --depth=1000 + + Args: + repo_path: Path to local repository + branch: Branch name to fetch + depth: Fetch depth + + Returns: + True if fetch succeeded, False otherwise + """ + effective_depth = depth or self.default_depth + + try: + args = ["fetch", "origin", branch, f"--depth={effective_depth}"] + await self.run_git_command(args, cwd=repo_path) + logger.info("Fetched branch %s with depth %d", branch, effective_depth) + return True + + except GitCommandError as e: + logger.warning("Failed to fetch branch %s: %s", branch, e.stderr) + return False + + async def get_default_branch(self, repo_path: Path) -> str: + """Get the default branch name (main, master, trunk, etc.).""" + try: + output = await self.run_git_command( + ["symbolic-ref", "refs/remotes/origin/HEAD", "--short"], + cwd=repo_path, + ) + branch = output.strip() + if branch.startswith("origin/"): + branch = branch[len("origin/"):] + return branch + + except GitCommandError: + logger.debug("Could not get default branch via symbolic-ref, trying fallback") + + for candidate in ["main", "master", "trunk", "develop"]: + try: + await self.run_git_command( + ["rev-parse", "--verify", f"origin/{candidate}"], + cwd=repo_path, + ) + return candidate + except GitCommandError: + continue + + return "main" + + async def list_remote_branches(self, repo_url: str) -> list[str]: + """List available remote branches without cloning. + + Uses git ls-remote --heads to list branches. + + Args: + repo_url: Repository URL + + Returns: + List of branch names (without refs/heads/ prefix) + """ + try: + output = await self.run_git_command( + ["ls-remote", "--heads", repo_url], + ) + + branches = [] + for line in output.strip().split("\n"): + if not line: + continue + parts = line.split("\t") + if len(parts) >= 2: + ref = parts[1] + if ref.startswith("refs/heads/"): + branches.append(ref[len("refs/heads/"):]) + + logger.debug("Found %d branches for %s", len(branches), repo_url) + return branches + + except GitCommandError as e: + logger.warning("Failed to list remote branches: %s", e.stderr) + return [] + + async def list_local_branches(self, repo_path: Path) -> list[str]: + """List branches available in local repository. + + Args: + repo_path: Path to local repository + + Returns: + List of remote tracking branch names + """ + try: + output = await self.run_git_command( + ["branch", "-r", "--format=%(refname:short)"], + cwd=repo_path, + ) + + branches = [] + for line in output.strip().split("\n"): + branch = line.strip() + if branch and not branch.endswith("/HEAD"): + if branch.startswith("origin/"): + branch = branch[len("origin/"):] + branches.append(branch) + + return branches + + except GitCommandError as e: + logger.warning("Failed to list local branches: %s", e.stderr) + return [] + + def cleanup_cache(self, max_age_days: int = 7) -> int: + """Remove repos not accessed in N days. + + Args: + max_age_days: Maximum age in days before removal + + Returns: + Number of repos removed + """ + import time + + max_age_seconds = max_age_days * 24 * 60 * 60 + now = time.time() + removed_count = 0 + + if not self.cache_dir.exists(): + return 0 + + for repo_dir in self.cache_dir.rglob(".git"): + repo_path = repo_dir.parent + try: + mtime = repo_path.stat().st_mtime + if now - mtime > max_age_seconds: + logger.info("Removing stale cached repo: %s", repo_path) + shutil.rmtree(repo_path, ignore_errors=True) + removed_count += 1 + except OSError: + continue + + return removed_count diff --git a/src/vuln_analysis/utils/git_utils.py b/src/vuln_analysis/utils/git_utils.py deleted file mode 100644 index 87ede34a7..000000000 --- a/src/vuln_analysis/utils/git_utils.py +++ /dev/null @@ -1,53 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -from pathlib import Path -from pathlib import PurePath - -from git import Repo - - -def get_repo_from_path(base_dir: str, git_repo: str = ".git") -> Repo: - """ - Utility function for getting GitPython `Repo` object representing a Git repository. - - Parameters - ---------- - base_dir : str - Path to base directory containing one or more Git repos - git_repo : str - Relative path to Git repo in base_dir, default is ".git" - - Returns - ------- - repo - GitPython `Repo` object representing the Git repository. - - Raises - ------ - ValueError - If the repository path does not exist. - """ - repo_path = base_dir / PurePath(git_repo) - repo_path = Path(repo_path) - if os.path.exists(repo_path): - try: - return Repo(repo_path) - except Exception as e: - raise ValueError(f"Invalid Git repository: {repo_path}") from e - else: - raise ValueError(f"Path {repo_path} does not exist") diff --git a/src/vuln_analysis/utils/http_utils.py b/src/vuln_analysis/utils/http_utils.py index 8a0f4a088..f58f87ea6 100644 --- a/src/vuln_analysis/utils/http_utils.py +++ b/src/vuln_analysis/utils/http_utils.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import time import typing from enum import Enum @@ -22,7 +21,8 @@ import requests import urllib3 -logger = logging.getLogger(__name__) +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class MimeTypes(Enum): diff --git a/src/vuln_analysis/utils/intel_retriever.py b/src/vuln_analysis/utils/intel_retriever.py index 245452c0b..215629fb9 100644 --- a/src/vuln_analysis/utils/intel_retriever.py +++ b/src/vuln_analysis/utils/intel_retriever.py @@ -14,24 +14,27 @@ # limitations under the License. import asyncio -import logging import os from contextlib import asynccontextmanager import aiohttp -from ..data_models.cve_intel import CveIntel -from ..data_models.cve_intel import CveIntelEpss -from ..data_models.cve_intel import CveIntelNvd -from ..data_models.cve_intel import CveIntelRhsa -from ..data_models.cve_intel import CveIntelUbuntu +from exploit_iq_commons.data_models.cve_intel import CveIntel, IntelPluginData +from exploit_iq_commons.data_models.cve_intel import CveIntelEpss +from exploit_iq_commons.data_models.cve_intel import CveIntelNvd +from exploit_iq_commons.data_models.cve_intel import CveIntelOsidb +from exploit_iq_commons.data_models.cve_intel import CveIntelRhsa +from exploit_iq_commons.data_models.cve_intel import CveIntelUbuntu from .clients.first_client import FirstClient from .clients.ghsa_client import GHSAClient from .clients.nvd_client import NVDClient +from .clients.osidb_client import OsidbClient from .clients.rhsa_client import RHSAClient from .clients.ubuntu_client import UbuntuClient +from ..data_models.plugin import PluginConfig, IntelPluginSchema -logger = logging.getLogger(__name__) +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class IntelRetriever: @@ -45,13 +48,20 @@ def __init__(self, ghsa_api_key: str | None = None, lang_code: str = 'en', max_retries: int = 10, - retry_on_client_errors: bool = True): + plugins_config: list[PluginConfig] | None = None, + retry_on_client_errors: bool = True, + rpm_user_type: str = "external"): """ Initialize the NISTCVERetriever with URL templates for vulnerability and CVE details. + + Parameters + ---------- + rpm_user_type : str + User profile type: "internal" (Red Hat VPN) enables OSIDB, "external" disables it. """ - # Create a shared session object for all requests self._session = session + self._rpm_user_type = rpm_user_type self._nvd_client = NVDClient(api_key=os.environ.get('NVD_API_KEY', nist_api_key), session=self._session, @@ -72,6 +82,17 @@ def __init__(self, retry_count=max_retries, retry_on_client_errors=retry_on_client_errors) + self._osidb_client = None + if self._rpm_user_type == "internal": + self._osidb_client = OsidbClient(session=self._session, + retry_count=max_retries, + retry_on_client_errors=retry_on_client_errors) + + self._intel_plugins = [] + if plugins_config is not None: + self._intel_plugins = [IntelPluginSchema.locate( + plugin_name=cfg.plugin_name, session=session, config=cfg.plugin_config) for cfg in plugins_config] + @asynccontextmanager async def _get_session(self, session: aiohttp.ClientSession | None = None): @@ -161,6 +182,33 @@ async def _get_epss_score(self, intel: CveIntel) -> CveIntelEpss | None: return None + async def _get_osidb_intel(self, intel: CveIntel) -> CveIntelOsidb | None: + """Fetch OSIDB intel for internal Red Hat users.""" + if self._osidb_client is None: + return None + + if not intel.has_cve_id(): + logger.debug("Skipping OSIDB retrieval since '%s' does not have an associated CVE ID", intel.vuln_id) + return None + + try: + intel.osidb = await self._osidb_client.get_intel(cve_id=intel.cve_id) + return intel.osidb + + except Exception as e: + logger.error("Error fetching OSIDB intel for %s: %s", intel.vuln_id, e, exc_info=True) + return None + + async def append_plugin_data(self, plugin: IntelPluginSchema, intel: CveIntel) -> IntelPluginData: + + try: + intel.plugin_data.append(await plugin.retrieve(intel.vuln_id)) + return intel.plugin_data + + except Exception as e: + logger.exception("Error fetching data from plugin %s : %s", intel.vuln_id, e, exc_info=True) + return None + async def retrieve(self, vuln_id: str) -> CveIntel: """ Asynchronously retrieve all relevant details for a given CVE ID. @@ -186,7 +234,6 @@ async def retrieve(self, vuln_id: str) -> CveIntel: vuln_id) return intel - # Run all the coroutines concurrently coros = [ self._get_nvd_intel(intel=intel), self._get_ubuntu_intel(intel=intel), @@ -194,6 +241,11 @@ async def retrieve(self, vuln_id: str) -> CveIntel: self._get_epss_score(intel=intel) ] + if self._rpm_user_type == "internal": + coros.append(self._get_osidb_intel(intel=intel)) + + coros.extend([self.append_plugin_data(intel=intel, plugin=plugin) for plugin in self._intel_plugins]) + await asyncio.gather(*coros) return intel diff --git a/src/vuln_analysis/utils/intel_source_score.py b/src/vuln_analysis/utils/intel_source_score.py new file mode 100644 index 000000000..1c595344c --- /dev/null +++ b/src/vuln_analysis/utils/intel_source_score.py @@ -0,0 +1,299 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import re +from aiq.builder.builder import Builder +from langchain_core.language_models.base import BaseLanguageModel + +from exploit_iq_commons.data_models.cve_intel import CveIntel +from ..functions.cve_calculate_intel_score import CVECalculateIntelScoreConfig +from exploit_iq_commons.utils import data_utils +from ..utils.prompting import additional_intel_prompting +from aiq.builder.framework_enum import LLMFrameworkEnum + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) + +class IntelScorer: + def __init__(self, + config: CVECalculateIntelScoreConfig, + builder: Builder): + self._config = config + self._builder = builder + + async def calculate_intel_score(self, intel: CveIntel) -> CveIntel: + llm = await self._builder.get_llm(llm_name=self._config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN) + assert isinstance(llm, BaseLanguageModel) + + response = await llm.ainvoke(self.__get_calculate_score_prompt(intel)) + + if os.environ.get("EXTENDED_VERBOSE_DEBUG", False): + logger.debug("\nresponse: %s", str(response.content)) + + score = self.__extract_score(response.content) + + if os.environ.get("EXTENDED_VERBOSE_DEBUG", False): + logger.debug("\nScore: %d", score) + + intel.intel_score = score + + return intel + + def __get_calculate_score_prompt(self, intel: CveIntel) -> str: + return """ +Evaluate CVE intelligence quality by scoring each criterion independently. + + + +Provide individual scores for each criterion based on the CVE data below. + +1. technical_specificity (max 20 points) + - How precise and in-depth are the technical details? + - Are specific vulnerable functions, methods, or code paths identified? + +2. clarity (max 10 points) + - Is the text well-structured and grammatically correct? + - Is the description clear and easy to understand? + +3. component_impact (max 15 points) + - Does it clearly state what is affected? + - Are the consequences explicitly described? + +4. reproducibility (max 15 points) + - Could an attacker understand how to exploit this from the description? + - Are attack vectors and preconditions described? + +5. vulnerable_function (max 15 points) + - Is a specific function, method, or code snippet named? + - Are vulnerable code locations identifiable? + +6. mitigation (max 10 points) + - Are patches, workarounds, or mitigations described? + - Is remediation guidance provided? + +7. environment (max 10 points) + - Is there context about the affected environment (OS, version, configuration)? + - Are deployment scenarios mentioned? + +8. configuration (max 5 points) + - Are relevant configuration settings or misconfigurations described? + + + +Return JSON only (no markdown, no code blocks): +{ + "scores": { + "technical_specificity": <0-20>, + "clarity": <0-10>, + "component_impact": <0-15>, + "reproducibility": <0-15>, + "vulnerable_function": <0-15>, + "mitigation": <0-10>, + "environment": <0-10>, + "configuration": <0-5> + }, + "justifications": { + "technical_specificity": "brief reason for score", + "clarity": "brief reason for score", + "component_impact": "brief reason for score", + "reproducibility": "brief reason for score", + "vulnerable_function": "brief reason for score", + "mitigation": "brief reason for score", + "environment": "brief reason for score", + "configuration": "brief reason for score" + } +} + +Do NOT calculate or include a total_score. Only provide the individual criterion scores. + + + + +Example Input: +CVE ID: CVE-2025-30204 +CVE Description: golang-jwt is a Go implementation of JSON Web Tokens. Starting in version 3.2.0 and prior to versions 5.2.2 and 4.5.2, the function parse.ParseUnverified splits (via a call to strings.Split) its argument (which is untrusted data) on periods. As a result, in the face of a malicious request whose Authorization header consists of Bearer followed by many period characters, a call to that function incurs allocations to the tune of O(n) bytes (where n stands for the length of the function's argument), with a constant factor of about 16. This issue is fixed in 5.2.2 and 4.5.2. +CVSS Vector: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H +CWE Name: CWE-405: Asymmetric Resource Consumption (Amplification) +Notable Vendors: Hashicorp + +Example Output (High Quality ~80): +{ + "scores": { + "technical_specificity": 18, + "clarity": 9, + "component_impact": 14, + "reproducibility": 14, + "vulnerable_function": 15, + "mitigation": 9, + "environment": 9, + "configuration": 4 + }, + "justifications": { + "technical_specificity": "Function parse.ParseUnverified identified with O(n) allocation details", + "clarity": "Well-structured with clear impact", + "component_impact": "golang-jwt and DoS impact explicitly stated", + "reproducibility": "Clear attack: malicious Authorization header with periods", + "vulnerable_function": "parse.ParseUnverified explicitly named", + "mitigation": "Patches 5.2.2 and 4.5.2 specified", + "environment": "Versions 3.2.0-5.2.2 listed", + "configuration": "Limited config details" + } +} + +Example Input: +CVE ID: CVE-2022-29810 +CVE Description: The Hashicorp go-getter library before 1.5.11 does not redact an SSH key from a URL query parameter. +CVSS Vector: CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N +CWE Name: CWE-532: Insertion of Sensitive Information into Log File +Notable Vendors: Hashicorp + +Example Output (Medium Quality ~62): +{ + "scores": { + "technical_specificity": 12, + "clarity": 8, + "component_impact": 12, + "reproducibility": 10, + "vulnerable_function": 8, + "mitigation": 8, + "environment": 8, + "configuration": 3 + }, + "justifications": { + "technical_specificity": "Moderate detail about SSH key issue", + "clarity": "Clear but brief", + "component_impact": "go-getter and credential exposure stated", + "reproducibility": "Attack path somewhat clear, lacks details", + "vulnerable_function": "General functionality, no specific function", + "mitigation": "Version 1.5.11 patches", + "environment": "Affected versions specified", + "configuration": "Minimal context" + } +} + +Example Input: +CVE ID: CVE-2022-2385 +CVE Description: A security issue was discovered in aws-iam-authenticator where an allow-listed IAM identity may be able to modify their username and escalate privileges. +CVSS Vector: CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +CWE Name: CWE-20: Improper Input Validation +Notable Vendors: Kubernetes + +Example Output (Low Quality ~20): +{ + "scores": { + "technical_specificity": 3, + "clarity": 5, + "component_impact": 5, + "reproducibility": 2, + "vulnerable_function": 0, + "mitigation": 3, + "environment": 4, + "configuration": 0 + }, + "justifications": { + "technical_specificity": "Very vague, no detail", + "clarity": "Brief but understandable", + "component_impact": "General privilege escalation mention", + "reproducibility": "No exploit details", + "vulnerable_function": "No functions identified", + "mitigation": "Patch mentioned", + "environment": "Minimal version info", + "configuration": "None" + } +} + + + + +""" + self.__render_template(additional_intel_prompting, intel) + "\n\n\nProvide your scoring JSON:" + + def __extract_score(self, text: str) -> int: + """ + Extract individual scores from LLM response and calculate total. + This prevents LLM arithmetic hallucinations by: + 1. Validating each score against its maximum + 2. Calculating total in code (not LLM) + """ + text = text.replace("```", "").replace("json", "").strip() + if os.environ.get("EXTENDED_VERBOSE_DEBUG", False): + logger.debug("\nRaw LLM response: %s", str(text)) + + try: + data = json.loads(text) + + # Extract individual scores + scores = data.get('scores', {}) + + # Define maximum values for each criterion + max_values = { + 'technical_specificity': 20, + 'clarity': 10, + 'component_impact': 15, + 'reproducibility': 15, + 'vulnerable_function': 15, + 'mitigation': 10, + 'environment': 10, + 'configuration': 5 + } + + # Validate and cap each score at its maximum + validated_scores = {} + for criterion, max_val in max_values.items(): + score = scores.get(criterion, 0) + # Ensure score is within valid range [0, max] + validated_scores[criterion] = max(0, min(score, max_val)) + + if score > max_val: + logger.warning( + "Score for '%s' (%d) exceeds maximum (%d), capping to max", + criterion, score, max_val + ) + + # Calculate total using code (not LLM) + total_score = sum(validated_scores.values()) + + # Final safety check: total should not exceed 100 + total_score = min(total_score, 100) + + if os.environ.get("EXTENDED_VERBOSE_DEBUG", False): + logger.debug("\nValidated scores: %s", validated_scores) + logger.debug("\nCalculated total: %d", total_score) + + except (json.JSONDecodeError, KeyError, TypeError) as e: + logger.error("Failed to parse scoring response: %s", e) + logger.debug("Problematic text: %s", text) + # Return a default low score on parse failure + return 0 + + return total_score + + def __render_template(self, template_str: str, intel: CveIntel) -> str: + from jinja2 import Template + + template_jinja = Template(template_str, trim_blocks=True, lstrip_blocks=True) + full_df = data_utils.merge_intel_and_plugin_data_convert_to_dataframe([intel]) + flat_dict = full_df.to_dict(orient="records")[0] + + if os.environ.get("EXTENDED_VERBOSE_DEBUG", False): + logger.debug("\nflat_dict: %s", str(flat_dict)) + + rendered_input_list = template_jinja.render(**flat_dict) + + if os.environ.get("EXTENDED_VERBOSE_DEBUG", False): + logger.debug("\nrendered_input_list: %s", str(rendered_input_list)) + + return rendered_input_list diff --git a/src/vuln_analysis/utils/intel_utils.py b/src/vuln_analysis/utils/intel_utils.py index 81cdfb9d2..b45be5649 100644 --- a/src/vuln_analysis/utils/intel_utils.py +++ b/src/vuln_analysis/utils/intel_utils.py @@ -13,12 +13,78 @@ # See the License for the specific language governing permissions and # limitations under the License. +import ipaddress +import re +from pathlib import Path +from urllib.parse import urlparse + from packaging.version import InvalidVersion from packaging.version import parse as parse_version from pydpkg import Dpkg from pydpkg.exceptions import DpkgVersionError -from vuln_analysis.data_models.cve_intel import CveIntelNvd +from typing import Any + +from exploit_iq_commons.data_models.cve_intel import CveIntelNvd, CveIntel +from vuln_analysis.utils.package_identifier import _RPM_NEVRA_RE +from exploit_iq_commons.utils.data_utils import DEFAULT_GIT_DIRECTORY +from exploit_iq_commons.utils.git_utils import get_repo_path_with_ref + + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + +logger = LoggingFactory.get_agent_logger(__name__) + +_MAX_RHSA_CANDIDATES = 20 + +_GO_VULN_RE = re.compile(r"pkg\.go\.dev/vuln/(GO-\d{4}-\d+)") +_OSV_API_URL = "https://api.osv.dev/v1/vulns/" +_OSV_TIMEOUT_SECONDS = 5 + +_FUNC_PATTERNS = { + "go": re.compile(r"func\s+(?:\([^)]*\)\s+)?(\w+)\s*\("), + "python": re.compile(r"def\s+(\w+)\s*\("), + "java": re.compile( + r"(?:public|private|protected|static|final|abstract|synchronized|native" + r"|void|int|long|float|double|boolean|char|byte|short|String|Object)" + r"\s+(\w+)\s*\(" + ), + "javascript": re.compile( + r"(?:function\s+(\w+)\s*\(" + r"|(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?(?:function|\([^)]*\)\s*=>|\w+\s*=>)" + r"|(\w+)\s*\([^)]*\)\s*\{)" + ), + "c": re.compile(r"(?:^|\s)(\w+)\s*\([^)]*\)\s*\{"), +} + +_NOISE_FUNC_NAMES = frozenset({ + "main", "init", "test", "setup", "teardown", + "if", "else", "for", "while", "do", "switch", "return", "sizeof", "typeof", + "describe", "it", "beforeEach", "afterEach", "beforeAll", "afterAll", + "expect", "assert", +}) + +_JAVA_TEST_RE = re.compile(r"^test[A-Z]") + +TEST_FILE_RE = re.compile( + r"(?:_test\.go" + r"|Test(?:s|Case)?\.java" + r"|(?:^|/)test_[^/]*\.py" + r"|_test\.py" + r"|\.(?:test|spec)\.(?:js|ts|jsx|tsx)" + r"|_test\.(?:c|cpp|cc|cxx|h)" + r"|(?:^|/)test(?:s)?/)" +) + +_PACKAGE_TOKEN_CHARS = r"A-Za-z0-9_.\/:@+\-" + +_ECO_MAP = { + "go": "go", "golang": "go", + "pypi": "python", "python": "python", "pip": "python", + "npm": "javascript", "javascript": "javascript", "node": "javascript", + "maven": "java", "java": "java", + "conan": "c", "c": "c", "cpp": "c", "c++": "c", +} def update_version(incoming_version, current_version, compare): @@ -154,3 +220,790 @@ def parse(configurations: list): version_info.append(obj) return version_info + + +# --------------------------------------------------------------------------- +# Critical-context helpers (used by pre_process_node in cve_agent.py) +# --------------------------------------------------------------------------- + +def build_critical_context(cve_intel_list) -> tuple[list[str], list[dict], list[str]]: + """Extract key facts from all available intel sources into a compact context. + + Returns (critical_context, candidate_packages, vulnerable_functions) where + candidate_packages contains dicts with 'name', 'source', and optional + 'ecosystem' keys, and vulnerable_functions is a deduplicated list of short + function names from GHSA. + """ + critical_context = [] + candidate_packages: list[dict] = [] + seen_packages: set[str] = set() + vulnerable_functions: set[str] = set() + + for cve_intel in cve_intel_list: + if cve_intel.nvd is not None: + if cve_intel.nvd.cve_description: + critical_context.append(f"CVE Description: {cve_intel.nvd.cve_description[:400]}") + if cve_intel.nvd.cwe_name: + critical_context.append(f"CWE: {cve_intel.nvd.cwe_name}") + + if cve_intel.ghsa is not None: + if cve_intel.ghsa.vulnerabilities: + for v in cve_intel.ghsa.vulnerabilities[:3]: + vuln = v if isinstance(v, dict) else (v.__dict__ if hasattr(v, '__dict__') else {}) + vf = vuln.get('vulnerable_functions', []) if isinstance(vuln, dict) else getattr(v, 'vulnerable_functions', []) + pkg = vuln.get('package', None) if isinstance(vuln, dict) else getattr(v, 'package', None) + + if vf: + critical_context.append(f"Vulnerable functions (GHSA): {', '.join(vf)}") + short_names = [f.rsplit('.', 1)[-1] for f in vf if '.' in f] + if short_names: + critical_context.append(f"Search keywords: {', '.join(short_names)}") + for f in vf: + vulnerable_functions.add(f.rsplit('.', 1)[-1]) + ver_range = vuln.get('vulnerable_version_range', '') if isinstance(vuln, dict) else getattr(v, 'vulnerable_version_range', '') + patched = vuln.get('first_patched_version', '') if isinstance(vuln, dict) else getattr(v, 'first_patched_version', '') + if pkg: + if isinstance(pkg, dict): + pkg_name = pkg.get("name", "") + pkg_eco = pkg.get("ecosystem", "") + if pkg_name: + critical_context.append(f"Vulnerable module ({pkg_eco}): {pkg_name}") + if ver_range: + ver_ctx = f"Vulnerable version range ({pkg_name}): {ver_range}" + if patched: + ver_ctx += f" | First patched version: {patched}" + critical_context.append(ver_ctx) + if pkg_name not in seen_packages: + seen_packages.add(pkg_name) + candidate_packages.append({"name": pkg_name, "source": "ghsa", "ecosystem": pkg_eco}) + elif isinstance(pkg, str): + critical_context.append(f"Affected package: {pkg}") + if pkg not in seen_packages: + seen_packages.add(pkg) + candidate_packages.append({"name": pkg, "source": "ghsa"}) + if cve_intel.ghsa.description and not any("CVE Description" in c for c in critical_context): + critical_context.append(f"CVE Description: {cve_intel.ghsa.description[:400]}") + + if cve_intel.rhsa is not None: + if cve_intel.rhsa.statement: + critical_context.append(f"RHSA Statement: {cve_intel.rhsa.statement[:300]}") + mitigation = getattr(cve_intel.rhsa, 'mitigation', None) + if mitigation: + mit_text = mitigation.get('value', '') if isinstance(mitigation, dict) else str(mitigation) + if mit_text: + critical_context.append(f"KNOWN MITIGATIONS: {mit_text[:500]}") + if cve_intel.rhsa.package_state: + pkgs = list(set(p.package_name for p in cve_intel.rhsa.package_state if p.package_name)) + rhsa_added = 0 + for p in pkgs: + if p not in seen_packages: + seen_packages.add(p) + candidate_packages.append({"name": p, "source": "rhsa"}) + rhsa_added += 1 + if rhsa_added >= _MAX_RHSA_CANDIDATES: + break + if len(pkgs) > 5: + critical_context.append( + f"Affected across {len(pkgs)} Red Hat products (sample: {', '.join(pkgs[:5])}). " + "Focus investigation on the vulnerable library/module, not individual products." + ) + elif pkgs: + numbered = ", ".join(f"{i+1}) {p}" for i, p in enumerate(pkgs)) + critical_context.append(f"INVESTIGATE EACH package: {numbered}.") + + if cve_intel.ubuntu is not None: + if cve_intel.ubuntu.ubuntu_description: + critical_context.append(f"Ubuntu note: {cve_intel.ubuntu.ubuntu_description[:200]}") + + if cve_intel.plugin_data: + for pd in cve_intel.plugin_data[:2]: + critical_context.append(f"{pd.label}: {pd.description[:200]}") + + if not critical_context: + critical_context = ["No CVE intel available. Investigate using tools."] + return critical_context, candidate_packages, sorted(vulnerable_functions) + + +async def enrich_go_from_osv( + cve_intel, + critical_context: list[str], +) -> None: + """Query the OSV API for Go module paths when GHSA has no package data. + + Looks for a pkg.go.dev/vuln/GO-XXXX-XXXX link in GHSA or NVD + references, fetches the advisory from OSV, and adds module paths + and vulnerable symbols as hints into critical_context only. + OSV data is lower confidence than GHSA advisory data and should + not be added to candidate_packages or vulnerable_functions. + Fails silently on any error. + """ + refs: list[str] = [] + if cve_intel.ghsa is not None: + refs.extend(getattr(cve_intel.ghsa, "references", None) or []) + if cve_intel.nvd is not None: + refs.extend(cve_intel.nvd.references or []) + + go_id = None + for ref in refs: + m = _GO_VULN_RE.search(ref if isinstance(ref, str) else "") + if m: + go_id = m.group(1) + break + + if not go_id: + return + + try: + import aiohttp + timeout = aiohttp.ClientTimeout(total=_OSV_TIMEOUT_SECONDS) + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.get(f"{_OSV_API_URL}{go_id}") as resp: + if resp.status != 200: + logger.warning("OSV lookup for %s returned status %d", go_id, resp.status) + return + osv_data = await resp.json() + + seen_paths: set[str] = set() + all_symbols: list[str] = [] + for affected in osv_data.get("affected", []): + eco_specific = affected.get("ecosystem_specific", {}) + for imp in eco_specific.get("imports", []): + path = imp.get("path", "") + if path and path not in seen_paths: + seen_paths.add(path) + critical_context.append(f"Vulnerable module (Go vuln DB hint): {path}") + symbols = imp.get("symbols", []) + all_symbols.extend(symbols) + + if all_symbols: + critical_context.append( + f"Vulnerable functions (Go vuln DB hint): {', '.join(all_symbols)}" + ) + short_names = [s.rsplit(".", 1)[-1] for s in all_symbols if "." in s] + unique_keywords = list(dict.fromkeys(all_symbols + short_names)) + critical_context.append(f"Search keywords: {', '.join(unique_keywords)}") + + if seen_paths: + logger.info("OSV enrichment for %s added Go modules: %s", go_id, seen_paths) + except Exception: + logger.warning("OSV enrichment failed for %s", go_id, exc_info=True) + + +def validate_go_vendor_packages(source_info, candidate_packages): + """Filter candidate packages to those present in the vendor/ directory. + + Returns (validated_packages, removed_package_names). If no packages + survive validation the original list is returned unchanged. + """ + code_si = next((si for si in source_info if si.type == "code"), None) + if code_si is None: + return candidate_packages, [] + repo_path = get_repo_path_with_ref(Path(DEFAULT_GIT_DIRECTORY), code_si.git_repo, code_si.ref) + vendor_path = repo_path / "vendor" + if not vendor_path.is_dir(): + return candidate_packages, [] + validated = [] + removed = [] + for pkg in candidate_packages: + pkg_name = pkg.get("name", "") + if (vendor_path / pkg_name).is_dir(): + validated.append(pkg) + else: + removed.append(pkg_name) + if validated: + return validated, removed + return candidate_packages, [] + + +async def enrich_go_candidates(cve_intel, source_info, critical_context, candidate_packages, vulnerable_functions_set): + """Enrich Go candidate packages with OSV data and validate against vendor/. + + Calls ``enrich_go_from_osv`` when GHSA lacks package data or vulnerable + function symbols, then prunes candidates not found in vendor/. + + Returns (candidate_packages, sorted_vulnerable_functions). + """ + ghsa_has_packages = any(c.get("source") == "ghsa" for c in candidate_packages) + if not ghsa_has_packages or not vulnerable_functions_set: + intel = cve_intel[0] if cve_intel else None + if intel: + await enrich_go_from_osv(intel, critical_context) + if candidate_packages: + candidate_packages, removed_pkgs = validate_go_vendor_packages(source_info, candidate_packages) + if removed_pkgs: + logger.info("Go vendor validation removed %d packages not in vendor/: %s", len(removed_pkgs), removed_pkgs) + return candidate_packages, sorted(vulnerable_functions_set) + + +def _ref_to_url(ref) -> str: + """Extract a URL string from a reference (GHSA dict or NVD string).""" + if isinstance(ref, str): + return ref + if isinstance(ref, dict): + return ref.get("url", "") + return "" + + +def _is_fix_ref(ref) -> bool: + """Return True if a GHSA reference is tagged as a fix.""" + return isinstance(ref, dict) and ref.get("type", "").upper() == "FIX" + + +def _get_ecosystem_patterns(ecosystem: str) -> list[re.Pattern]: + """Return regex patterns for the given ecosystem, or all patterns if unknown.""" + eco_lower = ecosystem.lower() if ecosystem else "" + key = _ECO_MAP.get(eco_lower) + if key: + return [_FUNC_PATTERNS[key]] + return list(_FUNC_PATTERNS.values()) + + +def _match_func_name(text: str, patterns: list[re.Pattern], out: set[str]) -> None: + """Match a function name from text and add to out set if valid.""" + for pat in patterns: + m = pat.search(text) + if m: + name = next((g for g in m.groups() if g), None) + if (name and len(name) > 1 + and name.lower() not in _NOISE_FUNC_NAMES + and not name.startswith("Test") + and not name.startswith("test_") + and not _JAVA_TEST_RE.match(name)): + out.add(name) + break + + +def extract_functions_from_parsed_patch(parsed_patch, ecosystem: str = "") -> set[str]: + """Extract function names from a ParsedPatch object. + + Checks added lines and @@ hunk section headers (which name the enclosing + function even when the signature itself isn't in the diff). + Skips test files. + """ + funcs: set[str] = set() + patterns = _get_ecosystem_patterns(ecosystem) + for pf in parsed_patch.files: + if TEST_FILE_RE.search(pf.target_path): + continue + for hunk in pf.hunks: + if hunk.section_header: + _match_func_name(hunk.section_header, patterns, funcs) + for line in hunk.added_lines: + _match_func_name(line, patterns, funcs) + return funcs + + +def _append_enrichment_to_context(all_funcs: set[str], critical_context: list[str], source_label: str) -> None: + """Append extracted function names to critical_context.""" + critical_context.append( + f"Vulnerable functions (remediation patch hint): {', '.join(sorted(all_funcs))}" + ) + short_names = [f.rsplit(".", 1)[-1] for f in all_funcs if "." in f] + if short_names: + unique = list(dict.fromkeys(list(all_funcs) + short_names)) + critical_context.append(f"Search keywords: {', '.join(unique)}") + logger.info("Patch enrichment extracted functions from %s: %s", source_label, sorted(all_funcs)) + + +async def enrich_vulnerable_functions_from_patch( + cve_intel_list, + critical_context: list[str], + vulnerable_functions: set[str], + ecosystem: str = "", + patch_result=None, +) -> None: + """Extract vulnerable function names from remediation patch commits. + + When GHSA/OSV have no vulnerable_functions, extracts function names from + patch diffs. Uses a pre-fetched patch_result when available (from the + cve_fetch_patches pipeline step), otherwise falls back to fetching via + fetch_patch_for_cve. Test files are skipped to avoid injecting test-only names. + """ + if vulnerable_functions: + return + + from vuln_analysis.utils.web_patch_fetcher import WebPatchResult + + if isinstance(patch_result, dict): + patch_result = WebPatchResult.model_validate(patch_result) + + if patch_result and patch_result.parsed_patch: + all_funcs = extract_functions_from_parsed_patch(patch_result.parsed_patch, ecosystem) + if all_funcs: + _append_enrichment_to_context(all_funcs, critical_context, f"pre-fetched patch ({patch_result.source})") + return + + try: + import aiohttp + from vuln_analysis.utils.web_patch_fetcher import fetch_patch_for_cve + + merged_candidates: dict[str, list[str]] = {} + for cve_intel in cve_intel_list: + per_intel = extract_commit_url_candidates(cve_intel) + for source, urls in per_intel.items(): + merged_candidates.setdefault(source, []).extend(urls) + + vuln_id = cve_intel_list[0].vuln_id if cve_intel_list else "" + cve_description = None + for cve_intel in cve_intel_list: + if cve_intel.nvd and cve_intel.nvd.cve_description: + cve_description = cve_intel.nvd.cve_description + break + + async with aiohttp.ClientSession() as session: + result = await fetch_patch_for_cve( + session=session, + candidates=merged_candidates, + vuln_id=vuln_id, + cve_description=cve_description, + ) + + if result and result.parsed_patch: + all_funcs = extract_functions_from_parsed_patch(result.parsed_patch, ecosystem) + if all_funcs: + _append_enrichment_to_context(all_funcs, critical_context, f"fetched patch ({result.source})") + return + else: + logger.info("Patch fetched for %s but no functions extracted", vuln_id) + except Exception: + logger.warning("Patch-based function extraction failed", exc_info=True) + + +def _package_token_boundary_pattern(rn: str) -> str: + esc = re.escape(rn) + c = _PACKAGE_TOKEN_CHARS + return rf"(?:^|(? bool: + if not rn: + return False + return re.search(_package_token_boundary_pattern(rn), entry) is not None + + +def _strip_rejected_package_token(entry: str, rn: str) -> str: + if not rn: + return entry + return re.sub(_package_token_boundary_pattern(rn), "", entry) + + +_COMMIT_URL_KEYWORDS = frozenset({ + "github.com/", "gitlab.com/", "gitlab.", "bitbucket.org/", + "/commit/", "/commits/", "/pull/", "/merge_requests/", + ".git", "git.kernel.org", "git.savannah", "cgit", +}) + +# Chromium issue tracker URL pattern - captures the bug ID +CHROMIUM_ISSUE_PATTERN = re.compile(r"https?://issues\.chromium\.org/issues/(\d+)") + +# --------------------------------------------------------------------------- +# Advisory URL extraction (for Reference Mining) +# --------------------------------------------------------------------------- + +# Advisory source patterns with (source_type, priority) - lower priority = higher quality +# Priority 1-2: Detailed technical mailing lists with commit refs, patches +# Priority 3-4: Vendor advisories with fix details +# Priority 5+: Generic advisories, less likely to have commit hints +ADVISORY_SOURCE_PATTERNS: dict[str, tuple[str, int]] = { + # High priority - detailed technical info, often has commit refs + "openwall.com/lists/oss-security": ("openwall", 1), + "seclists.org": ("mailing_list", 2), + "marc.info": ("mailing_list", 2), + "mail-archive.com": ("mailing_list", 2), + "lore.kernel.org": ("mailing_list", 2), + # Medium priority - vendor advisories with fix info + "apache.org/security": ("vendor_advisory", 3), + "httpd.apache.org/security": ("vendor_advisory", 3), + "kernel.org/doc": ("vendor_advisory", 3), + "debian.org/security": ("vendor_advisory", 3), + "access.redhat.com/errata": ("vendor_advisory", 4), + "access.redhat.com/security": ("vendor_advisory", 4), + "ubuntu.com/security": ("vendor_advisory", 4), + "security.gentoo.org": ("vendor_advisory", 4), + # Lower priority - general advisories (less likely to have commit hints) + "cve.org": ("cve_org", 5), + "cve.mitre.org": ("cve_org", 5), + "nvd.nist.gov": ("nvd", 5), +} + +# URL patterns that indicate commit/patch content (should be excluded from advisory mining) +_COMMIT_URL_PATTERNS = frozenset({ + "/commit/", "/commits/", "/pull/", "/merge_requests/", + ".patch", ".diff", "/raw/", "/blob/", +}) + + +def is_advisory_url(url: str) -> bool: + """Check if URL is an advisory page (not a commit/PR/patch link). + + Returns True if the URL does NOT contain patterns indicating + direct source code or patch content. + + Args: + url: URL string to check + + Returns: + True if URL appears to be an advisory page, False if it's a commit/patch URL + """ + if not url: + return False + url_lower = url.lower() + return not any(pattern in url_lower for pattern in _COMMIT_URL_PATTERNS) + + +# Allowed URL schemes for advisory fetching +_ALLOWED_URL_SCHEMES = frozenset({"http", "https"}) + + +def _is_safe_url(url: str) -> bool: + """Validate URL for safe fetching (SSRF protection). + + Ensures the URL uses http/https scheme and targets a DNS hostname + rather than a raw IP address. This blocks SSRF vectors including + localhost, private IPs, and cloud metadata endpoints. + + Args: + url: URL string to validate + + Returns: + True if URL is safe to fetch, False otherwise + """ + if not url: + return False + + try: + parsed = urlparse(url) + + if parsed.scheme not in _ALLOWED_URL_SCHEMES: + logger.debug("Rejected URL with disallowed scheme: %s", url) + return False + + hostname = parsed.hostname + if not hostname: + logger.debug("Rejected URL with missing hostname: %s", url) + return False + + # Reject raw IP addresses - only allow DNS hostnames + try: + ipaddress.ip_address(hostname) + logger.debug("Rejected IP-based URL (SSRF protection): %s", url) + return False + except ValueError: + # Not an IP address - this is a DNS hostname, which is allowed + pass + + return True + + except Exception: + logger.debug("Rejected malformed URL: %s", url, exc_info=True) + return False + + +def _classify_advisory_url(url: str) -> tuple[str, int]: + """Classify an advisory URL by source type and priority. + + Args: + url: URL string to classify + + Returns: + Tuple of (source_type, priority). Returns ("unknown", 10) if no pattern matches. + """ + url_lower = url.lower() + for pattern, (source_type, priority) in ADVISORY_SOURCE_PATTERNS.items(): + if pattern in url_lower: + return source_type, priority + return "unknown", 10 + + +def extract_advisory_urls(intel: CveIntel) -> list[tuple[str, str, int]]: + """Extract non-commit reference URLs from intel for advisory mining. + + Scans GHSA, NVD, RHSA, and Ubuntu references for URLs that are NOT + direct commit/patch links but may contain advisory content with hints + about fix commits, vulnerable functions, or affected files. + + This is the inverse of extract_commit_url_candidates() - it finds + advisory pages rather than direct commit URLs. + + Args: + intel: CveIntel object containing intel from various providers + + Returns: + List of (url, source_type, priority) tuples, sorted by priority (ascending). + Lower priority number = higher quality source. + Example: [("https://openwall.com/lists/oss-security/...", "openwall", 1), ...] + """ + seen_urls: set[str] = set() + results: list[tuple[str, str, int]] = [] + + def _process_refs(refs: list | None) -> None: + if not refs: + return + for ref in refs: + url = ref if isinstance(ref, str) else (ref.get("url", "") if isinstance(ref, dict) else "") + if not url or url in seen_urls: + continue + if not _is_safe_url(url): + continue + if not is_advisory_url(url): + continue + seen_urls.add(url) + source_type, priority = _classify_advisory_url(url) + results.append((url, source_type, priority)) + + # GHSA references + if intel.ghsa: + ghsa_refs = getattr(intel.ghsa, "references", None) + _process_refs(ghsa_refs) + + # NVD references + if intel.nvd: + _process_refs(intel.nvd.references) + + # RHSA references (may be newline-separated) + if intel.rhsa: + rhsa_refs = getattr(intel.rhsa, "references", None) + if isinstance(rhsa_refs, list): + flat_refs = [] + for ref in rhsa_refs: + if isinstance(ref, str) and "\n" in ref: + flat_refs.extend(ref.split("\n")) + else: + flat_refs.append(ref) + _process_refs(flat_refs) + + # Ubuntu references + if intel.ubuntu: + ubuntu_refs = getattr(intel.ubuntu, "references", None) + _process_refs(ubuntu_refs) + + # Sort by priority (ascending - lower is better) + results.sort(key=lambda x: x[2]) + return results + + +def extract_commit_url_candidates(intel: CveIntel) -> dict[str, list[str]]: + """Extract URLs from intel references that may contain commit/patch information. + + Scans GHSA, NVD, RHSA, and Ubuntu references for URLs containing keywords + that suggest they may point to source code commits or patches. Also includes + Chromium issue tracker URLs (issues.chromium.org/issues/) which can be + resolved to patches via Gerrit/Gitiles. + + Args: + intel: CveIntel object containing intel from various providers + + Returns: + Dict mapping provider name to list of matching URLs. + Example: {"ghsa": ["https://github.com/foo/bar/commit/abc123"], "nvd": []} + """ + + def _matches_keywords(url: str) -> bool: + url_lower = url.lower() + return any(kw in url_lower for kw in _COMMIT_URL_KEYWORDS) + + def _is_chromium_issue(url: str) -> bool: + return bool(CHROMIUM_ISSUE_PATTERN.match(url)) + + def _extract_refs(refs: list[str] | None) -> list[str]: + if not refs: + return [] + return [r for r in refs if isinstance(r, str) and (_matches_keywords(r) or _is_chromium_issue(r))] + + result: dict[str, list[str]] = {} + + # GHSA references (extra field, may not exist) + if intel.ghsa: + ghsa_refs = getattr(intel.ghsa, "references", None) + result["ghsa"] = _extract_refs(ghsa_refs) + + # NVD references (explicit field) + if intel.nvd: + result["nvd"] = _extract_refs(intel.nvd.references) + + # RHSA references (extra field, may not exist) + if intel.rhsa: + rhsa_refs = getattr(intel.rhsa, "references", None) + # RHSA references may be newline-separated strings + if isinstance(rhsa_refs, list): + flat_refs = [] + for ref in rhsa_refs: + if isinstance(ref, str) and "\n" in ref: + flat_refs.extend(ref.split("\n")) + else: + flat_refs.append(ref) + result["rhsa"] = _extract_refs(flat_refs) + else: + result["rhsa"] = [] + + # Ubuntu references (explicit field) + if intel.ubuntu: + ubuntu_refs = getattr(intel.ubuntu, "references", None) + result["ubuntu"] = _extract_refs(ubuntu_refs) + # Also include patches field URLs + if intel.ubuntu.patches: + patch_urls = [] + for pkg_patches in intel.ubuntu.patches.values(): + patch_urls.extend(pkg_patches or []) + result["ubuntu_patches"] = _extract_refs(patch_urls) + + return result + + +def filter_context_to_package(critical_context: list[str], selected: str, all_candidates: list[dict]) -> list[str]: + """Narrow CVE intel strings to the user-selected package after disambiguation. + Intel lines are built from GHSA, RHSA... and may mention several modules; this removes + references to *other* candidates so downstream classification / reachability + prompts are not biased toward the wrong package. + + Behavior: + * Replaces the RHSA-style ``INVESTIGATE EACH package:`` line with + ``Target package: ``. + * Drops entire ``Vulnerable module (...)`` / ``Affected package:`` lines when a + *rejected* candidate appears as a **whole token** (not a substring inside a + longer path like ``com`` in ``github.com``). + * For all other lines, removes each rejected name wherever it appears as a + whole token, then collapses runs of whitespace. + + This is best-effort text cleanup, not full natural-language redaction. + """ + rejected_names = {c["name"] for c in all_candidates if c["name"] != selected} + # Longest names first so a shorter rejected string cannot leave a suffix of a longer one. + rejected_ordered = sorted(rejected_names, key=len, reverse=True) + filtered = [] + for entry in critical_context: + if entry.startswith("INVESTIGATE EACH package:"): + filtered.append(f"Target package: {selected}") + continue + if entry.startswith("Vulnerable module (") or entry.startswith("Affected package:"): + if any(_rejected_package_present_as_token(entry, rn) for rn in rejected_ordered): + continue + for rn in rejected_ordered: + entry = _strip_rejected_package_token(entry, rn) + # Token removal leaves gaps; normalize spaces, merge ", ," from list removals, then collapse again. + entry = re.sub(r"\s+", " ", entry).strip() + entry = re.sub(r",\s*,", ", ", entry) + entry = re.sub(r"\s+", " ", entry).strip() + filtered.append(entry) + return filtered + + +def extract_vuln_packages_from_intel(cve_intel: CveIntel) -> list[dict[str, Any]]: + """ + Extract vulnerable package names and version ranges from intel sources. + + Args: + cve_intel: CveIntel object containing NVD, GHSA, and RHSA data + + Returns: + List of dicts with package info: + { + "source": "nvd" | "ghsa" | "rhsa", + "package": str, + "ecosystem": str | None, + "version_start_incl": str | None, + "version_start_excl": str | None, + "version_end_incl": str | None, + "version_end_excl": str | None, + "vulnerable_range": str | None, # GHSA format like "< 2.9.2" + "first_patched": str | None, + "fix_state": str | None, # RHSA fix_state ("affected", "not affected", etc.) + } + """ + packages: list[dict[str, Any]] = [] + + # Extract from NVD configurations + if cve_intel.nvd and cve_intel.nvd.configurations: + for config in cve_intel.nvd.configurations: + packages.append({ + "source": "nvd", + "package": config.package, + "ecosystem": config.system, + "version_start_incl": config.versionStartIncluding, + "version_start_excl": config.versionStartExcluding, + "version_end_incl": config.versionEndIncluding, + "version_end_excl": config.versionEndExcluding, + "vulnerable_range": None, + "first_patched": None, + "fix_state": None, + }) + + # Extract from GHSA vulnerabilities + if cve_intel.ghsa and cve_intel.ghsa.vulnerabilities: + for vuln in cve_intel.ghsa.vulnerabilities: + # Handle both dict and object formats + if isinstance(vuln, dict): + pkg_info = vuln.get("package", {}) + pkg_name = pkg_info.get("name") if isinstance(pkg_info, dict) else None + pkg_ecosystem = pkg_info.get("ecosystem") if isinstance(pkg_info, dict) else None + vuln_range = vuln.get("vulnerable_version_range") + first_patched = vuln.get("first_patched_version") + else: + pkg_info = getattr(vuln, "package", {}) + pkg_name = pkg_info.get("name") if isinstance(pkg_info, dict) else None + pkg_ecosystem = pkg_info.get("ecosystem") if isinstance(pkg_info, dict) else None + vuln_range = getattr(vuln, "vulnerable_version_range", None) + first_patched = getattr(vuln, "first_patched_version", None) + + if pkg_name: + packages.append({ + "source": "ghsa", + "package": pkg_name, + "ecosystem": pkg_ecosystem, + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + "vulnerable_range": vuln_range, + "first_patched": first_patched, + "fix_state": None, + }) + + # Extract from RHSA package_state (affected packages with fix status) + if cve_intel.rhsa and cve_intel.rhsa.package_state: + for ps in cve_intel.rhsa.package_state: + if ps.package_name: + packages.append({ + "source": "rhsa", + "package": ps.package_name, + "ecosystem": "rpm", + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + "vulnerable_range": None, + "first_patched": None, + "fix_state": ps.fix_state, + }) + + # Extract from RHSA affected_release (fixed versions) + if cve_intel.rhsa and hasattr(cve_intel.rhsa, 'affected_release') and cve_intel.rhsa.affected_release: + for entry in cve_intel.rhsa.affected_release: + pkg_str = entry.get("package") if isinstance(entry, dict) else getattr(entry, "package", None) + if pkg_str: + # Parse NEVRA format: name-[epoch:]version-release.arch + m = _RPM_NEVRA_RE.match(pkg_str) + if m: + name = m.group(1) + version = m.group(3) + release_arch = m.group(4) + release = release_arch.rsplit(".", 1)[0] if "." in release_arch else release_arch + ver_rel = f"{version}-{release}" + else: + name = pkg_str + ver_rel = None + + packages.append({ + "source": "rhsa", + "package": name, + "ecosystem": "rpm", + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": ver_rel, # Fixed version = end of vulnerable range + "vulnerable_range": f"< {ver_rel}" if ver_rel else None, + "first_patched": ver_rel, + "fix_state": None, + }) + + return packages diff --git a/src/vuln_analysis/utils/js_extended_parser.py b/src/vuln_analysis/utils/js_extended_parser.py deleted file mode 100644 index ba597eb87..000000000 --- a/src/vuln_analysis/utils/js_extended_parser.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -from typing import Any -from typing import List -from typing import Tuple - -import esprima -from langchain_community.document_loaders.parsers.language.javascript import JavaScriptSegmenter - -logger = logging.getLogger(__name__) - - -class ExtendedJavaScriptSegmenter(JavaScriptSegmenter): - """Extended JavaScript segmenter that handles shebang and ES optional chaining.""" - - def __init__(self, code: str): - """Initialize the segmenter with preprocessed code.""" - super().__init__(code) - # Skip files with shebang (#!) line since they typically contain non-standard JavaScript syntax - if code.startswith("#!"): - logger.warning("File contains a shebang line. Skipping parsing.") - self.skip_file = True - # esprima-python parser limitation: cannot handle JavaScript optional chaining syntax, - # fallback to regular property access - else: - self.skip_file = False - self.code = self.code.replace("?.", ".") - - def _parse_with_fallback(self) -> Any: - """Try to parse code as script first, then as module if that fails.""" - try: - logger.debug("Attempting to parse as a script...") - return esprima.parseScript(self.code, loc=True) - except esprima.Error: - logger.debug("Script parsing failed. Trying module parsing...") - try: - return esprima.parseModule(self.code, loc=True) - except esprima.Error as e: - logger.error("Module parsing failed: %s", str(e)) - return None - - def extract_functions_classes(self) -> List[str]: - """Extract functions, classes and exports from the code.""" - if self.skip_file: - return [] - - tree = self._parse_with_fallback() - if tree is None: - return [] - - functions_classes = [] - for node in tree.body: - # Handle direct function/class declarations - if isinstance(node, (esprima.nodes.FunctionDeclaration, esprima.nodes.ClassDeclaration)): - functions_classes.append(self._extract_code(node)) - # Handle exported declarations - elif isinstance(node, esprima.nodes.ExportNamedDeclaration): - if isinstance(node.declaration, (esprima.nodes.FunctionDeclaration, esprima.nodes.ClassDeclaration)): - functions_classes.append(self._extract_code(node)) - - return functions_classes - - def simplify_code(self) -> str: - """Simplify the code by replacing function/class bodies with comments.""" - if self.skip_file: - return self.code - - tree = self._parse_with_fallback() - if tree is None: - return self.code - - simplified_lines = self.source_lines[:] - indices_to_del: List[Tuple[int, int]] = [] - - for node in tree.body: - if isinstance(node, (esprima.nodes.FunctionDeclaration, esprima.nodes.ClassDeclaration)): - start, end = node.loc.start.line - 1, node.loc.end.line - simplified_lines[start] = f"// Code for: {simplified_lines[start]}" - indices_to_del.append((start + 1, end)) - elif isinstance(node, esprima.nodes.ExportNamedDeclaration): - if isinstance(node.declaration, (esprima.nodes.FunctionDeclaration, esprima.nodes.ClassDeclaration)): - start, end = node.loc.start.line - 1, node.loc.end.line - simplified_lines[start] = f"// Code for: {simplified_lines[start]}" - indices_to_del.append((start + 1, end)) - - for start, end in reversed(indices_to_del): - del simplified_lines[start:end] - - return "\n".join(line for line in simplified_lines) diff --git a/src/vuln_analysis/utils/justification_parser.py b/src/vuln_analysis/utils/justification_parser.py index 8186b4369..ac94ed19f 100644 --- a/src/vuln_analysis/utils/justification_parser.py +++ b/src/vuln_analysis/utils/justification_parser.py @@ -13,10 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging from textwrap import dedent -logger = logging.getLogger(__name__) +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) class JustificationParser: @@ -25,35 +25,84 @@ class JustificationParser: AFFECTED_STATUS_COL_NAME = "affected_status" JUSTIFICATION_PROMPT = dedent(""" - The summary provided below (delimited with XML tags), generated by the software agent named Vulnerability - Analysis for Container Security, evaluates a specific CVE (Common Vulnerabilities and Exposures) against the - backdrop of a software package or environment information. This information may include a Software Bill of - Materials (SBOM), source code, and dependency documentation among others related to environments such as - container images. Vulnerability Analysis for Container Security's role is to analyze this data to ascertain if the CVE - impacts the software environment, determining the necessity for a patch. - - Your task is to review Vulnerability Analysis for Container Security's analysis and classify the situation into - one of the following categories based on the described scenario: - - false_positive: The association between the software package and the CVE is incorrect, either due to an - erroneous package identification or a mismatched CVE. - - code_not_present: The software product is unaffected as it does not contain the code or library that harbors - the vulnerability. - - code_not_reachable: During runtime, the vulnerable code is not executed. - - requires_configuration: The exploitability of this issue depends on whether a specific configuration option is - enabled or disabled. In this case, the configuration is set in a manner that prevents exploitability. - - requires_dependency: Exploitability is contingent upon a missing dependency. - - requires_environment: A specific environment, absent in this case, is required for exploitability. - - compiler_protected: Exploitability hinges on the setting or unsetting of a compiler flag. In this case, the - flag is set in a manner that prevents exploitability. - - runtime_protected: Mechanisms are in place that prevent exploits during runtime. - - perimeter_protected: Protective measures block attacks at the physical, logical, or network perimeters. - - mitigating_control_protected: Implemented controls mitigate the likelihood or impact of the vulnerability. - - uncertain: Not enough information to determine the package's exploitability status. - - vulnerable: the package is actually vulnerable to the CVE and needs to be patched. - - Response only with the category name on the first line, and reasoning on the second line. - {summary} - """).strip("\n") + +Review the CVE exploitability investigation summary and perform two tasks: +1. Classify the exploitability status into one of 12 predefined categories +2. Provide reasoning for your classification, citing specific evidence from the investigation summary + + + +Select the single most appropriate category based on the investigation findings. + +CLASSIFICATION CATEGORIES (in logical precedence order): + +1. false_positive - CVE-to-package association is incorrect (wrong package or mismatched CVE) + +2. code_not_present - Vulnerable code/library is absent from the container + (If code is not present, subsequent factors are irrelevant) + +3. code_not_reachable - Vulnerable code exists but is never executed at runtime + (Only applicable if code IS present but execution path analysis shows no calls) + +4. requires_configuration - Exploitation requires specific configuration that is disabled + (Configuration prevents exploitation) + +5. requires_dependency - Exploitation requires a dependency that is missing + +6. requires_environment - Exploitation requires specific environment that is absent + +7. compiler_protected - Compiler flags prevent exploitation + +8. runtime_protected - Runtime mechanisms (ASLR, DEP, sandboxing) prevent exploitation + +9. perimeter_protected - Network/physical/logical perimeter defenses block exploitation + +10. mitigating_control_protected - Other security controls reduce risk + +11. uncertain - Insufficient information to determine exploitability + +12. vulnerable - Package is actually vulnerable and needs patching + +EXPLOITATION CONDITIONS: +For a CVE to be classified as "vulnerable", ALL of these must be true: +- Vulnerable code is PRESENT in the container +- Vulnerable code is USED/CALLED by the application +- Vulnerable code is REACHABLE from attack surfaces (user input, network, file processing) +- No effective mitigations or protections are in place + +IF EXPLOITATION CONDITIONS ARE NOT MET: +Select the PRIMARY reason for non-exploitability following the logical precedence +order above. For example: +- If code is not present: "code_not_present" (even if other factors would also prevent it) +- If code is not reachable: "code_not_reachable" (not "requires_environment") +- If missing dependency prevents it: "requires_dependency" + +The categories are ordered by logical precedence. Work through the list from top +to bottom and select the first category that applies to the situation. + +Use "uncertain" only as a final fallback when the investigation truly lacks +sufficient information to make any determination. + + + +Provide exactly two lines: + +Line 1: category_name (exact category name from the list above) +Line 2: reasoning (brief explanation citing specific evidence from the summary) + +Do not include labels like "Category:" or "Reasoning:". Just the values on separate lines. + + + +code_not_reachable +The vulnerable PIL.ImageMath.eval function exists in the installed Pillow library, but call chain analysis confirmed it is never invoked from application code. The application only uses PIL.Image.open() and PIL.Image.thumbnail() functions, which do not call ImageMath.eval. + + + +{summary} + + +Provide your justification classification and reasoning on two separate lines:""").strip("\n") RAW_TO_FINAL_JUSTIFICATION_MAP = { "false_positive": "false_positive", diff --git a/src/vuln_analysis/utils/llm_engine_utils.py b/src/vuln_analysis/utils/llm_engine_utils.py index 786596944..1857bdd89 100644 --- a/src/vuln_analysis/utils/llm_engine_utils.py +++ b/src/vuln_analysis/utils/llm_engine_utils.py @@ -13,76 +13,171 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +import re import typing +from pathlib import Path from ordered_set import OrderedSet -from vuln_analysis.data_models.dependencies import VulnerableDependencies -from vuln_analysis.data_models.input import AgentMorpheusEngineInput -from vuln_analysis.data_models.input import AgentMorpheusInput +from exploit_iq_commons.data_models.common import AnalysisType +from exploit_iq_commons.data_models.dependencies import CheckedNotVulnerablePackage, VulnerableDependencies +from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput +from exploit_iq_commons.data_models.input import AgentMorpheusInput from vuln_analysis.data_models.output import AgentMorpheusEngineOutput from vuln_analysis.data_models.output import AgentMorpheusOutput +from vuln_analysis.data_models.output import OutputPayload from vuln_analysis.data_models.output import ChecklistItemOutput from vuln_analysis.data_models.output import JustificationOutput +from vuln_analysis.data_models.output import CVSSOutput +from vuln_analysis.functions.cve_generate_cvss import CVSS_VECTOR_STRING, CVSS_SCORE +from vuln_analysis.utils.web_patch_fetcher import WebPatchResult +from vuln_analysis.functions.code_agent_graph_defs import PatchFile +from vuln_analysis.utils.intel_utils import TEST_FILE_RE +from vuln_analysis.functions.cve_checker_report import infer_language_from_path, NON_CODE_LANGUAGES from vuln_analysis.data_models.state import AgentMorpheusEngineState +from aiq.builder.builder import Builder -logger = logging.getLogger(__name__) +from vuln_analysis.functions.cve_calculate_intel_score import CVECalculateIntelScoreConfig + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id + +logger = LoggingFactory.get_agent_logger(__name__) def preprocess_engine_input(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineState: assert message.info.intel is not None, "The input message must have intel information" - # Return early if there are no SBOM packages - sbom = message.info.sbom.packages - image = f"{message.input.image.name}:{message.input.image.tag}" - if not sbom: - raise ValueError(f"No SBOM packages found for image {image}. Skipping the LLM Engine.") + # Return early if analysis_type is not source and there are no SBOM packages + if message.input.image.analysis_type != AnalysisType.SOURCE: + image = f"{message.input.image.name}:{message.input.image.tag}" + sbom = message.info.sbom.packages + if not sbom: + raise ValueError(f"No SBOM packages found for image {image}. Skipping the LLM Engine.") am_input: AgentMorpheusInput = message.input - vulnerable_dependencies = [] - for v in message.info.vulnerable_dependencies: - vulnerable_sbom_packages = v.vulnerable_sbom_packages - vulnerable_dependencies.append([p.name for p in vulnerable_sbom_packages]) + # Scan through the VDC output for CVE's to run through the agent + if message.info.vulnerable_dependencies is None: + cve_version_checker_run_agent = [True for _ in am_input.scan.vulns] + else: + cve_version_checker_run_agent = [ + len(v.vulnerable_sbom_packages) > 0 or len(v.vuln_package_intel_sources) == 0 + for v in message.info.vulnerable_dependencies + ] - has_vuln_package_info_flags = [len(v.vuln_package_intel_sources) > 0 for v in message.info.vulnerable_dependencies] + # Look through the intel for CVE's with sufficient intel for the agent + sufficient_intel_run_agent = [intel.has_sufficient_intel_for_agent for intel in message.info.intel] - has_sufficient_intel_flags = [intel.has_sufficient_intel_for_agent for intel in message.info.intel] + # Create the filter list by combining the VDC and intel criteria + run_agent = [all(tup) for tup in zip(cve_version_checker_run_agent, sufficient_intel_run_agent)] vuln_ids = [vuln.vuln_id for vuln in am_input.scan.vulns] - filtered_vuln_ids = [] - for vuln_id, vuln_deps, has_vuln_package_info, has_sufficient_intel in zip( - vuln_ids, vulnerable_dependencies, has_vuln_package_info_flags, - has_sufficient_intel_flags): - if len(vuln_deps) > 0 or (not has_vuln_package_info and has_sufficient_intel): - filtered_vuln_ids.append(vuln_id) + vulns_for_agent = [] + for vuln_id, should_run_agent in zip(vuln_ids, run_agent): + if should_run_agent: + vulns_for_agent.append(vuln_id) # Drop duplicate vuln_ids - unique_vulns = list(OrderedSet(filtered_vuln_ids)) - if len(filtered_vuln_ids) > len(unique_vulns): + unique_vulns = list(OrderedSet(vulns_for_agent)) + if len(vulns_for_agent) > len(unique_vulns): logger.warning( "Input contains duplicate vuln_ids. Passing only the first instance of each vuln_id to the LLM Engine.") - filtered_vuln_ids = unique_vulns + vulns_for_agent = unique_vulns filtered_intel = [] for i in message.info.intel: - if i.vuln_id in filtered_vuln_ids: + if i.vuln_id in vulns_for_agent: filtered_intel.append(i) - logger.info("Passing %d vuln_id(s) with vulnerable dependencies to the LLM Engine", len(filtered_vuln_ids)) - - return AgentMorpheusEngineState(code_vdb_path=message.info.vdb.code_vdb_path, - doc_vdb_path=message.info.vdb.doc_vdb_path, - code_index_path=message.info.vdb.code_index_path, - cve_intel=filtered_intel) + logger.info("Passing %d vuln_id(s) with vulnerable dependencies to the LLM Engine", len(vulns_for_agent)) + + vdb = message.info.vdb + return AgentMorpheusEngineState(code_vdb_path=vdb.code_vdb_path if vdb else None, + doc_vdb_path=vdb.doc_vdb_path if vdb else None, + code_index_path=vdb.code_index_path if vdb else None, + cve_intel=filtered_intel, + original_input=message) + + +_MAX_SNIPPET_LINES = 12 +_MAX_PATCH_FILES = 5 +_MD_LINK_RE = re.compile(r'([!\[\]()])') + + +def _build_full_pipeline_details_md(patch_result: WebPatchResult | dict | None) -> str | None: + """Build markdown details from a fetched patch result for the UI client.""" + if not patch_result: + return None + if isinstance(patch_result, dict): + patch_result = WebPatchResult.model_validate(patch_result) + if not patch_result.parsed_patch: + return None + + lines: list[str] = [] + + lines.append("## Vulnerability Fix Patch") + lines.append("") + + if patch_result.patch_url: + lines.append(f"**Patch source:** {_MD_LINK_RE.sub(r'\\\1', patch_result.patch_url)}") + lines.append("") + if patch_result.source: + lines.append(f"**Found via:** {_MD_LINK_RE.sub(r'\\\1', patch_result.source)}") + lines.append("") + if patch_result.commit_message: + first_line = patch_result.commit_message.strip().split("\n")[0][:200] + lines.append(f"**Commit message:** {_MD_LINK_RE.sub(r'\\\1', first_line)}") + lines.append("") + + parsed = patch_result.parsed_patch + code_pairs: list[tuple[PatchFile, str]] = [] + for pf in parsed.files: + path = pf.clean_target_path + lang = infer_language_from_path(path) + if lang and lang not in NON_CODE_LANGUAGES and not TEST_FILE_RE.search(path): + code_pairs.append((pf, lang)) + + for pf, lang in code_pairs[:_MAX_PATCH_FILES]: + target = pf.clean_target_path + lines.append(f"### `{target}`") + lines.append("") + + for hunk in pf.hunks: + if hunk.removed_lines: + lines.append("**Vulnerable code (removed):**") + code = "\n".join(hunk.removed_lines[:_MAX_SNIPPET_LINES]) + lines.append(f"```{lang}") + lines.append(code) + lines.append("```") + if len(hunk.removed_lines) > _MAX_SNIPPET_LINES: + lines.append(f"_... +{len(hunk.removed_lines) - _MAX_SNIPPET_LINES} more lines_") + lines.append("") + + if hunk.added_lines: + lines.append("**Fix code (added):**") + code = "\n".join(hunk.added_lines[:_MAX_SNIPPET_LINES]) + lines.append(f"```{lang}") + lines.append(code) + lines.append("```") + if len(hunk.added_lines) > _MAX_SNIPPET_LINES: + lines.append(f"_... +{len(hunk.added_lines) - _MAX_SNIPPET_LINES} more lines_") + lines.append("") + + if len(code_pairs) > _MAX_PATCH_FILES: + lines.append(f"_... +{len(code_pairs) - _MAX_PATCH_FILES} more files in patch_") + + result = "\n".join(lines).strip() + return result or None def parse_agent_morpheus_engine_output(vuln_id: str, checklist_results: list[dict[str, typing.Any]], summary: str, - justification: dict[str, str]) -> AgentMorpheusEngineOutput: + justification: dict[str, str], + intel_score: int, + cvss: dict[str, str] | None, + patch_result: WebPatchResult | dict | None = None) -> AgentMorpheusEngineOutput: """ Parse the output fields for a single vulnerability into an AgentMorpheusEngineOutput object. """ @@ -97,10 +192,22 @@ def parse_agent_morpheus_engine_output(vuln_id: str, reason=justification["justification"], status=justification["affected_status"]) + # Combine CVSS model outputs into a single CVSSOutput object + if cvss: + cvss_output = CVSSOutput(vector_string=cvss[CVSS_VECTOR_STRING], + score=cvss[CVSS_SCORE]) + else: + cvss_output=None + + details = _build_full_pipeline_details_md(patch_result) + return AgentMorpheusEngineOutput(vuln_id=vuln_id, checklist=checklist_output, summary=summary, - justification=justification_output) + justification=justification_output, + intel_score=intel_score, + cvss=cvss_output, + details=details) def build_deficient_intel_output(vuln_id: str) -> AgentMorpheusEngineOutput: @@ -110,6 +217,8 @@ def build_deficient_intel_output(vuln_id: str) -> AgentMorpheusEngineOutput: justification = JustificationOutput(label="insufficient_intel", reason="Insufficient intel available for CVE", status="UNKNOWN") + cvss = None + return AgentMorpheusEngineOutput( vuln_id=vuln_id, checklist=[ @@ -118,25 +227,63 @@ def build_deficient_intel_output(vuln_id: str) -> AgentMorpheusEngineOutput: intermediate_steps=None) ], summary=summary, - justification=justification) - + justification=justification, + intel_score=0, + cvss=cvss) + + +def build_no_vuln_packages_output( + vuln_id: str, + checked_not_vulnerable: list[CheckedNotVulnerablePackage] | None = None +) -> AgentMorpheusEngineOutput: + if checked_not_vulnerable: + n = len(checked_not_vulnerable) + reasons = {pkg.reason for pkg in checked_not_vulnerable} + package_lines = "\n".join( + f"- {pkg.name} ({pkg.version})" for pkg in checked_not_vulnerable + ) + if len(reasons) == 1: + shared_reason = next(iter(reasons)) + summary = f"Not vulnerable — {shared_reason}." + justification_reason = (f"Not vulnerable — {shared_reason}. {n} package(s)\n" + f" checked:\n" + f"{package_lines}") + response_text = (f"Version check completed — {n} package(s) matched but none are vulnerable. " + f"Reason: {shared_reason}.\n\n" + f"{package_lines}") + else: + package_lines_with_reason = "\n".join( + f"- {pkg.name} ({pkg.version}) — {pkg.reason}" + for pkg in checked_not_vulnerable + ) + summary = f"Not vulnerable — {n} package(s) checked." + justification_reason = (f"Not vulnerable. {n} package(s) checked:\n\n" + f"{package_lines_with_reason}") + response_text = (f"Version check completed — {n} package(s) matched but none are vulnerable.\n\n" + f"{package_lines_with_reason}") + else: + summary = ("The VulnerableDependencyChecker did not find any vulnerable packages " + "or dependencies in the SBOM.") + justification_reason = "No vulnerable packages or dependencies were detected in the SBOM." + response_text = ("The VulnerableDependencyChecker did not find any vulnerable packages " + "or dependencies in the SBOM and so the agent was bypassed.") -def build_no_vuln_packages_output(vuln_id: str) -> AgentMorpheusEngineOutput: - summary = ("The VulnerableDependencyChecker did not find any vulnerable packages " - "or dependencies in the SBOM.") justification = JustificationOutput(label="false_positive", - reason="No vulnerable packages or dependencies were detected in the SBOM.", + reason=justification_reason, status="FALSE") + cvss = None + return AgentMorpheusEngineOutput( vuln_id=vuln_id, checklist=[ ChecklistItemOutput(input="Agent bypassed: no vulnerable packages detected. Checklist not generated.", - response=("The VulnerableDependencyChecker did not find any vulnerable packages " - "or dependencies in the SBOM and so the agent was bypassed."), + response=response_text, intermediate_steps=None) ], summary=summary, - justification=justification) + justification=justification, + intel_score=0, + cvss=cvss) def build_no_sbom_output(vuln_id: str) -> AgentMorpheusEngineOutput: @@ -146,6 +293,8 @@ def build_no_sbom_output(vuln_id: str) -> AgentMorpheusEngineOutput: justification = JustificationOutput(label="no_sbom_packages", reason="No SBOM packages found for image.", status="UNKNOWN") + cvss = None + return AgentMorpheusEngineOutput(vuln_id=vuln_id, checklist=[ ChecklistItemOutput( @@ -154,15 +303,43 @@ def build_no_sbom_output(vuln_id: str) -> AgentMorpheusEngineOutput: intermediate_steps=None) ], summary=summary, - justification=justification) + justification=justification, + intel_score=0, + cvss=cvss) +def build_low_intel_score_output(vuln_id: str, intel_score: int) -> AgentMorpheusEngineOutput: + summary = ("There is poor quality intel available to determine vulnerability. There is not enough gathered intel" + " for the agent to make an informed decision.") + justification = JustificationOutput(label="poor_quality_intel", + reason="Poor quality intel for CVE", + status="UNKNOWN") + cvss = None + + return AgentMorpheusEngineOutput( + vuln_id=vuln_id, + checklist=[], + summary=summary, + justification=justification, + intel_score=intel_score, + cvss=cvss + ) + def postprocess_engine_output(message: AgentMorpheusEngineInput, result: AgentMorpheusEngineState) -> AgentMorpheusOutput: - vulnerable_dependencies: list[VulnerableDependencies] = message.info.vulnerable_dependencies + trace_id.set(message.input.scan.id) + vulnerable_dependencies: list[VulnerableDependencies] | None = message.info.vulnerable_dependencies - no_vulns = [vuln_dep.vuln_id for vuln_dep in vulnerable_dependencies if len(vuln_dep.vulnerable_sbom_packages) == 0] + if vulnerable_dependencies is None: + vdc_skipped_vulns = [] + vdc_map: dict[str, VulnerableDependencies] = {} + else: + vdc_skipped_vulns = [ + v.vuln_id for v in vulnerable_dependencies + if len(v.vulnerable_sbom_packages) == 0 and len(v.vuln_package_intel_sources) > 0 + ] + vdc_map = {v.vuln_id: v for v in vulnerable_dependencies} deficient_intel = [i.vuln_id for i in message.info.intel if not i.has_sufficient_intel_for_agent] @@ -171,29 +348,74 @@ def postprocess_engine_output(message: AgentMorpheusEngineInput, # For each vuln_id, get LLM Engine output if it exists # or create placeholder output if it skipped the workflow - if not message.info.sbom.packages: + if not message.input.image.analysis_type == AnalysisType.SOURCE and not message.info.sbom.packages: output = [build_no_sbom_output(vuln_id) for vuln_id in input_vuln_ids] else: output: list[AgentMorpheusEngineOutput] = [] output_vuln_ids = list(result.final_summaries.keys()) + + poor_quality_intel_vul = result.poor_quality_intel_vul + intel_map = {cve.vuln_id: cve for cve in message.info.intel} + for vuln_id in input_vuln_ids: if vuln_id in output_vuln_ids: + justification = result.justifications[vuln_id] + is_vulnerable = justification.get("affected_status") == "TRUE" output.append( parse_agent_morpheus_engine_output(vuln_id=vuln_id, checklist_results=result.checklist_results[vuln_id], summary=result.final_summaries[vuln_id], - justification=result.justifications[vuln_id])) + justification=justification, + intel_score=intel_map[vuln_id].intel_score, + cvss=result.cvss_results.get(vuln_id, None), + patch_result=result.patch_results.get(vuln_id) if is_vulnerable else None)) elif vuln_id in deficient_intel: output.append(build_deficient_intel_output(vuln_id)) - elif vuln_id in no_vulns: - output.append(build_no_vuln_packages_output(vuln_id)) + elif vuln_id in poor_quality_intel_vul: + output.append(build_low_intel_score_output(vuln_id, poor_quality_intel_vul[vuln_id])) + elif vuln_id in vdc_skipped_vulns: + checked_not_vulnerable = vdc_map[vuln_id].checked_not_vulnerable if vuln_id in vdc_map else None + output.append(build_no_vuln_packages_output(vuln_id, checked_not_vulnerable)) else: assert False, "CVE has vulnerable dependencies but there is no workflow output." for out in output: - logger.info("Vulnerability '%s' affected status: %s. Label: %s", + logger.info("Vulnerability '%s' affected status: %s. Label: %s. CVSS score: %s", out.vuln_id, out.justification.status, - out.justification.label) + out.justification.label, + out.cvss.score if out.cvss else "-") + + payload = OutputPayload(analysis=output, vex=result.vex) + return AgentMorpheusOutput(input=message.input, info=message.info, output=payload) + +def finalize_preprocess_engine_input(message: AgentMorpheusEngineInput, engine_state: AgentMorpheusEngineState, builder: Builder) -> AgentMorpheusEngineState: + config = builder.get_function_config("cve_calculate_intel_score") + assert isinstance(config, CVECalculateIntelScoreConfig) + + if config.generate_intel_score: + cve_list = [] + poor_quality_intel_vul: dict[str, int] = {} + + # Create a map of the full cve's for convenience + intel_map = {cve.vuln_id: cve for cve in message.info.intel} + + for filtered_cve in engine_state.cve_intel: + full_cve = intel_map.get(filtered_cve.vuln_id) + + if full_cve: + # Check the intel score is below the threshold + if full_cve.get_intel_score() < config.intel_low_score: + poor_quality_intel_vul[filtered_cve.vuln_id] = full_cve.get_intel_score() + + # Check if to force analysis + if config.insist_analysis: + cve_list.append(filtered_cve) + else: + cve_list.append(filtered_cve) + + engine_state.cve_intel = cve_list + engine_state.poor_quality_intel_vul = poor_quality_intel_vul + + return engine_state - return AgentMorpheusOutput(input=message.input, info=message.info, output=output) diff --git a/src/vuln_analysis/utils/lock_file_parsers.py b/src/vuln_analysis/utils/lock_file_parsers.py new file mode 100644 index 000000000..60166c26f --- /dev/null +++ b/src/vuln_analysis/utils/lock_file_parsers.py @@ -0,0 +1,243 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Lock file parsers for extracting dependency {package: version} mappings. + +Each ecosystem has its own parser function that can be imported directly. +""" + +import os +import re +import subprocess +from pathlib import Path + +from exploit_iq_commons.utils.dep_tree import TRANSITIVE_ENV_NAME, resolve_mvn_command, run_command +from exploit_iq_commons.utils.java_utils import parse_depgraph_line +from exploit_iq_commons.logging.loggers_factory import LoggingFactory + + +def _pep503_normalize(name: str) -> str: + """Normalize package name per PEP 503. + + Converts all runs of dashes, underscores, and dots to a single dash, + and lowercases the result. This ensures equivalent package names + compare equal (e.g., zope.interface == zope-interface == zope_interface). + """ + return re.sub(r'[-_.]', '-', name.lower()) + +logger = LoggingFactory.get_agent_logger(__name__) + + +# ============================================================ +# IMPLEMENTED PARSERS +# ============================================================ + +def parse_go_modules(repo_path: Path) -> dict[str, str]: + """ + Get resolved Go module versions using 'go list -m all'. + + This runs Go's Minimal Version Selection (MVS) algorithm to determine + the actual versions that will be compiled into the binary, rather than + parsing go.sum which may contain superseded versions. + + Delegates to GoDependencyTreeBuilder to avoid duplicating Go command + execution logic. + + Args: + repo_path: Path to the repository root containing go.mod + + Returns: + dict mapping module paths to versions (without 'v' prefix) + + Raises: + RuntimeError: If 'go list -m all' fails + """ + from exploit_iq_commons.utils.dep_tree import GoDependencyTreeBuilder + + packages = GoDependencyTreeBuilder.get_resolved_modules(repo_path) + logger.debug("Parsed %d packages from 'go list -m all'", len(packages)) + return packages + + +def parse_python_venv(repo_path: Path) -> dict[str, str]: + """ + Parse installed Python dependencies from venv using uv pip freeze. + + Extracts package names and versions from an installed virtual environment + by running `uv pip freeze`. Package names are normalized per PEP 503 + (dashes, underscores, and dots all become dashes, lowercased) for + consistent matching. + + Args: + repo_path: Path to repository containing the transitive_env venv + + Returns: + dict mapping normalized package names to versions + + Raises: + FileNotFoundError: If venv doesn't exist + """ + venv_path = repo_path / TRANSITIVE_ENV_NAME + if not venv_path.exists(): + raise FileNotFoundError(f"Virtual environment not found at {venv_path}") + + venv_python = venv_path / "bin" / "python" + cmd = ['uv', 'pip', 'freeze', '--python', str(venv_python)] + output = run_command(cmd) + + packages: dict[str, str] = {} + if not output: + logger.warning("uv pip freeze returned empty output for %s", venv_path) + return packages + + for line in output.splitlines(): + line = line.strip() + if not line: + continue + + # Skip editable installs (-e) + if line.startswith('-e'): + logger.debug("Skipping editable install: %s", line) + continue + + # Skip direct URL references (package @ url) + if ' @ ' in line: + logger.debug("Skipping URL reference: %s", line) + continue + + # Parse package==version format + if '==' in line: + name, version = line.split('==', 1) + normalized_name = _pep503_normalize(name) + packages[normalized_name] = version + + logger.debug("Parsed %d packages from Python venv", len(packages)) + return packages + + +# ============================================================ +# JAVA ECOSYSTEM +# ============================================================ + + +def parse_dependency_tree_txt(repo_path: Path) -> dict[str, str]: + """ + Parse dependency_tree.txt generated by depgraph-maven-plugin. + + The file is generated by JavaDependencyTreeBuilder.build_tree() using: + mvn com.github.ferstl:depgraph-maven-plugin:aggregate -DgraphFormat=text + + Format: groupId:artifactId:version:type:scope (indented tree structure) + Example: org.apache.activemq:artemis-openwire-protocol:2.28.0:bundle:compile + + Args: + repo_path: Path to directory containing dependency_tree.txt + + Returns: + dict mapping groupId:artifactId to version + + Raises: + FileNotFoundError: If dependency_tree.txt doesn't exist + """ + import re + + tree_file = repo_path / "dependency_tree.txt" + if not tree_file.exists(): + raise FileNotFoundError(f"dependency_tree.txt not found at {tree_file}") + + packages: dict[str, str] = {} + + for line in tree_file.read_text().splitlines(): + coord = _extract_maven_coordinate(line) + if coord: + group, artifact, version = coord + ga_key = f"{group}:{artifact}" + if ga_key not in packages: + packages[ga_key] = version + + logger.debug("Parsed %d packages from dependency_tree.txt", len(packages)) + return packages + + +def _extract_maven_coordinate(line: str) -> tuple[str, str, str] | None: + """ + Extract (groupId, artifactId, version) from a depgraph-maven-plugin line. + + Handles formats like: + - org.apache.activemq:artemis-server:2.28.0:jar:compile + - +- org.apache.activemq:artemis-core-client:2.28.0:jar:compile + - | +- io.netty:netty-buffer:4.1.86.Final:jar:compile + - (com.google.guava:guava:31.1-jre:jar:compile) + + Args: + line: A single line from dependency_tree.txt + + Returns: + Tuple of (groupId, artifactId, version) or None if not parseable + """ + coord = parse_depgraph_line(line) + if coord is None: + return None + return (coord.group_id, coord.artifact_id, coord.version) + + +def _generate_dependency_tree_txt(repo_path: Path) -> bool: + """ + Generate dependency_tree.txt using depgraph-maven-plugin. + + Uses the same command as JavaDependencyTreeBuilder.build_tree() but without + the targetIncludes filter (we want all dependencies for verification). + + Args: + repo_path: Path to the repository root containing pom.xml + + Returns: + True if successful, False otherwise. + """ + mvn_cmd = resolve_mvn_command(repo_path) + output_file = repo_path / "dependency_tree.txt" + + settings_path = os.getenv("JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH") + + cmd = [ + mvn_cmd, + "com.github.ferstl:depgraph-maven-plugin:4.0.3:aggregate", + "-DgraphFormat=text", + "-DshowGroupIds", + "-DshowVersions", + "-DshowTypes", + "-DoutputDirectory=.", + "-DoutputFileName=dependency_tree.txt", + "-DclasspathScope=runtime", + ] + + if settings_path: + cmd.insert(2, "-s") + cmd.insert(3, settings_path) + + try: + result = subprocess.run( + cmd, + cwd=repo_path, + capture_output=True, + timeout=300, + ) + return result.returncode == 0 and output_file.exists() + except (subprocess.TimeoutExpired, FileNotFoundError) as e: + logger.warning("depgraph-maven-plugin failed: %s", e) + return False + + diff --git a/src/vuln_analysis/utils/osv_patch_retriever.py b/src/vuln_analysis/utils/osv_patch_retriever.py new file mode 100644 index 000000000..132cd62b5 --- /dev/null +++ b/src/vuln_analysis/utils/osv_patch_retriever.py @@ -0,0 +1,571 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""OSV Patch Retriever - fetch upstream fix patches from OSV when RPM patches are unavailable. + +.. deprecated:: + This module is deprecated. Use :mod:`vuln_analysis.utils.web_patch_fetcher` instead. + The new module provides a unified interface for fetching patches from: + - Intel references (GHSA, NVD, RHSA, Ubuntu) + - OSV API + - kernel.org (in addition to GitHub) + + Migration guide: + - Replace `OSVPatchRetriever` with `OSVClient` from `web_patch_fetcher` + - Replace `fetch_ubuntu_patch` with `WebPatchFetcher.fetch_from_intel_refs` +""" + +import warnings + +warnings.warn( + "osv_patch_retriever is deprecated, use web_patch_fetcher instead", + DeprecationWarning, + stacklevel=2, +) + +from __future__ import annotations + +import os +import re +from typing import TYPE_CHECKING + +import aiohttp +from pydantic import BaseModel +from unidiff import PatchSet + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.utils.async_http_utils import request_with_retry +from vuln_analysis.functions.code_agent_graph_defs import OSVPatchResult + +if TYPE_CHECKING: + from vuln_analysis.functions.code_agent_graph_defs import ParsedPatch + +logger = LoggingFactory.get_agent_logger(__name__) + +_OSV_API_URL = os.environ.get("OSV_API_URL", "https://api.osv.dev/v1/vulns/") +_OSV_TIMEOUT_SECONDS = int(os.environ.get("OSV_TIMEOUT_SECONDS", "10")) +_GITHUB_PATCH_TIMEOUT_SECONDS = int(os.environ.get("GITHUB_PATCH_TIMEOUT_SECONDS", "30")) + +_BINARY_FILE_EXTENSIONS = frozenset({ + '.uu', '.uue', '.iso', '.bin', '.gz', '.bz2', '.xz', '.zip', '.tar', '.tgz', '.tbz2', + '.png', '.jpg', '.jpeg', '.gif', '.bmp', '.ico', '.webp', + '.pdf', '.doc', '.docx', '.xls', '.xlsx', + '.exe', '.dll', '.so', '.dylib', '.a', '.o', '.obj', + '.pyc', '.pyo', '.class', '.jar', '.war', +}) + +_GITHUB_REPO_PATTERN = re.compile(r"https?://github\.com/([^/]+/[^/]+?)(?:\.git)?/?$") + + +class OSVAffectedRange(BaseModel): + """Represents a Git range from an OSV affected block.""" + repo_url: str | None = None + fixed_commit: str | None = None + introduced_commit: str | None = None + + + +def _is_binary_file_path(path: str) -> bool: + """Check if file path has a binary file extension.""" + path_lower = path.lower() + return any(path_lower.endswith(ext) for ext in _BINARY_FILE_EXTENSIONS) + + +def _version_in_range(version: str, introduced: str | None, fixed: str | None) -> bool: + """Check if provided upstream version falls within [introduced, fixed) range. + + Returns True if version >= introduced (or introduced is None) AND version < fixed (or fixed is None). + """ + try: + from packaging.version import parse as parse_version + v = parse_version(version) + if introduced: + try: + if v < parse_version(introduced): + return False + except Exception: + pass + if fixed: + try: + if v >= parse_version(fixed): + return False + except Exception: + pass + return True + except Exception: + return True + + +def _parse_patch_content(patch_content: str, patch_filename: str) -> "ParsedPatch | None": + """Parse patch content string into structured ParsedPatch model. + + Reuses the same logic as code_agent_graph_defs.parse_patch_file but works on string content. + """ + from vuln_analysis.functions.code_agent_graph_defs import ParsedPatch, PatchFile, PatchHunk + + try: + patch_set = PatchSet.from_string(patch_content) + except Exception: + logger.warning("_parse_patch_content: failed to parse patch content") + return None + + files: list[PatchFile] = [] + for patched_file in patch_set: + if patched_file.is_binary_file: + continue + if _is_binary_file_path(patched_file.target_file): + continue + + hunks: list[PatchHunk] = [] + for hunk in patched_file: + context, removed, added = [], [], [] + for line in hunk: + if line.is_context: + context.append(str(line.value).rstrip("\n")) + elif line.is_removed: + removed.append(str(line.value).rstrip("\n")) + elif line.is_added: + added.append(str(line.value).rstrip("\n")) + + hunks.append(PatchHunk( + source_start=hunk.source_start, + source_length=hunk.source_length, + target_start=hunk.target_start, + target_length=hunk.target_length, + section_header=hunk.section_header or "", + context_lines=context, + removed_lines=removed, + added_lines=added, + )) + + files.append(PatchFile( + source_path=patched_file.source_file, + target_path=patched_file.target_file, + hunks=hunks, + is_new_file=patched_file.is_added_file, + is_deleted_file=patched_file.is_removed_file, + )) + + return ParsedPatch(patch_filename=patch_filename, files=files) + + +def _extract_commit_metadata(patch_content: str) -> tuple[str | None, str | None, str | None]: + """Extract commit message, author, and date from GitHub .patch format. + + GitHub .patch format starts with: + From Mon Sep 17 00:00:00 2001 + From: Author Name + Date: Tue, 1 Jan 2024 12:00:00 +0000 + Subject: [PATCH] Commit message + + Extended commit message... + --- + + """ + lines = patch_content.split('\n') + author = None + date = None + subject_lines = [] + in_subject = False + + for line in lines: + if line.startswith('From:'): + author = line[5:].strip() + elif line.startswith('Date:'): + date = line[5:].strip() + elif line.startswith('Subject:'): + in_subject = True + subject_part = line[8:].strip() + if subject_part.startswith('[PATCH'): + idx = subject_part.find(']') + if idx != -1: + subject_part = subject_part[idx + 1:].strip() + subject_lines.append(subject_part) + elif in_subject: + if line.startswith('---') or line.startswith('diff --git'): + break + if line.strip() == '': + in_subject = False + else: + subject_lines.append(line.strip()) + + commit_message = ' '.join(subject_lines).strip() if subject_lines else None + return commit_message, author, date + + +_UBUNTU_PATCH_URL_PATTERN = re.compile( + r"(?:upstream:\s*)?(https://github\.com/[^/]+/[^/]+/commit/([a-f0-9]+))" +) + + +async def fetch_ubuntu_patch( + session: aiohttp.ClientSession, + cve_id: str, + package_name: str, + patch_refs: list[str], + timeout: int = _GITHUB_PATCH_TIMEOUT_SECONDS, +) -> OSVPatchResult | None: + """Fetch patch from Ubuntu intel patches field. + + .. deprecated:: + Use :class:`vuln_analysis.utils.web_patch_fetcher.WebPatchFetcher.fetch_from_intel_refs` instead. + + Parses Ubuntu patch refs like: "upstream: https://github.com/curl/curl/commit/39d1976b7f..." + + Args: + session: aiohttp ClientSession for HTTP requests + cve_id: CVE identifier (e.g., "CVE-2024-1234") + package_name: Package name (for logging) + patch_refs: List of patch reference strings from Ubuntu intel + timeout: Timeout for GitHub fetch in seconds + + Returns: + OSVPatchResult with parsed patch data, or None if no valid patch found + """ + github_timeout = aiohttp.ClientTimeout(total=timeout) + + for patch_ref in patch_refs: + match = _UBUNTU_PATCH_URL_PATTERN.search(patch_ref) + if not match: + logger.debug("Ubuntu patch ref does not match GitHub commit pattern: %s", patch_ref) + continue + + commit_url = match.group(1) + commit_sha = match.group(2) + patch_url = f"{commit_url}.patch" + repo_url = commit_url.rsplit("/commit/", 1)[0] + + logger.info("Ubuntu: Fetching patch for %s from %s", cve_id, patch_url) + + try: + async with request_with_retry( + session=session, + request_kwargs={ + 'method': 'GET', + 'url': patch_url, + 'timeout': github_timeout, + }, + max_retries=3, + sleep_time=0.5, + log_on_error=False, + ) as response: + patch_content = await response.text() + except aiohttp.ClientResponseError as e: + if e.status == 404: + logger.info("Ubuntu: GitHub patch not found: %s", patch_url) + else: + logger.warning("Ubuntu: GitHub patch fetch failed: %s - %s", patch_url, e) + continue + except Exception as e: + logger.warning("Ubuntu: GitHub patch fetch failed: %s - %s", patch_url, e) + continue + + if not patch_content: + continue + + commit_message, commit_author, commit_date = _extract_commit_metadata(patch_content) + parsed_patch = _parse_patch_content(patch_content, f"{cve_id}_{commit_sha[:8]}.patch") + + if parsed_patch: + logger.info("Ubuntu: Successfully fetched and parsed patch for %s", cve_id) + return OSVPatchResult( + cve_id=cve_id, + fixed_commit=commit_sha[:8], + repo_url=repo_url, + patch_url=patch_url, + patch_content=patch_content, + parsed_patch=parsed_patch, + commit_message=commit_message, + commit_author=commit_author, + commit_date=commit_date, + ) + + logger.info("Ubuntu: No valid GitHub commit patch found for %s in %d refs", cve_id, len(patch_refs)) + return None + + +class OSVPatchRetriever: + """Retrieve upstream fix patches from OSV when RPM patches are unavailable. + + .. deprecated:: + Use :class:`vuln_analysis.utils.web_patch_fetcher.OSVClient` instead. + The new class provides the same functionality with additional support + for kernel.org and better integration with intel references. + + Usage: + async with aiohttp.ClientSession() as session: + retriever = OSVPatchRetriever(session=session) + result = await retriever.get_fix_patch("CVE-2024-1234", "3.0.7", "openssl") + if result and result.parsed_patch: + # Use result.parsed_patch for agent context + pass + """ + + def __init__( + self, + session: aiohttp.ClientSession, + osv_timeout: int = _OSV_TIMEOUT_SECONDS, + github_timeout: int = _GITHUB_PATCH_TIMEOUT_SECONDS, + ): + warnings.warn( + "OSVPatchRetriever is deprecated, use OSVClient from web_patch_fetcher instead", + DeprecationWarning, + stacklevel=2, + ) + self._session = session + self._osv_timeout = aiohttp.ClientTimeout(total=osv_timeout) + self._github_timeout = aiohttp.ClientTimeout(total=github_timeout) + + async def get_fix_patch( + self, + cve_id: str, + upstream_version: str, + package_name: str | None = None, + ) -> OSVPatchResult | None: + """Main entry point - orchestrates the full workflow. + + Args: + cve_id: CVE identifier (e.g., "CVE-2024-1234") + upstream_version: Upstream version from TargetPackage.version (e.g., "3.0.7") + package_name: Optional package name to help match the correct affected block + + Returns: + OSVPatchResult with patch data, or None if no fix found + """ + try: + osv_data = await self._query_osv(cve_id) + if not osv_data: + return None + + # 1. Try to get the highly-specific patch URL from references first + patch_url = self._extract_commit_from_references(osv_data) + fixed_commit = None + repo_url = None + if patch_url: + # Extract repo_url and fixed_commit from the patch_url for the result object + repo_url = patch_url.split('/commit/')[0] if '/commit/' in patch_url else patch_url.split('/pull/')[0] + fixed_commit = patch_url.split('/')[-1].replace('.patch', '') + logger.info("OSV: Found precise fix commit in references for %s", cve_id) + else: + # second try to find the fix commit from the affected block + affected = self._find_matching_affected(osv_data, package_name) + if not affected: + logger.info("OSV: No affected block with fix found for %s", cve_id) + return None + + range_info = self._extract_fix_commit(affected) + if not range_info.fixed_commit or not range_info.repo_url: + logger.info("OSV: No fixed commit found for %s", cve_id) + return None + + patch_url = self._build_patch_url(range_info.repo_url, range_info.fixed_commit) + fixed_commit = range_info.fixed_commit[:8] + repo_url = range_info.repo_url + if not patch_url: + logger.info("OSV: Could not build patch URL for %s (non-GitHub repo?)", cve_id) + return None + + patch_content = await self._fetch_github_patch(patch_url) + if not patch_content: + return None + + commit_message, commit_author, commit_date = _extract_commit_metadata(patch_content) + parsed_patch = _parse_patch_content(patch_content, f"{cve_id}_{fixed_commit}.patch") + + return OSVPatchResult( + cve_id=cve_id, + fixed_commit=fixed_commit, + repo_url=repo_url, + patch_url=patch_url, + patch_content=patch_content, + parsed_patch=parsed_patch, + commit_message=commit_message, + commit_author=commit_author, + commit_date=commit_date, + ) + + except Exception: + logger.warning("OSV patch retrieval failed for %s", cve_id, exc_info=True) + return None + + async def _query_osv(self, cve_id: str) -> dict | None: + """Query OSV API for CVE data. + + Args: + cve_id: CVE identifier + + Returns: + OSV vulnerability data dict, or None on failure + """ + url = f"{_OSV_API_URL}{cve_id}" + try: + async with request_with_retry( + session=self._session, + request_kwargs={ + 'method': 'GET', + 'url': url, + 'timeout': self._osv_timeout, + }, + max_retries=3, + sleep_time=0.5, + log_on_error=False, + ) as response: + return await response.json() + except aiohttp.ClientResponseError as e: + if e.status == 404: + logger.info("OSV: CVE %s not found", cve_id) + else: + logger.warning("OSV query failed for %s: %s", cve_id, e) + return None + except Exception as e: + logger.warning("OSV query failed for %s: %s", cve_id, e) + return None + + def _extract_commit_from_references(self, osv_data: dict) -> str | None: + """Attempt to find the exact fix commit URL from the OSV references array. + + Args: + osv_data: OSV vulnerability data dict + + Returns: + The patch URL if found, otherwise None + """ + references = osv_data.get("references", []) + + for ref in references: + if ref.get("type") == "FIX": + url = ref.get("url", "") + # We look for GitHub URLs containing either /commit/ or /pull/ + if "github.com" in url and ("/commit/" in url or "/pull/" in url): + if not url.endswith(".patch"): + return f"{url}.patch" + return url + + return None + + def _find_matching_affected( + self, + osv_data: dict, + package_name: str | None = None, + ) -> dict | None: + """Find an affected block that has a GIT range with a fixed commit. + + Args: + osv_data: OSV vulnerability data + package_name: Optional package name to filter affected blocks + + Returns: + Matching affected block dict, or None if no match + """ + + + for affected in osv_data.get("affected", []): + + + for range_block in affected.get("ranges", []): + if range_block.get("type") == "GIT": + for event in range_block.get("events", []): + if "fixed" in event: + return affected + + return None + + def _extract_fix_commit(self, affected: dict) -> OSVAffectedRange: + """Extract the fixed commit hash and repo URL from an affected block. + + Args: + affected: OSV affected block + + Returns: + OSVAffectedRange with repo_url and fixed_commit + """ + result = OSVAffectedRange() + + ranges = affected.get("ranges", []) + for range_block in ranges: + if range_block.get("type") != "GIT": + continue + + repo = range_block.get("repo") + if repo: + result.repo_url = repo + + events = range_block.get("events", []) + for event in events: + if "introduced" in event and event["introduced"] != "0": + result.introduced_commit = event["introduced"] + if "fixed" in event: + result.fixed_commit = event["fixed"] + + if result.fixed_commit: + break + + return result + + def _build_patch_url(self, repo_url: str, commit_sha: str) -> str | None: + """Build GitHub patch URL from repo URL and commit SHA. + + Args: + repo_url: Git repository URL (e.g., "https://github.com/openssl/openssl") + commit_sha: Git commit hash + + Returns: + Patch URL (e.g., "https://github.com/openssl/openssl/commit/.patch"), + or None if not a GitHub repo + """ + match = _GITHUB_REPO_PATTERN.match(repo_url) + if not match: + if "github.com" in repo_url: + parts = repo_url.rstrip('/').split('/') + if len(parts) >= 2: + repo_path = '/'.join(parts[-2:]).replace('.git', '') + return f"https://github.com/{repo_path}/commit/{commit_sha}.patch" + logger.debug("Non-GitHub repo URL: %s", repo_url) + return None + + repo_path = match.group(1) + return f"https://github.com/{repo_path}/commit/{commit_sha}.patch" + + async def _fetch_github_patch(self, patch_url: str) -> str | None: + """Download patch content from GitHub. + + Args: + patch_url: URL to the .patch file + + Returns: + Patch content string, or None on failure + """ + try: + async with request_with_retry( + session=self._session, + request_kwargs={ + 'method': 'GET', + 'url': patch_url, + 'timeout': self._github_timeout, + }, + max_retries=3, + sleep_time=0.5, + log_on_error=False, + ) as response: + return await response.text() + except aiohttp.ClientResponseError as e: + if e.status == 404: + logger.info("GitHub patch not found: %s", patch_url) + else: + logger.warning("GitHub patch fetch failed: %s - %s", patch_url, e) + return None + except Exception as e: + logger.warning("GitHub patch fetch failed: %s - %s", patch_url, e) + return None diff --git a/src/vuln_analysis/utils/output_formatter.py b/src/vuln_analysis/utils/output_formatter.py index 1773dde02..c01b9d9be 100644 --- a/src/vuln_analysis/utils/output_formatter.py +++ b/src/vuln_analysis/utils/output_formatter.py @@ -20,7 +20,7 @@ from dateutil.parser import parse from vuln_analysis.data_models.output import AgentMorpheusOutput -from vuln_analysis.utils.data_utils import safe_getattr +from exploit_iq_commons.utils.data_utils import safe_getattr def generate_vulnerability_reports(model_dict: AgentMorpheusOutput, output_dir): @@ -104,12 +104,12 @@ def _add_header(markdown_content, model_dict: AgentMorpheusOutput): """ input_image = model_dict.input.image # iterate over a list of dict objects, with vuln_id and ghsa being 2 keys in each element - for output in model_dict.output: + for output in model_dict.output.analysis: cve_id = output.vuln_id markdown_content[cve_id].append(f"# Vulnerability Analysis Report for {cve_id}") markdown_content[cve_id].append(f"> **Container Analyzed:** `{input_image.name}:{input_image.tag}`\n\n") # Only add SBOM info if it is a file location - if input_image.sbom_info.type == "file": + if input_image.sbom_info and input_image.sbom_info.type == "file": markdown_content[cve_id].append(f"> **SBOM Info:** `{input_image.sbom_info}`\n\n") markdown_content[cve_id].append(f"> **Status:** {_get_expoiltability_text(output.justification.status)}") @@ -279,7 +279,7 @@ def _add_table_of_contents(markdown_content, model_dict: AgentMorpheusOutput): None This function modifies `markdown_content` in place. """ - for entry in model_dict.output: + for entry in model_dict.output.analysis: cve_id = entry.vuln_id checklist = entry.checklist markdown_content[cve_id].append("### Checklist ") @@ -313,7 +313,7 @@ def _add_checklist_info(markdown_content, model_dict: AgentMorpheusOutput): None This function modifies `markdown_content` in place. """ - for entry in model_dict.output: + for entry in model_dict.output.analysis: cve_id = entry.vuln_id checklist = entry.checklist if checklist: @@ -424,7 +424,7 @@ def _add_vulnerability_analysis(markdown_content, model_dict: AgentMorpheusOutpu None This function modifies `markdown_content` in place. """ - for entry in model_dict.output: + for entry in model_dict.output.analysis: cve_id = entry.vuln_id summary = entry.summary justification = entry.justification @@ -454,6 +454,9 @@ def _add_vulnerable_sboms(markdown_content, model_dict: AgentMorpheusOutput): None This function modifies `markdown_content` in place. """ + if model_dict.info.vulnerable_dependencies is None: + return + vulnerable_sboms_per_cve_id = { i.vuln_id: i.vulnerable_sbom_packages for i in model_dict.info.vulnerable_dependencies diff --git a/src/vuln_analysis/utils/package_identifier.py b/src/vuln_analysis/utils/package_identifier.py new file mode 100644 index 000000000..614c820d2 --- /dev/null +++ b/src/vuln_analysis/utils/package_identifier.py @@ -0,0 +1,555 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import re + +from univers import versions + +from exploit_iq_commons.data_models.checker_status import EnumIdentifyResult, PackageCheckerStatus, PackageIdentifyResult +from exploit_iq_commons.data_models.cve_intel import CveIntel +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from exploit_iq_commons.utils.string_utils import package_names_match +from exploit_iq_commons.data_models.common import TargetPackage + +logger = LoggingFactory.get_agent_logger(__name__) + +# NEVRA = Name-Epoch-Version-Release-Architecture, the standard RPM package naming format. +_RPM_NEVRA_RE = re.compile(r"^(.+?)-(?:(\d+):)?(\d\S*?)-(\S+)$") +_DIST_TAG_RE = re.compile(r"(el\d+)") +_ARCH_SUFFIXES = frozenset({"x86_64", "aarch64", "i686", "noarch", "s390x", "ppc64le", "armv7hl", "src"}) + + +def _strip_arch_suffix(release_arch: str) -> str: + """Remove .arch suffix if present, preserving dist tags like .el6_10.""" + if "." in release_arch: + base, suffix = release_arch.rsplit(".", 1) + if suffix in _ARCH_SUFFIXES: + return base + return release_arch + + +def _extract_dist_tag(release: str) -> str | None: + """Extract the RHEL dist-tag family (e.g. 'el8') from a release string.""" + m = _DIST_TAG_RE.search(release) + return m.group(1) if m else None + + +_RHEL_VERSION_RE = re.compile(r"el(\d+)") + +_AFFECTED_FIX_STATES = frozenset({ + "affected", "fix deferred", "under investigation", +}) +_NOT_AFFECTED_FIX_STATES = frozenset({ + "not affected", + "will not fix", + "out of support scope", +}) + + +def _extract_rhel_version(distro_tag: str | None) -> str | None: + """Extract RHEL major version from dist-tag (e.g., 'el7' -> '7', 'el10' -> '10').""" + if not distro_tag: + return None + m = _RHEL_VERSION_RE.match(distro_tag) + return m.group(1) if m else None + + +def _is_fedora_profile() -> bool: + """Check if running in Fedora/external profile mode via RPM_USER_TYPE env var.""" + return os.environ.get("RPM_USER_TYPE", "internal").lower() == "external" + + +def _match_package_state_for_distro( + package_states: list, + target_name: str, + target_distro: str | None, +): + """Find the PackageState entry matching target package name and distro. + + Returns the matching PackageState or None if no match found. + Returns None for Fedora packages (RHSA data is RHEL-specific). + """ + if _is_fedora_profile(): + return None + + rhel_version = _extract_rhel_version(target_distro) + + for ps in package_states: + if not ps.package_name or not package_names_match(target_name, ps.package_name): + continue + if rhel_version is None: + return ps # No distro info, return first name match + # Match by CPE (e.g., "cpe:/o:redhat:enterprise_linux:7") + if ps.cpe and f":enterprise_linux:{rhel_version}" in ps.cpe: + return ps + # Match by product_name (e.g., "Red Hat Enterprise Linux 7") + if ps.product_name and re.search(rf"\b{rhel_version}\b", ps.product_name): + return ps + return None + + +def _interpret_fix_state(fix_state: str | None) -> EnumIdentifyResult | None: + """Interpret RHSA fix_state into an identification result (PackageIdentify step 2). + + Returns: + EnumIdentifyResult.NO for not affected / will not fix / out of support scope + EnumIdentifyResult.YES for affected, fix deferred, under investigation + None if fix_state is unknown or missing (fall through to NVD checks) + """ + if not fix_state: + return None + state = fix_state.lower().strip() + if state in _NOT_AFFECTED_FIX_STATES: + return EnumIdentifyResult.NO + if state in _AFFECTED_FIX_STATES: + return EnumIdentifyResult.YES + return None # Unknown state, fall through to other checks + + +def extract_nvd_version_info(intel: CveIntel, target_name: str) -> tuple[str, str]: + """Extract version range and fixed version from NVD configurations. + + Standalone function for reuse across modules without instantiating PackageIdentifier. + + Args: + intel: CveIntel object containing NVD data. + target_name: Target package name for filtering configurations. + + Returns: + Tuple of (affected_version_range, fixed_version). + Returns ("", "") if no NVD data is available. + """ + if intel.nvd is None or not intel.nvd.configurations: + return "", "" + + ranges = [] + fixed_version = "" + + for config in intel.nvd.configurations: + if not package_names_match(target_name, config.package): + continue + + parts = [] + if config.versionStartIncluding: + parts.append(f">={config.versionStartIncluding}") + if config.versionStartExcluding: + parts.append(f">{config.versionStartExcluding}") + if config.versionEndIncluding: + parts.append(f"<={config.versionEndIncluding}") + if config.versionEndExcluding: + parts.append(f"<{config.versionEndExcluding}") + if not fixed_version: + fixed_version = config.versionEndExcluding + + if parts: + ranges.append(" && ".join(parts)) + + affected_range = " OR ".join(ranges) if ranges else "" + return affected_range, fixed_version + + +class PackageIdentifier: + """ + Deterministic PackageIdentify phase: resolves package identity from intel, + cross-references the SBOM, checks version ranges, and locates RPMs in cache. + """ + + def __init__( + self, + target_package: TargetPackage, + ): + self._target_package = target_package + + + def identify(self, intel: CveIntel | None) -> tuple[PackageCheckerStatus, PackageIdentifyResult]: + """Run PackageIdentify for a single CVE. + + Step 1 — RHSA scope: target must appear in ``package_state`` or + ``affected_release`` when Red Hat published either list; else + ``PKG_IDENT_CVE_MISMATCH``. + Step 2 — Vulnerability posture: ``PKG_IDENT_NOT_VUL`` when RHSA/NVD + shows the target is already fixed, or when fix_state is "will not fix" + or "out of support scope". When fix_state is "not affected", investigation + continues (status=OK) to verify the RHSA assessment and catch potential errors. + """ + + package_identify = PackageIdentifyResult() + status = PackageCheckerStatus.OK + if intel is None: + status = PackageCheckerStatus.ERROR_PKG_IDENT_NO_INTEL + return status, package_identify + + if not self._is_cve_for_target_package(intel): + status = PackageCheckerStatus.PKG_IDENT_CVE_MISMATCH + package_identify.is_target_package_affected = EnumIdentifyResult.NO + return status, package_identify + + package_identify.is_target_package_affected = self._is_target_package_affected(intel,package_identify) + + package_identify.is_target_package_fixed = self._is_target_package_fixed(intel,package_identify) + + # Determine if we should early-exit with PKG_IDENT_NOT_VUL + if package_identify.is_target_package_fixed == EnumIdentifyResult.YES: + # Package is already running the fixed version + status = PackageCheckerStatus.PKG_IDENT_NOT_VUL + elif package_identify.is_target_package_affected == EnumIdentifyResult.NO: + # RHSA says not affected - check if we should continue investigation + # "not affected" continues to verify; other states (will not fix, out of support) early-exit + if package_identify.rhsa_fix_state != "not affected": + status = PackageCheckerStatus.PKG_IDENT_NOT_VUL + # else: status stays OK, continue investigation to verify RHSA assessment + + return status, package_identify + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _find_and_locate_rpm(self, intel: CveIntel) -> list[str]: + """Extract deduplicated RPM package names from RHSA package_state.""" + packages = self._extract_rhsa(intel) + packages = [p for p in packages if "/" not in p.get("package_name", "/")] + seen: set[str] = set() + names: list[str] = [] + for pkg in packages: + name = pkg.get("package_name") + if name and name not in seen: + seen.add(name) + names.append(name) + return names + + def _is_cve_for_target_package(self, intel: CveIntel) -> bool: + """Step 1: target is in scope for this CVE per Red Hat package lists. + + Returns True if: + - OSIDB affects list contains a matching ps_component, OR + - RHSA has no package lists, OR + - Target appears in RHSA package_state or affected_release. + Returns False when intel lists packages but target matches none. + """ + target_name = self._target_package.name + + # Priority 1: Check OSIDB affects (most reliable source when available) + if intel.osidb and intel.osidb.affects: + for affect in intel.osidb.affects: + if affect.ps_component and package_names_match(target_name, affect.ps_component): + logger.debug( + "OSIDB match: target=%s matches ps_component=%s", + target_name, affect.ps_component + ) + return True + + # Priority 2: Fall back to RHSA package lists + if not intel.rhsa: + return True + + has_package_state = bool(intel.rhsa.package_state) + has_affected_release = bool( + getattr(intel.rhsa, "affected_release", None) + ) + if not has_package_state and not has_affected_release: + return True + + if has_package_state: + for ps in intel.rhsa.package_state: + if ps.package_name and package_names_match(target_name, ps.package_name): + return True + if has_affected_release and self._target_in_affected_release(intel): + return True + return False + + def _target_in_affected_release(self, intel: CveIntel) -> bool: + """True when target package name matches an RHSA affected_release NEVRA.""" + target_name = self._target_package.name + for entry in intel.rhsa.affected_release or []: + raw = entry.get("package") if isinstance(entry, dict) else getattr(entry, "package", None) + if not raw: + continue + m = _RPM_NEVRA_RE.match(raw) + if not m: + continue + name = m.group(1) + if "/" in name: + continue + if package_names_match(target_name, name): + return True + return False + + def _is_target_package_affected( + self, intel: CveIntel, package_identify: PackageIdentifyResult, + ) -> EnumIdentifyResult: + """Determine whether the target package is affected by this CVE. + + Priority 1: Check RHSA fix_state (distro-specific vendor assessment). + Priority 2: Fall back to NVD version range check. + Only returns NO with definitive proof; defaults to UNKNOWN otherwise. + """ + rpm_names = self._find_and_locate_rpm(intel) + if not rpm_names: + return EnumIdentifyResult.UNKNOWN + package_identify.affected_rpm_list = rpm_names + + target_name = self._target_package.name + target_version = self._target_package.version + target_release = self._target_package.release + target_distro = _extract_dist_tag(target_release) if target_release else None + + # Priority 1: Check RHSA fix_state (distro-specific vendor assessment) + if intel.rhsa and intel.rhsa.package_state: + matched_ps = _match_package_state_for_distro( + intel.rhsa.package_state, target_name, target_distro + ) + if matched_ps: + result = _interpret_fix_state(matched_ps.fix_state) + if result is not None: + logger.debug( + "RHSA fix_state=%s for %s on %s -> %s", + matched_ps.fix_state, target_name, target_distro, result.value + ) + if result == EnumIdentifyResult.NO: + # Store the raw fix_state for downstream decision-making + package_identify.rhsa_fix_state = matched_ps.fix_state.lower().strip() if matched_ps.fix_state else None + package_identify.conclusion_reason = ( + f"RHSA fix_state indicates package is not vulnerable for analysis. " + f"Package: {target_name}, Distro: {target_distro or 'unknown'}, " + f"fix_state: '{matched_ps.fix_state}'" + ) + return result + + # Priority 2: Fall back to NVD version range check + name_matched = any(package_names_match(target_name, name) for name in rpm_names) + + if name_matched: + if target_version: + in_range = self._version_in_affected_range(target_version, intel) + if not in_range: + version_range_str = self._format_nvd_version_range(intel, target_name) + package_identify.conclusion_reason = ( + f"Target version is outside NVD affected version range. " + f"Package: {target_name}-{target_version}, " + f"Affected range: {version_range_str}" + ) + return EnumIdentifyResult.NO + return EnumIdentifyResult.YES + return EnumIdentifyResult.YES + + if target_version and intel.nvd and intel.nvd.configurations: + in_range = self._version_in_affected_range(target_version, intel) + if not in_range: + version_range_str = self._format_nvd_version_range(intel, target_name) + package_identify.conclusion_reason = ( + f"Target version is outside NVD affected version range. " + f"Package: {target_name}-{target_version}, " + f"Affected range: {version_range_str}" + ) + return EnumIdentifyResult.NO + return EnumIdentifyResult.UNKNOWN + + return EnumIdentifyResult.UNKNOWN + + def _format_nvd_version_range(self, intel: CveIntel, target_name: str) -> str: + """Format NVD version range for human-readable output.""" + affected_range, _ = extract_nvd_version_info(intel, target_name) + return affected_range if affected_range else "any version (no range specified)" + + def _is_target_package_fixed(self, intel: CveIntel, package_identify: PackageIdentifyResult) -> EnumIdentifyResult: + """Determine whether the target package is already running the fixed version. + + Task 1: populate fixed_rpm_list from RHSA affected_release. + Task 2: compare target version+release against fix NVR. + """ + fix_entries = self._extract_fixed_rpms(intel) + if not fix_entries: + return EnumIdentifyResult.UNKNOWN + package_identify.fixed_rpm_list = [e["nevra"] for e in fix_entries] + + target_name = self._target_package.name + matching = [e for e in fix_entries if package_names_match(target_name, e["name"])] + if not matching: + return EnumIdentifyResult.UNKNOWN + + # NOTE: Version comparison disabled to test Option A (rely entirely on Verify phase). + # fixed_rpm_list is still populated for reference/logging. + # To re-enable, uncomment the block below. + return EnumIdentifyResult.UNKNOWN + + # --- DISABLED: Version comparison logic --- + # target_version = self._target_package.version + # target_release = self._target_package.release + # + # fix = matching[0] + # try: + # target_nvr = f"{target_version}-{target_release}" + # fix_nvr = f"{fix['version']}-{fix['release']}" + # + # target_dist = _extract_dist_tag(target_release) if target_release else None + # fix_dist = _extract_dist_tag(fix["release"]) + # if target_dist and fix_dist and target_dist != fix_dist: + # logger.debug( + # "Cross-stream fix comparison skipped: target=%s fix=%s", + # target_dist, fix_dist, + # ) + # return EnumIdentifyResult.UNKNOWN + # + # if versions.RpmVersion(target_nvr) >= versions.RpmVersion(fix_nvr): + # package_identify.conclusion_reason = ( + # f"Target package version is at or above the fix version. " + # f"Target: {target_name}-{target_nvr}, Fix: {fix_nvr}" + # ) + # return EnumIdentifyResult.YES + # return EnumIdentifyResult.NO + # except Exception as exc: + # logger.debug("Fix version comparison failed: %s", exc) + # return EnumIdentifyResult.UNKNOWN + + + def _version_in_affected_range(self, target_version: str, intel: CveIntel) -> bool: + """Check if target_version falls within any NVD configuration affected range.""" + if intel.nvd is None or not intel.nvd.configurations: + return True # no range data -> conservatively assume affected + + target_name = self._target_package.name + matched_any_config = False + for config in intel.nvd.configurations: + if not package_names_match(target_name, config.package): + continue + matched_any_config = True + version_range = [ + config.versionStartExcluding, + config.versionEndExcluding, + config.versionStartIncluding, + config.versionEndIncluding, + ] + if all(v is None for v in version_range): + continue + try: + if self._check_version_in_range(target_version, version_range): + return True + except Exception as exc: + logger.debug("Version comparison failed for %s: %s", target_version, exc) + return True # conservative: assume affected on error + + if not matched_any_config: + return True # no NVD data for this package -> conservatively assume affected + return False + + @staticmethod + def _check_version_in_range(version_to_check: str, version_range: list[str | None]) -> bool: + """Reuse the same logic as VulnerableDependencyChecker._check_version_in_range.""" + ver_start_excl, ver_end_excl, ver_start_incl, ver_end_incl = version_range + + all_versions = [v for v in version_range if v is not None] + [version_to_check] + has_el = any("el" in str(v) for v in all_versions) + has_deb = any("deb" in str(v) or "ubuntu" in str(v) for v in all_versions) + + if has_el: + vfunc = versions.RpmVersion + elif has_deb: + vfunc = versions.DebianVersion + else: + vfunc = versions.GenericVersion + + vtc = vfunc(version_to_check) + vsi = vfunc(ver_start_incl) if ver_start_incl else None + vse = vfunc(ver_start_excl) if ver_start_excl else None + vei = vfunc(ver_end_incl) if ver_end_incl else None + vee = vfunc(ver_end_excl) if ver_end_excl else None + + if vsi: + if not (vsi <= vtc): + return False + elif vse: + if not (vse < vtc): + return False + + if vei: + if not (vtc <= vei): + return False + elif vee: + if not (vtc < vee): + return False + + return True + + # ------------------------------------------------------------------ + # Intel extraction + # ------------------------------------------------------------------ + + @staticmethod + def _extract_rhsa(intel: CveIntel) -> list[dict]: + if intel.rhsa is None or not intel.rhsa.package_state: + return [] + packages = [] + for ps in intel.rhsa.package_state: + if ps.package_name: + packages.append({"package_name": ps.package_name}) + return packages + + @staticmethod + def _extract_fixed_rpms(intel: CveIntel) -> list[dict]: + """Extract all fix entries from RHSA affected_release. + + Returns a list of dicts with keys: nevra, name, version, release. + """ + if intel.rhsa is None or not hasattr(intel.rhsa, "affected_release"): + return [] + releases = intel.rhsa.affected_release + if not releases: + return [] + results: list[dict] = [] + for entry in releases: + raw = entry.get("package") if isinstance(entry, dict) else getattr(entry, "package", None) + if not raw: + continue + m = _RPM_NEVRA_RE.match(raw) + if not m: + continue + name = m.group(1) + if "/" in name: + continue + version = m.group(3) + release_arch = m.group(4) + release = _strip_arch_suffix(release_arch) + results.append({"nevra": raw, "name": name, "version": version, "release": release}) + return results + + @staticmethod + def _extract_fix_info(intel: CveIntel | None, resolved_name: str) -> dict: + """Extract fix NVR from RHSA affected_release for the resolved package. + + Returns a dict with keys nevra, name, version, release when a matching + fix entry is found; empty dict otherwise. + """ + if intel is None or intel.rhsa is None or not hasattr(intel.rhsa, "affected_release"): + return {} + releases = intel.rhsa.affected_release + if not releases: + return {} + for entry in releases: + raw = entry.get("package") if isinstance(entry, dict) else getattr(entry, "package", None) + if not raw: + continue + m = _RPM_NEVRA_RE.match(raw) + if not m: + continue + name = m.group(1) + if name.lower() != resolved_name.lower(): + continue + version = m.group(3) + release_arch = m.group(4) + release = _strip_arch_suffix(release_arch) + return {"nevra": raw, "name": name, "version": version, "release": release} + return {} + diff --git a/src/vuln_analysis/utils/package_matchers.py b/src/vuln_analysis/utils/package_matchers.py new file mode 100644 index 000000000..245c78277 --- /dev/null +++ b/src/vuln_analysis/utils/package_matchers.py @@ -0,0 +1,697 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Ecosystem-specific package matchers for verifying vulnerable package presence. + +Each ecosystem has different naming conventions and lock file formats. +This module provides a unified interface for matching vulnerable packages +from CVE intel against project dependencies. +""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any +import json +import os +import shutil +import subprocess +import tempfile + +from exploit_iq_commons.utils.dep_tree import Ecosystem +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.utils.lock_file_parsers import _pep503_normalize + +logger = LoggingFactory.get_agent_logger(__name__) + + +@dataclass +class PackageMatch: + """Result of matching a vulnerable package against dependencies.""" + found: bool + dep_name: str | None = None + installed_version: str | None = None + intel_package: str | None = None + source: str | None = None + version_info: dict[str, Any] = field(default_factory=dict) + + +class EcosystemPackageMatcher(ABC): + """Abstract base for ecosystem-specific package matching.""" + + @property + @abstractmethod + def ecosystem(self) -> Ecosystem: + """Return the ecosystem this matcher handles.""" + pass + + @abstractmethod + def parse_dependencies( + self, repo_path: Path, git_repo: str | None = None, ref: str | None = None + ) -> dict[str, str]: + """ + Parse lock file to extract {package: version} mapping. + + Args: + repo_path: Path to the repository root + git_repo: Optional git repository URL (for locking) + ref: Optional git ref (for locking) + + Returns: + dict mapping package names to versions + + Raises: + FileNotFoundError: If lock file doesn't exist + """ + pass + + @abstractmethod + def find_package( + self, + intel_package: str, + dependencies: dict[str, str] + ) -> list[PackageMatch]: + """ + Match intel package name against dependencies. + + Handles ecosystem-specific naming differences (e.g., Go module paths, + Python underscore/dash normalization, Maven groupId:artifactId). + + Args: + intel_package: Package name from CVE intel + dependencies: Dict of {package: version} from lock file + + Returns: + List of PackageMatch objects (empty if no match found) + """ + pass + + +# ============================================================ +# GO ECOSYSTEM +# ============================================================ + +class GoPackageMatcher(EcosystemPackageMatcher): + """ + Go-specific package matcher. + + Handles Go module paths like: + - golang.org/x/net + - github.com/user/repo + - k8s.io/api + + Intel may have full module path or just the package name. + """ + + @property + def ecosystem(self) -> Ecosystem: + return Ecosystem.GO + + def parse_dependencies( + self, repo_path: Path, git_repo: str | None = None, ref: str | None = None + ) -> dict[str, str]: + from vuln_analysis.utils.lock_file_parsers import parse_go_modules + return parse_go_modules(repo_path) + + def find_package( + self, + intel_package: str, + dependencies: dict[str, str] + ) -> list[PackageMatch]: + # Exact match first + if intel_package in dependencies: + return [PackageMatch( + found=True, + dep_name=intel_package, + installed_version=dependencies[intel_package], + intel_package=intel_package, + )] + + intel_lower = intel_package.lower() + + # Go modules: check if intel_package is a suffix of any dependency + # e.g., intel: "golang.org/x/net" should match dep: "golang.org/x/net" + # or intel: "net" might need to match "golang.org/x/net" + for dep_name, version in dependencies.items(): + dep_lower = dep_name.lower() + + # Check exact case-insensitive match + if dep_lower == intel_lower: + return [PackageMatch( + found=True, + dep_name=dep_name, + installed_version=version, + intel_package=intel_package, + )] + + # Check if intel package is a suffix (for short names) + # e.g., "x/net" matches "golang.org/x/net" + if dep_lower.endswith(f"/{intel_lower}"): + return [PackageMatch( + found=True, + dep_name=dep_name, + installed_version=version, + intel_package=intel_package, + )] + + return [] + + +# ============================================================ +# PYTHON ECOSYSTEM +# ============================================================ + +class PythonPackageMatcher(EcosystemPackageMatcher): + """ + Python-specific package matcher. + + Handles PyPI naming conventions per PEP 503: + - Dashes, underscores, and dots are all equivalent + (e.g., zope.interface == zope-interface == zope_interface) + - Case insensitivity + + Dependencies are parsed from an installed virtual environment using + `uv pip freeze`. Package names are normalized per PEP 503 for consistent + matching. + """ + + @property + def ecosystem(self) -> Ecosystem: + return Ecosystem.PYTHON + + def parse_dependencies( + self, repo_path: Path, git_repo: str | None = None, ref: str | None = None + ) -> dict[str, str]: + from vuln_analysis.utils.lock_file_parsers import parse_python_venv + return parse_python_venv(repo_path) + + def find_package( + self, + intel_package: str, + dependencies: dict[str, str] + ) -> list[PackageMatch]: + normalized = _pep503_normalize(intel_package) + + if normalized in dependencies: + return [PackageMatch( + found=True, + dep_name=normalized, + installed_version=dependencies[normalized], + intel_package=intel_package, + )] + + return [] + + +# ============================================================ +# JAVASCRIPT ECOSYSTEM +# ============================================================ + +class JavaScriptPackageMatcher(EcosystemPackageMatcher): + """ + JavaScript-specific package matcher. + + Handles npm naming conventions: + - Scoped packages (@angular/core, @types/node) + - Case insensitivity + - Multiple versions of the same package in the dependency tree + + Dependencies are parsed using `npm ls --json --all` which returns the + full installed dependency tree. Multiple versions of the same package + are tracked internally and returned as separate PackageMatch results. + """ + + MAX_MATCHES = 10 + + def __init__(self): + self._all_versions: dict[str, list[str]] = {} + + @property + def ecosystem(self) -> Ecosystem: + return Ecosystem.JAVASCRIPT + + def parse_dependencies( + self, repo_path: Path, git_repo: str | None = None, ref: str | None = None + ) -> dict[str, str]: + """ + Parse installed npm dependencies using npm ls --json --all. + + Args: + repo_path: Path to the repository root containing node_modules + git_repo: Optional git repository URL (unused) + ref: Optional git ref (unused) + + Returns: + dict mapping package names to versions (first-seen version) + + Raises: + FileNotFoundError: If node_modules doesn't exist + """ + node_modules = repo_path / "node_modules" + if not node_modules.exists(): + raise FileNotFoundError(f"node_modules not found at {repo_path}") + + npm_tree = self._get_npm_tree(repo_path) + if npm_tree is None: + return {} + + first_seen: dict[str, str] = {} + self._all_versions = {} + self._flatten_npm_tree(npm_tree, first_seen, self._all_versions) + + logger.debug("Parsed %d packages from npm ls", len(first_seen)) + return first_seen + + def _get_npm_tree(self, repo_path: Path) -> dict | None: + """ + Run npm ls --json --all and return parsed JSON. + + Returns None if npm ls fails or returns invalid JSON. + """ + try: + result = subprocess.run( + ["npm", "ls", "--json", "--all", "--ignore-scripts"], + capture_output=True, + text=True, + cwd=repo_path, + timeout=60, + ) + except subprocess.TimeoutExpired: + logger.error("npm ls timed out after 60s for %s", repo_path) + return None + + if not result.stdout: + logger.warning("npm ls returned empty output for %s", repo_path) + return None + + try: + return json.loads(result.stdout) + except json.JSONDecodeError as e: + logger.warning("Failed to parse npm ls output: %s", e) + return None + + def _flatten_npm_tree( + self, + node: dict, + first_seen: dict[str, str], + all_versions: dict[str, list[str]] + ): + """ + Recursively flatten npm ls JSON tree to extract package versions. + + Populates both first_seen (for interface compatibility) and + all_versions (for multi-version tracking). + + Args: + node: Current node in the npm tree + first_seen: Dict to store first-seen version of each package + all_versions: Dict to store all versions of each package + """ + dependencies = node.get("dependencies", {}) + + for dep_name, dep_info in dependencies.items(): + if not isinstance(dep_info, dict): + continue + + version = dep_info.get("version", "unknown") + + if dep_name not in first_seen: + first_seen[dep_name] = version + + if dep_name not in all_versions: + all_versions[dep_name] = [] + + if version not in all_versions[dep_name]: + all_versions[dep_name].append(version) + + self._flatten_npm_tree(dep_info, first_seen, all_versions) + + def find_package( + self, + intel_package: str, + dependencies: dict[str, str] + ) -> list[PackageMatch]: + """ + Match intel package name against npm dependencies. + + Uses cascading match strategy: + 1. Exact match (case-insensitive) + 2. Scoped suffix match: "core" matches "@angular/core" + + Returns PackageMatch for all versions of matching packages + tracked in _all_versions. + """ + matches: list[PackageMatch] = [] + intel_lower = intel_package.lower() + matched_packages: set[str] = set() + + for dep_name in dependencies.keys(): + dep_lower = dep_name.lower() + + # Strategy 1: Exact match (case-insensitive) + if dep_lower == intel_lower: + matched_packages.add(dep_name) + continue + + # Strategy 2: Scoped suffix match + # e.g., intel "core" matches "@angular/core" + if dep_lower.endswith(f"/{intel_lower}"): + matched_packages.add(dep_name) + continue + + for dep_name in matched_packages: + versions = self._all_versions.get(dep_name, [dependencies.get(dep_name, "unknown")]) + for version in versions: + matches.append(PackageMatch( + found=True, + dep_name=dep_name, + installed_version=version, + intel_package=intel_package, + )) + if len(matches) >= self.MAX_MATCHES: + break + if len(matches) >= self.MAX_MATCHES: + break + + if len(matches) > 1: + logger.info( + "Intel '%s' matched %d package versions: %s", + intel_package, + len(matches), + ", ".join(f"{m.dep_name}@{m.installed_version}" for m in matches) + ) + + return matches + + +# ============================================================ +# JAVA ECOSYSTEM +# ============================================================ + + +class JavaPackageMatcher(EcosystemPackageMatcher): + """ + Java-specific package matcher. + + Handles Maven coordinates in two formats: + - Full groupId:artifactId from GHSA (e.g., org.apache.logging.log4j:log4j-core) + - Artifact-only names from NVD (e.g., log4j, artemis) + + Uses cascading match strategy: + 1. Exact groupId:artifactId match (if intel contains ":") + 2. Artifact prefix match (for NVD artifact-only names) + + Returns up to MAX_MATCHES results with warning if multiple found. + + Concurrency: Uses lock+copy pattern for dependency_tree.txt to prevent + race conditions where concurrent threads might read a partially-written file. + Reuses DocumentEmbedding._get_repo_lock() for consistent per-repo locking. + """ + + MAX_MATCHES = 5 + + @property + def ecosystem(self) -> Ecosystem: + return Ecosystem.JAVA + + def parse_dependencies( + self, repo_path: Path, git_repo: str | None = None, ref: str | None = None + ) -> dict[str, str]: + """ + Parse Java dependencies using lock+copy pattern for thread safety. + + The dependency_tree.txt file is generated by maven and read by multiple + threads. To prevent race conditions: + 1. Acquire per-repo lock (via DocumentEmbedding._get_repo_lock) + 2. Generate file if missing + 3. Copy to temp file while holding lock + 4. Release lock + 5. Parse from temp file (thread-safe) + 6. Clean up temp file + + Args: + repo_path: Path to the repository root + git_repo: Git repository URL (required for locking) + ref: Git ref (required for locking) + """ + from vuln_analysis.utils.lock_file_parsers import ( + parse_dependency_tree_txt, + _generate_dependency_tree_txt, + ) + + # If git_repo/ref not provided, fall back to unlocked behavior + if not git_repo or not ref: + logger.warning("git_repo/ref not provided to JavaPackageMatcher; skipping lock") + return self._parse_unlocked(repo_path) + + from exploit_iq_commons.utils.document_embedding import DocumentEmbedding + + repo_lock = DocumentEmbedding._get_repo_lock(git_repo, ref) + temp_dir: str | None = None + + with repo_lock: + tree_file = repo_path / "dependency_tree.txt" + + if not tree_file.exists(): + logger.info("dependency_tree.txt not found, generating via depgraph-maven-plugin") + if not _generate_dependency_tree_txt(repo_path): + logger.warning("Could not resolve Java dependencies for %s", repo_path) + return {} + + if not tree_file.exists(): + logger.warning("dependency_tree.txt still not found after generation for %s", repo_path) + return {} + + # Copy to temp directory while holding the lock to get a consistent snapshot + try: + temp_dir = tempfile.mkdtemp(prefix="java_deps_") + shutil.copy(tree_file, Path(temp_dir) / "dependency_tree.txt") + logger.debug("Copied dependency_tree.txt to temp dir: %s", temp_dir) + except OSError as e: + logger.warning("Failed to copy dependency_tree.txt to temp: %s", e) + if temp_dir and os.path.exists(temp_dir): + shutil.rmtree(temp_dir, ignore_errors=True) + return {} + + # Parse from temp directory outside the lock (thread-safe, each thread has its own copy) + try: + return parse_dependency_tree_txt(Path(temp_dir)) + except FileNotFoundError: + logger.warning("Temp dependency_tree.txt disappeared: %s", temp_dir) + return {} + finally: + if temp_dir and os.path.exists(temp_dir): + shutil.rmtree(temp_dir, ignore_errors=True) + + def _parse_unlocked(self, repo_path: Path) -> dict[str, str]: + """Fallback parsing without locking (original behavior).""" + from vuln_analysis.utils.lock_file_parsers import ( + parse_dependency_tree_txt, + _generate_dependency_tree_txt, + ) + + try: + return parse_dependency_tree_txt(repo_path) + except FileNotFoundError: + pass + + logger.info("dependency_tree.txt not found, generating via depgraph-maven-plugin") + if _generate_dependency_tree_txt(repo_path): + return parse_dependency_tree_txt(repo_path) + + logger.warning("Could not resolve Java dependencies for %s", repo_path) + return {} + + def find_package( + self, + intel_package: str, + dependencies: dict[str, str] + ) -> list[PackageMatch]: + matches: list[PackageMatch] = [] + intel_lower = intel_package.lower() + + # Strategy 1: Exact groupId:artifactId match (for GHSA data) + if ":" in intel_package: + # Exact case-sensitive match + if intel_package in dependencies: + return [PackageMatch( + found=True, + dep_name=intel_package, + installed_version=dependencies[intel_package], + intel_package=intel_package, + )] + + # Case-insensitive fallback + for ga, version in dependencies.items(): + if ga.lower() == intel_lower: + return [PackageMatch( + found=True, + dep_name=ga, + installed_version=version, + intel_package=intel_package, + )] + + # No GA match found + return [] + + # Strategy 2: Artifact prefix match (for NVD artifact-only data) + for ga, version in dependencies.items(): + # Extract artifactId from groupId:artifactId + artifact = ga.split(":")[-1] + if artifact.lower().startswith(intel_lower): + matches.append(PackageMatch( + found=True, + dep_name=ga, + installed_version=version, + intel_package=intel_package, + )) + if len(matches) >= self.MAX_MATCHES: + break + + if len(matches) > 1: + logger.warning( + "Intel '%s' matched %d artifacts: %s", + intel_package, + len(matches), + ", ".join(m.dep_name or "" for m in matches) + ) + + return matches + + +# ============================================================ +# C/C++ ECOSYSTEM +# ============================================================ + +class CppPackageMatcher(EcosystemPackageMatcher): + """ + C/C++ package matcher for container SBOM-based analysis. + + Handles RPM package naming conventions: + - Exact match (case-insensitive): "openssl" matches "openssl" + - Prefix match: "openssl" matches "openssl-libs", "openssl-devel" + - Library suffix variations: handles -libs, -devel, -common suffixes + + For container analysis, dependencies come from SBOM (not lock files). + """ + + MAX_MATCHES = 10 + + @property + def ecosystem(self) -> Ecosystem: + return Ecosystem.C_CPP + + def parse_dependencies( + self, repo_path: Path, git_repo: str | None = None, ref: str | None = None + ) -> dict[str, str]: + """ + For C/C++ container analysis, dependencies come from SBOM not repo. + + This method is called when repo_path is provided (SOURCE analysis), + but for IMAGE analysis, use get_sbom_dependencies() instead. + + Args: + repo_path: Path to the repository root (unused for C/C++) + git_repo: Optional git repository URL (unused) + ref: Optional git ref (unused) + """ + return self.get_sbom_dependencies() + + def get_sbom_dependencies(self) -> dict[str, str]: + """Get dependencies from the SBOM via RPMDependencyManager.""" + from exploit_iq_commons.utils.source_rpm_downloader import RPMDependencyManager + sbom = RPMDependencyManager.get_instance().sbom + return {pkg.name: pkg.version for pkg in sbom} + + def find_package( + self, + intel_package: str, + dependencies: dict[str, str] + ) -> list[PackageMatch]: + """ + Match intel package name against SBOM dependencies. + + Uses cascading match strategy: + 1. Exact match (case-insensitive) + 2. Prefix match: intel "openssl" matches "openssl-libs" + 3. Base name match: handles packages where intel has full name + + Returns multiple matches when appropriate (e.g., openssl matches + openssl, openssl-libs, openssl-devel). + """ + matches: list[PackageMatch] = [] + intel_lower = intel_package.lower() + + for dep_name, version in dependencies.items(): + dep_lower = dep_name.lower() + + # Match strategies: + # 1. Exact match (case-insensitive) + # 2. Prefix match - intel "openssl" matches "openssl-libs" + # 3. Reverse prefix - intel "libxml2-devel" matches dep "libxml2" + if (dep_lower == intel_lower or + dep_lower.startswith(intel_lower + "-") or + intel_lower.startswith(dep_lower + "-")): + matches.append(PackageMatch( + found=True, + dep_name=dep_name, + installed_version=version, + intel_package=intel_package, + )) + if len(matches) >= self.MAX_MATCHES: + break + + if len(matches) > 1: + logger.info( + "Intel '%s' matched %d packages: %s", + intel_package, + len(matches), + ", ".join(m.dep_name or "" for m in matches) + ) + + return matches + + +# ============================================================ +# DISPATCHER +# ============================================================ + +PACKAGE_MATCHERS: dict[Ecosystem, type[EcosystemPackageMatcher]] = { + Ecosystem.GO: GoPackageMatcher, + Ecosystem.PYTHON: PythonPackageMatcher, + Ecosystem.JAVASCRIPT: JavaScriptPackageMatcher, + Ecosystem.JAVA: JavaPackageMatcher, + Ecosystem.C_CPP: CppPackageMatcher, +} + + +def get_package_matcher(ecosystem: Ecosystem) -> EcosystemPackageMatcher: + """ + Get the appropriate package matcher for an ecosystem. + + Args: + ecosystem: The ecosystem to get a matcher for + + Returns: + An instance of the appropriate EcosystemPackageMatcher + + Raises: + NotImplementedError: If no matcher is available for the ecosystem + """ + matcher_cls = PACKAGE_MATCHERS.get(ecosystem) + if matcher_cls is None: + raise NotImplementedError(f"No package matcher available for ecosystem: {ecosystem}") + return matcher_cls() diff --git a/src/vuln_analysis/utils/prompt_factory.py b/src/vuln_analysis/utils/prompt_factory.py new file mode 100644 index 000000000..45b07ef99 --- /dev/null +++ b/src/vuln_analysis/utils/prompt_factory.py @@ -0,0 +1,349 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from vuln_analysis.utils.prompting import AGENT_SYS_PROMPT +from vuln_analysis.tools.tool_names import ToolNames + +# Structure: (tool_name, language) -> language-specific input format and call instructions +TOOL_ECOSYSTEM_REGISTRY: dict[tuple[str, str], str] = { + # Code Semantic Search + (ToolNames.CODE_SEMANTIC_SEARCH, "python"): ( + "Input: natural language query. Use Python identifiers: e.g. 'urllib.parse url handling', " + "'PIL.Image image processing', 'asyncio usage'." + ), + (ToolNames.CODE_SEMANTIC_SEARCH, "go"): ( + "Input: natural language query. Use Go identifiers: e.g. 'encoding/json unmarshaling', " + "'net/http handler', 'context.Context propagation'." + ), + (ToolNames.CODE_SEMANTIC_SEARCH, "java"): ( + "Input: natural language query. Use Java identifiers: e.g. 'ObjectInputStream deserialization', " + "'javax.servlet request handling', 'java.util collections'." + ), + (ToolNames.CODE_SEMANTIC_SEARCH, "javascript"): ( + "Input: natural language query. Use JS identifiers: e.g. 'require() imports', " + "'express middleware', 'JSON.parse usage'." + ), + (ToolNames.CODE_SEMANTIC_SEARCH, "c"): ( + "Input: natural language query. Use C/C++ identifiers: e.g. 'EVP_EncryptInit encryption', " + "'strcpy buffer handling', 'malloc memory allocation'." + ), + # Docs Semantic Search + (ToolNames.DOCS_SEMANTIC_SEARCH, "python"): ( + "Input: natural language question. Phrase for Python docs: e.g. 'How does this app handle image uploads?'" + ), + (ToolNames.DOCS_SEMANTIC_SEARCH, "go"): ( + "Input: natural language question. Phrase for Go docs: e.g. 'How is HTTP routing configured?'" + ), + (ToolNames.DOCS_SEMANTIC_SEARCH, "java"): ( + "Input: natural language question. Phrase for Java docs: e.g. 'Application architecture and dependencies'" + ), + (ToolNames.DOCS_SEMANTIC_SEARCH, "javascript"): ( + "Input: natural language question. Phrase for JS docs: e.g. 'How does the API handle requests?'" + ), + (ToolNames.DOCS_SEMANTIC_SEARCH, "c"): ( + "Input: natural language question. Phrase for C/C++ docs: e.g. 'Cryptographic library usage'" + ), + # Code Keyword Search + (ToolNames.CODE_KEYWORD_SEARCH, "python"): ( + "Input: exact text. Python patterns: 'urllib.parse', 'from X import Y', 'import asyncio'." + ), + (ToolNames.CODE_KEYWORD_SEARCH, "go"): ( + "Input: exact text. Go patterns: 'import \"', 'pkg.FunctionName', 'encoding/json'." + ), + (ToolNames.CODE_KEYWORD_SEARCH, "java"): ( + "Input: exact text. Java patterns: 'import com.', 'ClassName.method', 'javax.servlet'." + ), + (ToolNames.CODE_KEYWORD_SEARCH, "javascript"): ( + "Input: exact text. JS patterns: 'require(', 'import { ', 'process.env'." + ), + (ToolNames.CODE_KEYWORD_SEARCH, "c"): ( + "Input: exact text. C/C++ patterns: 'EVP_EncryptInit_ex2', 'OSSL_PARAM', function names." + ), + # Function Locator + (ToolNames.FUNCTION_LOCATOR, "python"): ( + "Input: 'package_name,function_name'. Example: 'urllib,parse' or 'PIL.Image,open'. " + "Call first before Call Chain Analyzer." + ), + (ToolNames.FUNCTION_LOCATOR, "go"): ( + "Input: 'package_path,FunctionName'. Example: 'github.com/pkg/errors,New'. " + "Call first before Function Caller Finder or Call Chain Analyzer." + ), + (ToolNames.FUNCTION_LOCATOR, "java"): ( + "Input format 2(java): 'maven_gav,class_name.method_name'." + "Example 2(java): 'commons-beanutils:commons-beanutils:1.0.0,PropertyUtilsBean.setSimpleProperty'." + ), + (ToolNames.FUNCTION_LOCATOR, "javascript"): ( + "Input: 'package_name,function_name'. Example: 'express,Router' or npm scope paths. " + "Call first before Call Chain Analyzer." + ), + (ToolNames.FUNCTION_LOCATOR, "c"): ( + "Input: 'library_name,function_name'. Example: 'libxml2,xmlParseDocument' or 'openssl,EVP_EncryptInit_ex2'. " + "Call first before Call Chain Analyzer." + ), + # Call Chain Analyzer + (ToolNames.CALL_CHAIN_ANALYZER, "python"): ( + "Input: use validated 'package,function' from Function Locator. Example: 'urllib,parse'." + ), + (ToolNames.CALL_CHAIN_ANALYZER, "go"): ( + "Input format 1: 'package_name,function_name' Example: 'github.com/pkg/errors,New'." + ), + (ToolNames.CALL_CHAIN_ANALYZER, "java"): ( + "Input: use validated maven_gav,class.method from Function Locator." + ), + (ToolNames.CALL_CHAIN_ANALYZER, "javascript"): ( + "Input: use validated package,function from Function Locator." + ), + (ToolNames.CALL_CHAIN_ANALYZER, "c"): ( + "Input format 1: 'package_name,function_name'. Example: 'openssl,EVP_EncryptInit_ex2'." + ), + # Function Caller Finder (Go only) + (ToolNames.FUNCTION_CALLER_FINDER, "go"): ( + "Input: 'package_name,lib.function(args)'. Example: 'github.com/myapp,errors.New(\"text\")'. " + "Go-only. Finds callers of std/lib functions." + ), + # CVE Web Search (language-agnostic) + (ToolNames.CVE_WEB_SEARCH, "python"): " vulnerability search terms.", + (ToolNames.CVE_WEB_SEARCH, "go"): " vulnerability search terms.", + (ToolNames.CVE_WEB_SEARCH, "java"): "vulnerability search terms.", + (ToolNames.CVE_WEB_SEARCH, "javascript"): "vulnerability search terms.", + (ToolNames.CVE_WEB_SEARCH, "c"): "vulnerability search terms.", + # Container Analysis Data (language-agnostic) + (ToolNames.CONTAINER_ANALYSIS_DATA, "python"): "Input: query string for pre-analyzed container findings.", + (ToolNames.CONTAINER_ANALYSIS_DATA, "go"): "Input: query string for pre-analyzed container findings.", + (ToolNames.CONTAINER_ANALYSIS_DATA, "java"): "Input: query string for pre-analyzed container findings.", + (ToolNames.CONTAINER_ANALYSIS_DATA, "javascript"): "Input: query string for pre-analyzed container findings.", + (ToolNames.CONTAINER_ANALYSIS_DATA, "c"): "Input: query string for pre-analyzed container findings.", +} + +# Optional: Add language-specific usage hints +FL_INPUT_FORMATS: dict[str, str] = { + "python": ( + "Input: 'package_name,function_name' or 'package_name,module.ClassName'.\n" + " Example: 'werkzeug,formparser.MultiPartParser'." + ), + "go": ( + "Input: 'package_path,FunctionName'.\n" + " Example: 'github.com/pkg/errors,New'." + ), + "java": ( + "Input: 'maven_gav,class_name.method_name' or 'maven_gav,fqcn.method_name' (preferred).\n" + " Example: 'commons-beanutils:commons-beanutils:1.9.4,PropertyUtilsBean.getProperty'." + ), + "javascript": ( + "Input: 'package_name,function_name'.\n" + " Example: 'express,Router'." + ), + "c": ( + "Input: 'library_name,function_name'.\n" + " Example: 'openssl,EVP_EncryptInit_ex2'." + ), +} + +FL_EXAMPLES: dict[str, str] = { + "python": "'werkzeug,formparser.MultiPartParser'", + "go": "'github.com/pkg/errors,New'", + "java": "'commons-beanutils:commons-beanutils:1.9.4,PropertyUtilsBean.getProperty'", + "javascript": "'express,Router'", + "c": "'openssl,EVP_EncryptInit_ex2'", +} + +FEW_SHOT_EXAMPLES: dict[str, str] = { + "python": "Example: Function Locator input 'urllib,parse'; Code Keyword Search for 'urllib.parse'.", + "go": "Example: Function Caller Finder 'pkg,errors.New(\"x\")'; Function Locator 'github.com/pkg,New'.", + "java": "Example: Function Locator 'group:artifact:ver,Class.method'; Code Keyword Search 'import com.'.", + "javascript": "Example: Code Keyword Search 'require('; Function Locator 'package_name,exported'.", + "c": "Example: Function Locator 'openssl,EVP_EncryptInit_ex2'; Code Keyword Search 'EVP_aes_'.", +} + +# Per-language strategy for when to use which tools +TOOL_SELECTION_STRATEGY: dict[str, str] = { + "python": ( + "Use Code Keyword Search first for exact import/function lookups (e.g. urllib.parse, PIL.Image). " + "For reachability: call Function Locator first with package,function (e.g. urllib,parse), then Call Chain Analyzer with validated names. " + "Use Docs Semantic Search for architecture questions; Code Semantic Search for 'how is X used' patterns." + ), + "go": ( + "For std/lib function reachability: call Function Locator first to validate package paths (e.g. github.com/pkg/errors,New), " + "then use Function Caller Finder to find which app functions call the library method, then Call Chain Analyzer with validated names. " + "Use Code Keyword Search for import paths; Docs/Code Semantic Search for handler and middleware patterns." + ), + "java": ( + "Use Function Locator first with maven GAV format (group:artifact:version,ClassName.methodName). " + "For reachability, use Call Chain Analyzer with validated names from Function Locator. " + "Use Code Keyword Search for import com. and javax. patterns; Docs Semantic Search for Spring/servlet architecture." + ), + "javascript": ( + "Use Code Keyword Search first for require(, import {, and package patterns. " + "For reachability: Function Locator with package_name,exported, then Call Chain Analyzer. " + "Use Docs/Code Semantic Search for middleware, API, and npm package usage patterns." + ), + "c": ( + "Use Code Keyword Search first for exact function name lookups (e.g. PQescapeLiteral, EVP_EncryptInit_ex2). " + "For reachability: call Function Locator first with library_name,function_name (e.g. libpq,PQescapeLiteral) to validate the package and function, " + "then Call Chain Analyzer with validated names to check if it is reachable from application code." + ), +} + +TOOL_SELECTION_STRATEGY_NON_REACHABILITY: dict[str, str] = { + "python": ( + "Use Code Keyword Search first for exact import/function lookups " + "(e.g. urllib.parse, PIL.Image). " + "Pay attention to whether results appear in the main application code " + "or in package dependencies (transitive_env/). " + "Use Function Locator to validate package and function names " + "(e.g. urllib,parse). " + "Use Code Semantic Search to understand how a component is used; " + "Docs Semantic Search for architecture questions. " + "Use CVE Web Search for additional vulnerability context." + ), + "go": ( + "Use Code Keyword Search for import paths and function names. " + "Pay attention to whether results appear in the main application code " + "or in vendor/ dependencies. " + "Use Function Locator to validate package paths and function names " + "(e.g. github.com/pkg/errors,New). " + "Use Code Semantic Search to understand usage patterns; " + "Docs Semantic Search for architecture. " + "Use CVE Web Search for additional vulnerability context." + ), + "java": ( + "Use Code Keyword Search for import statements and class/method patterns " + "(e.g. import com., javax.servlet). " + "Pay attention to whether results appear in the main application code " + "or in library dependencies. " + "Use Function Locator with maven GAV format to validate names " + "(e.g. group:artifact:version,ClassName.methodName). " + "Use Docs Semantic Search for Spring/servlet architecture; " + "Code Semantic Search for usage patterns. " + "Use CVE Web Search for additional vulnerability context." + ), + "javascript": ( + "Use Code Keyword Search first for require(, import {, and package patterns. " + "Pay attention to whether results appear in the main application code " + "or in node_modules/ dependencies. " + "Use Function Locator to validate package and function names " + "(e.g. lodash,defaultsDeep). " + "Use Code/Docs Semantic Search for middleware, API, and npm package " + "usage patterns. " + "Use CVE Web Search for additional vulnerability context." + ), + "c": ( + "Use Code Keyword Search first for exact function name lookups " + "(e.g. PQescapeLiteral, EVP_EncryptInit_ex2). " + "Pay attention to whether results appear in the main application code " + "or in rpm_libs/ dependencies. " + "Use Function Locator to validate library and function names " + "(e.g. libpq,PQescapeLiteral). " + "Use Code Semantic Search to understand usage patterns; " + "Docs Semantic Search for library documentation. " + "Use CVE Web Search for additional vulnerability context." + ), +} + +TOOL_GENERAL_DESCRIPTIONS: dict[str, str] = { + ToolNames.CODE_SEMANTIC_SEARCH: "Searches container source code using semantic search. " + "Finds how functions, libraries, or components are used in the codebase. " + "Answers questions about code implementation and dependencies.", + ToolNames.DOCS_SEMANTIC_SEARCH: "Searches container documentation using semantic search. " + "Answers questions about application purpose, architecture, and features.", + ToolNames.CODE_KEYWORD_SEARCH: "Performs keyword search on container source code for exact text matches. " + "Input should be a function name, class name, or code pattern. " + "Use this first before semantic search tools for precise lookups.", + ToolNames.FUNCTION_LOCATOR: "Mandatory first step for code path analysis. Validates package names, locates functions using fuzzy matching, and provides ecosystem type" + "Make sure the input format is matching exactly one of the following formats:", + + ToolNames.CALL_CHAIN_ANALYZER: "Checks if a function from a package is reachable from application code through the call chain." + "Make sure the input format is matching exactly one of the following formats:", + ToolNames.FUNCTION_CALLER_FINDER: "Finds functions in a package that call a specific library function.", + ToolNames.CVE_WEB_SEARCH: "Searches the web for information about CVEs, vulnerabilities, libraries, " + "and security advisories not available in the container.", + ToolNames.CONTAINER_ANALYSIS_DATA: "Retrieves pre-analyzed container image source code analysis results for the current CVE. " + "Returns a list of findings with 'response' (summary) and 'intermediate_steps' (reasoning) fields. " + "No input required - uses context automatically.", +} + + +class PromptFactory: + def __init__( + self, + template: str, + registry: dict, + examples: dict, + general_descriptions: dict[str, str], + tool_selection_strategy: dict[str, str], + ): + self.template = template + self.registry = registry + self.examples = examples + self.general_descriptions = general_descriptions + self.tool_selection_strategy = tool_selection_strategy + + def build_prompt( + self, + sys_prompt: str, + enabled_tools: list[str], + language: str, + tool_instructions: str | None = None, + ) -> str: + # Build AVAILABLE_TOOLS in format: tool_name(*args, **kwargs) - description + available_tools_lines = [] + for tool_name in enabled_tools: + desc = self.general_descriptions.get(tool_name, "Standard analysis tool.") + available_tools_lines.append(f"{tool_name}(*args, **kwargs) - {desc}") + tools_section = "\n".join(available_tools_lines) + + if tool_instructions is None: + # Fallback: build from language-specific registry + tool_instructions = "" + + #lang_examples = self.examples.get(language, "") + #disable examples for now + lang_examples = "" + tool_selection_strategy = self.tool_selection_strategy.get( + language, f"Optimize for {language} ecosystem patterns." + ) + + return self.template.format( + sys_prompt=sys_prompt, + tools=tools_section, + tool_selection_strategy=tool_selection_strategy, + tool_instructions=f"{tool_instructions}\n\n{lang_examples}" + ) + +# --- Usage --- +LANGGRAPH_SYSTEM_PROMPT_TEMPLATE = """{sys_prompt} + +Answer the investigation question using the available tools. If the input is not a question, formulate it into a question first. A Tool Selection Strategy is provided to help you decide which tools to use. Focus on answering the question. Include your intermediate reasoning in the final answer. + + +{tools} + + + +{tool_selection_strategy} + + +{tool_instructions} + +RESPONSE: +{{""" + +factory = PromptFactory( + LANGGRAPH_SYSTEM_PROMPT_TEMPLATE, + TOOL_ECOSYSTEM_REGISTRY, + FEW_SHOT_EXAMPLES, + TOOL_GENERAL_DESCRIPTIONS, + TOOL_SELECTION_STRATEGY, +) diff --git a/src/vuln_analysis/utils/prompting.py b/src/vuln_analysis/utils/prompting.py index b746bd3c7..0f6fdfd10 100644 --- a/src/vuln_analysis/utils/prompting.py +++ b/src/vuln_analysis/utils/prompting.py @@ -20,107 +20,418 @@ # pylint: disable=line-too-long -SUMMARY_PROMPT = """Summarize the exploitability investigation results of a Common Vulnerabilities and Exposures (CVE) \ -based on the provided Checklist and Findings. Write a concise paragraph focusing only on checklist items with \ -definitive answers. Begin your response by clearly stating whether the CVE is exploitable. Disregard any ambiguous \ -checklist items. -Checklist and Findings: -{response}""" - -# Define a system prompt that sets the context for the language model's task. This prompt positions the assistant -# as a powerful entity capable of investigating CVE impacts on container images. -AGENT_SYS_PROMPT = ( - "You are a very powerful assistant who helps investigate the impact of reported Common Vulnerabilities and " - "Exposures (CVE) on container images. Information about the container image under investigation is " - "stored in vector databases available to you via tools.") -# Based on original prompt: https://github.com/langchain-ai/langchain/blob/master/libs/langchain/langchain/agents/mrkl/prompt.py -AGENT_PROMPT_TEMPLATE = """If the input is not a question, formulate it into a question first. Include intermediate thought in the final answer. You have access to the following tools: +def build_tool_descriptions(tool_names: list[str]) -> list[str]: + """ + Build tool descriptions based on enabled tools. + + Returns simple, descriptive tool descriptions that can be formatted + with context-specific text by calling functions. + + Args: + tool_names: List of enabled tool names (after filtering) + + Returns: + List of tool description strings (without bullet points or formatting) + """ + from vuln_analysis.tools.tool_names import ToolNames + + descriptions = [] + + if ToolNames.CODE_SEMANTIC_SEARCH in tool_names: + descriptions.append( + f"{ToolNames.CODE_SEMANTIC_SEARCH}: Searches source code using semantic understanding" + ) + + if ToolNames.DOCS_SEMANTIC_SEARCH in tool_names: + descriptions.append( + f"{ToolNames.DOCS_SEMANTIC_SEARCH}: Searches package documentation for architecture/design info" + ) + + if ToolNames.CODE_KEYWORD_SEARCH in tool_names: + descriptions.append( + f"{ToolNames.CODE_KEYWORD_SEARCH}: Exact text matching for function names, class names, or imports" + ) + + if ToolNames.FUNCTION_CALLER_FINDER in tool_names and ToolNames.CALL_CHAIN_ANALYZER in tool_names: + descriptions.append( + f"{ToolNames.CALL_CHAIN_ANALYZER}: Checks if functions are reachable from application code\n" + f"{ToolNames.FUNCTION_CALLER_FINDER}: Finds which functions call specific library functions\n" + f"Use '{ToolNames.FUNCTION_CALLER_FINDER}' + '{ToolNames.CALL_CHAIN_ANALYZER}' together to trace function reachability" + ) + elif ToolNames.CALL_CHAIN_ANALYZER in tool_names: + descriptions.append( + f"{ToolNames.CALL_CHAIN_ANALYZER}: Checks if functions are reachable from application code" + ) + elif ToolNames.FUNCTION_CALLER_FINDER in tool_names: + descriptions.append( + f"{ToolNames.FUNCTION_CALLER_FINDER}: Finds which functions call specific library functions" + ) + + if ToolNames.CVE_WEB_SEARCH in tool_names: + descriptions.append( + f"{ToolNames.CVE_WEB_SEARCH}: External vulnerability information lookup" + ) + + if ToolNames.CONTAINER_ANALYSIS_DATA in tool_names: + descriptions.append( + f"{ToolNames.CONTAINER_ANALYSIS_DATA}: Retrieves findings from earlier container scan analysis" + ) + + if ToolNames.FUNCTION_LIBRARY_VERSION_FINDER in tool_names: + descriptions.append( + f"{ToolNames.FUNCTION_LIBRARY_VERSION_FINDER}: Finds installed version of a library/package. " + f"Input: package name (e.g., 'commons-beanutils'). Returns matching packages with versions from the dependency tree" + ) + + return descriptions + + + + +SUMMARY_PROMPT = """ +Summarize CVE exploitability investigation results into a clear, evidence-based +paragraph. The investigation results consist of checklist items (questions) and +their corresponding conclusions from the security analysis. + + + +Write a 3-5 sentence paragraph following this structure: + +1. VERDICT (sentence 1): Begin with explicit statement + - "The CVE is exploitable" / "The CVE is not exploitable" / "Exploitability is uncertain" + +2. EVIDENCE (sentences 2-4): Support with specific findings + - Cite concrete results: functions found/absent, reachability status, configuration states + - Use technical details: function names, file paths, components + - Connect findings to exploitability conditions + +3. FOCUS: Use only definitive checklist results; ignore inconclusive items + + + +The CVE is not exploitable in this container. Investigation confirmed that while +Python 3.10.0 is installed (vulnerable version), the urllib.parse module is never +imported or called in the application codebase (verified via code search). Additionally, +code analysis revealed that the application does not accept URL inputs from untrusted +sources; all URL handling occurs only with internally generated URLs from configuration +files. The combination of no urllib.parse usage and lack of external URL input eliminates +the attack vector described in the CVE. + + + +{response} + + +Write your summary paragraph:""" +AGENT_SYS_PROMPT = ( + "You are an expert security analyst investigating Common Vulnerabilities and " + "Exposures (CVE) in container images. Your role is to methodically answer " + "investigation questions using available tools to determine if vulnerabilities " + "are exploitable in the specific container context. You have access to the " + "container's source code, documentation, and dependency information through " + "specialized search and analysis tools." +) + +AGENT_PROMPT_TEMPLATE = """ +Answer the investigation question using the available tools. If the input is not a question, formulate it into a question first. A Tool Selection Strategy is provided to help you decide which tools to use. Focus on answering the question. Include your intermediate reasoning in the final answer. + + + {tools} + + + +{tool_selection_strategy} + -Use the following format (start each response with one of the following prefixes: [Question, Thought, Action, Action Input, Final Answer]): + +Follow this format exactly (start each line with one of the specified prefixes): Question: the input question you must answer -Thought: you should always think about what to do -Action: the action to take, should be one of [{tool_names}] -Action Input: the input to the action -Observation: the result of the action -... (this Thought/Action/Action Input/Observation can repeat N times) +Thought: think about what action to take next +Action: the tool to use, must be one of [{tool_names}] +Action Input: the specific input for that tool +Observation: the result returned by the tool +... (repeat Thought/Action/Action Input/Observation cycle as many times as needed) Thought: I now know the final answer -Final Answer: the final answer to the original input question +Final Answer: provide the answer with supporting evidence from your investigation + -Begin! + + -Question: {input} + +{input} + + +Begin your investigation: Thought:{agent_scratchpad}""" -AGENT_EXAMPLES_FOR_PROMPT = """Example 1: - - Question: 1. Identify the version of Python: Check which version(s) of Python are installed in the container image. The vulnerability affects versions up to and including 3.11.3. - Thought: I should check if Python is installed. I will check the Software Bill of Materials. - Action: SBOM Package Checker - Action Input: Python - Observation: 3.10.0 - Thought: Python 3.10.0 is installed, I need to check if the installed version is vulnerable. To do this I'll compare the installed version to the vulnerable version. - Action: container software version comparator - Action Input: 3.10.0, 3.11.3 - Observation: True - Thought: The installed software is vulnerable. I now know the answer. - Final Answer: Python version 3.10.0 is installed and is vulnerable to the CVE. - - Example 2: - - Question: Assess the threat that CVE-20xx-xxxxx poses to the container. - Thought: I should search for more information on CVE-20xx-xxxxx. - Action: Internet Search - Action Input: What is CVE-20xx-xxxxx? - Observation: CVE-20xx-xxxxx causes memory leaks and possible denial of service attack vectors when using urllib.parse - Thought: I should check the code base of the container for instances of urllib.parse - Action: Container Image QA System - Action Input: Is urllib.parse present in the code? - Observation: - Question: Is urllib.parse present in the code? - Helpful answer: No, that function is not called in the code. - Thought: Since the function is not called in the code, the container is not vulnerable. I know the final answer. - Final Answer: The function urllib.parse is not present in the code, so the container is not vulnerable. - - Example 3: - - Question: Check if the container is using Java Runtime Environment (JRE). If it is not using JRE, then it is not vulnerable to CVE-xxxx-xxxxx. - Thought: I should check if JRE is installed. I will check the Software Bill of Materials. - Action: SBOM Package Checker - Action Input: JRE - Observation: False - Thought: JRE is not present in the container. I now know the answer. - Final Answer: JRE is not installed in the container. Therefore, it is not vulnerable to CVE-20xx-xxxxx. - - Example 4: - - Question: Check if the container is using Apache. If it is not using Apache, then it is not vulnerable to CVE-xxxx-xxxxx. - Thought: I should check if Apache is installed. I will check the Software Bill of Materials. - Action: SBOM Package Checker - Action Input: Apache - Observation: 1.0.1 - Thought: Apache is present in the container. I now know the answer. - Final Answer: Apache is installed in the container. Therefore, it is potentially vulnerable to CVE-20xx-xxxxx. +AGENT_EXAMPLES_FOR_PROMPT = """ +Example 1: Python library function lookup (Code Keyword Search) + +Question: Does the codebase import or call urllib.parse? +Thought: I need to search for exact usage of urllib.parse in the codebase. I'll use keyword search for precise matching. +Action: Code Keyword Search +Action Input: urllib.parse +Observation: Found 3 matches: +- src/utils/url_validator.py:12: from urllib.parse import urlparse +- src/api/endpoints.py:45: parsed = urllib.parse.urlparse(user_url) +- tests/test_urls.py:8: import urllib.parse +Thought: The function is present and actively used in production code. I now know the answer. +Final Answer: Yes, urllib.parse is imported and used in the application. Specifically, url_validator.py imports urlparse, and endpoints.py line 45 calls urllib.parse.urlparse(user_url) to process URLs. + +Example 2: Negative result - function not present + +Question: Does the application call the PIL.ImageMath.eval function? +Thought: I should search for calls to PIL.ImageMath.eval using keyword search for exact matching. +Action: Code Keyword Search +Action Input: PIL.ImageMath.eval +Observation: No matches found +Thought: No direct matches. Let me search for the broader ImageMath module to see if it's imported differently. +Action: Code Keyword Search +Action Input: ImageMath +Observation: No matches found +Thought: The module and function are not present in the codebase. I now know the answer. +Final Answer: No, PIL.ImageMath.eval is not called anywhere in the codebase. The ImageMath module is not imported, so this vulnerability vector does not apply to this container. + +Example 3: C/C++ library with call chain analysis (multiple tools) + +Question: Is the OpenSSL function EVP_EncryptInit_ex2 reachable from application code? +Thought: I need to determine if this OpenSSL function can be reached through the application's call chain. First, let me check for direct calls in the codebase. +Action: Code Keyword Search +Action Input: EVP_EncryptInit_ex2 +Observation: No direct calls found in application code +Thought: No direct calls, but the function might be called indirectly through wrapper functions. Let me find which application functions call this OpenSSL function. +Action: Function Caller Finder +Action Input: openssl,EVP_EncryptInit_ex2() +Observation: ['myapp,encrypt_data', 'myapp,secure_transmit'] +Thought: Two application functions call the OpenSSL function. Now I need to check if these functions are reachable from the application entry points. +Action: Call Chain Analyzer +Action Input: myapp,encrypt_data +Observation: (True, ['main', 'handle_request', 'process_upload', 'encrypt_data']) +Thought: I found a complete call chain from main to the function that uses the vulnerable OpenSSL API. I now know the answer. +Final Answer: Yes, EVP_EncryptInit_ex2 is reachable from application code. The call path is: main → handle_request → process_upload → encrypt_data, where encrypt_data calls the vulnerable OpenSSL function. This means the vulnerability could be exploitable if other conditions are met. - """ +""" + +AGENT_EXAMPLES_FOR_PROMPT_2 = """ + +Example 4: Understanding application architecture (semantic searches) + +Question: Does the application process images from untrusted sources? +Thought: I need to understand how the application handles images and where they come from. Let me start by checking the documentation for image handling architecture. +Action: Docs Semantic Search +Action Input: How does the application handle image uploads and image processing? +Observation: The application includes an image processing service that accepts image uploads from users via REST API endpoints at /api/upload. Images are validated for file type and size, then resized and stored in cloud storage. +Thought: The documentation indicates user uploads are accepted. Let me examine the code to understand what validation occurs before image processing. +Action: Code Semantic Search +Action Input: image upload validation and PIL image processing +Observation: Found image_processor.py which validates file extensions (.jpg, .png, .gif) and MIME types before passing images to PIL (Pillow library) for resizing. The validate_image() function checks basic properties but processes the image using PIL.Image.open() and PIL.Image.thumbnail(). +Thought: Images come from users via API endpoints and are processed with PIL after basic validation. I now have sufficient information. +Final Answer: Yes, the application processes images from untrusted sources. User-uploaded images are accepted via the /api/upload REST endpoint and processed using the Pillow (PIL) library. While basic validation (file extension and MIME type) is performed, the images originate from untrusted users and are processed before storage, creating potential exploit vectors. + +Example 5: Specific coding pattern detection (keyword + semantic) + +Question: Does the code modify the keylen or ivlen parameters in OSSL_PARAM arrays after calling EVP initialization functions? +Thought: This is a very specific coding pattern in C code. Let me first search for OSSL_PARAM usage in the codebase. +Action: Code Keyword Search +Action Input: OSSL_PARAM +Observation: Found 2 matches: +- src/crypto/cipher_utils.c:45: OSSL_PARAM params[3]; +- src/crypto/cipher_utils.c:78: OSSL_PARAM params[5]; +Thought: Found OSSL_PARAM usage. Now I need to understand if the code modifies keylen or ivlen after EVP initialization. Let me search for this pattern semantically. +Action: Code Semantic Search +Action Input: OSSL_PARAM array modification of keylen or ivlen after EVP_EncryptInit_ex2 EVP_DecryptInit_ex2 EVP_CipherInit_ex2 calls +Observation: The code in cipher_utils.c creates OSSL_PARAM arrays with keylen and ivlen values set during initialization. The arrays are passed to EVP_EncryptInit_ex2() and EVP_DecryptInit_ex2() but are not modified afterward. The parameter values are set once during array creation using OSSL_PARAM_construct_*() functions. +Thought: I have examined the OSSL_PARAM usage pattern. The vulnerable pattern (modifying keylen/ivlen after initialization) is not present. +Final Answer: No, the codebase does not modify keylen or ivlen parameters in OSSL_PARAM arrays after calling EVP initialization functions. The code in cipher_utils.c creates OSSL_PARAM arrays with keylen and ivlen values set during initialization, passes them to EVP initialization functions, and does not subsequently modify these parameter values. The vulnerable coding pattern described in the CVE is not present. + +Example 6: Configuration-specific check (keyword search for config patterns) + +Question: Does the application use any of the affected cipher modes: CCM, GCM, OCB, RC2, RC4, or RC5? +Thought: I need to search for usage of these specific cipher modes in the codebase. Let me start with GCM. +Action: Code Keyword Search +Action Input: EVP_aes_.*_gcm +Observation: Found 1 match: src/crypto/cipher_utils.c:120: cipher = EVP_aes_256_gcm(); +Thought: Found GCM mode usage. Let me check for the other cipher modes mentioned. +Action: Code Keyword Search +Action Input: EVP_aes_.*_ccm +Observation: No matches found +Thought: No CCM. Let me check for OCB, RC2, RC4, RC5 together. +Action: Code Keyword Search +Action Input: RC2|RC4|RC5|OCB +Observation: No matches found +Thought: I found one of the affected cipher modes (GCM) but not the others. I have the answer. +Final Answer: Yes, the application uses GCM (Galois/Counter Mode), which is one of the affected cipher modes. The code uses EVP_aes_256_gcm() in cipher_utils.c at line 120. The other affected cipher modes (CCM, OCB, RC2, RC4, RC5) are not used in the codebase. +""" -def get_agent_prompt(sys_prompt: str | None = None, prompt_examples: bool = False) -> str: +CVSS_SYS_PROMPT = ( + "You are a cybersecurity expert specializing in Common Vulnerabilities and Exposures (CVE) analysis and CVSS (Common Vulnerability Scoring System) metric evaluation." +) + +CVSS_PROMPT_TEMPLATE = """ +Determine the correct CVSS metric value for this vulnerability in the context of the analyzed container image. Select the value whose definition most accurately matches evidence from the container analysis. + + + +{tools} + + + +Use the ReAct format to complete the task, and follow the below rules to determine what next steps to take. + +FORMAT REQUIREMENTS: +- Start your response with "Question:" and "Metric:" (appear ONCE only at top) +- Do NOT repeat "Question:" or "Metric:" after first occurrence +- After every Thought, provide Action + Action Input (unless ready for Final Answer) +- Final Thought must include: Selected: (); Definition Matched: ; Evidence: +- Final Answer must use exact format: : + +TOOL USAGE: +- NEVER output "Action: None" - this is forbidden +- Call "Container Analysis Data" tool ONCE at start; reuse observation throughout +- Never call same tool twice with identical or semantically similar inputs +- Exhaust all relevant tools before making final selection + +VALUE SELECTION: +- Select the value whose definition ACCURATELY matches your evidence +- Never select if definition contradicts your findings +- Only if you've exhausted all tools without sufficient info, select value marked "least_impact: true" +- When using fallback, make ABSOLUTELY sure to pick the value with least_impact: true + + + +Question: the input question about the CVSS metric +Metric: the metric being evaluated in format: () +Thought: your reasoning about what to do next +Action: the tool to use, must be one of [{tool_names}] +Action Input: the input for that tool +Observation: the result from the tool +... (repeat Thought/Action/Action Input/Observation as needed) +Thought: Final reasoning. Selected: (); Definition Matched: ; Evidence: +Final Answer: : + + + + +Example 1: Scope metric with multiple tool calls + +Question: Based on the container analysis, what is the appropriate Scope (S) value for this CVE? +Metric: Scope (S) +Thought: I need to understand if the vulnerability impact extends beyond the vulnerable component. I'll start by retrieving the container analysis data. +Action: Container Analysis Data +Action Input: +Observation: Previous analysis found: "The vulnerable function urllib.parse is present and called. The application processes URLs from external API requests. Container runs with default user privileges but shares network namespace with host system." +Thought: The analysis mentions shared network namespace with host. Let me search for more details about container isolation and scope. +Action: Code Semantic Search +Action Input: container isolation network namespace boundary privilege escalation cross-container +Observation: Found docker-compose.yml configuring network_mode: host, which means the container shares the host's network stack. Documentation indicates the application can access host network interfaces. +Thought: I now know the answer because the container shares the host network namespace, meaning an exploit could affect resources beyond the vulnerable container itself. Selected: Changed (C); Definition Matched: The vulnerability affects resources beyond the security scope of the vulnerable component; Evidence: "network_mode: host", "shares network namespace with host system" +Final Answer: S:C + +Example 2: Attack Vector with definitive answer from initial data + +Question: What is the Attack Vector (AV) for this vulnerability in the container context? +Metric: Attack Vector (AV) +Thought: I need to understand how the vulnerability can be exploited. Let me get the container analysis data first. +Action: Container Analysis Data +Action Input: +Observation: Previous analysis found: "PIL.ImageMath.eval is called in image_processor.py. The function processes user-uploaded images from the /api/upload endpoint which is publicly exposed on port 8080." +Thought: I now know the answer because the vulnerable functionality is accessible via a public network endpoint that accepts user input. Selected: Network (N); Definition Matched: The vulnerable component is bound to the network stack and the attack can be initiated remotely over the internet; Evidence: "publicly exposed on port 8080", "/api/upload endpoint" +Final Answer: AV:N + +Example 3: Privileges Required with additional investigation + +Question: What level of privileges are required to exploit this vulnerability? +Metric: Privileges Required (PR) +Thought: I need to determine if the attacker needs authentication or privileges. Let me check the container analysis first. +Action: Container Analysis Data +Action Input: +Observation: Previous analysis found: "The vulnerable urllib.parse function is called when processing URLs. The application has multiple endpoints." +Thought: The initial analysis doesn't clarify authentication requirements. Let me search for authentication and authorization controls. +Action: Code Semantic Search +Action Input: authentication required API endpoint authorization middleware access control +Observation: Found middleware/auth.py which implements JWT authentication on all /api/* endpoints except /api/health. The vulnerable URL processing occurs in the /api/parse endpoint which requires valid JWT tokens. +Thought: I now know the answer because the vulnerable endpoint requires JWT authentication. Selected: Low (L); Definition Matched: The attacker must have low-level privileges or must be authenticated to exploit the vulnerability; Evidence: "requires valid JWT tokens", "authentication on all /api/* endpoints" +Final Answer: PR:L + +Example 4: Fallback to least impact when uncertain + +Question: What is the Attack Complexity (AC) for this vulnerability? +Metric: Attack Complexity (AC) +Thought: I need to understand the conditions required for successful exploitation. Let me retrieve the analysis data. +Action: Container Analysis Data +Action Input: +Observation: Previous analysis found: "The vulnerable EVP_EncryptInit_ex2 function is reachable through encrypt_data(). The function uses GCM mode." +Thought: The analysis doesn't provide enough detail about exploitation complexity. Let me search for specific conditions or prerequisites. +Action: Code Semantic Search +Action Input: EVP_EncryptInit_ex2 GCM mode keylen ivlen OSSL_PARAM timing race condition +Observation: Found usage of EVP_EncryptInit_ex2 with OSSL_PARAM arrays but no clear indication of whether exploitation requires specific timing, race conditions, or unusual circumstances. +Thought: I now know the answer because after exhausting available tools, I cannot determine if the vulnerability requires special conditions beyond the attacker's control. Due to insufficient information, I must select the least impact value as fallback. Selected: High (H); Definition Matched: Successful attack depends on conditions beyond the attacker's control; Evidence: None +Final Answer: AC:H + + + +Now determine the correct CVSS metric value by using available tools to gather +evidence, then select the value whose definition best matches your findings. + + +{input} + + +{agent_scratchpad}""" + +def get_agent_prompt(sys_prompt: str | None = None, + prompt_examples: bool = False) -> str: """ - Get the agent prompt based on the system prompt and whether to include examples. + Get the agent prompt template. + + Tool selection strategy is injected via PromptTemplate partial_variables. + + Args: + sys_prompt: Optional system prompt override + prompt_examples: Whether to include few-shot examples + + Returns: + Complete agent prompt template with {tool_selection_strategy} variable """ sys_prompt = sys_prompt or AGENT_SYS_PROMPT - + + # Select template with or without examples if prompt_examples: - prompt_template = AGENT_PROMPT_TEMPLATE.replace("Begin!\n\n", AGENT_EXAMPLES_FOR_PROMPT + "Begin!\n\n") + prompt_template = AGENT_PROMPT_TEMPLATE.replace( + "", + "\n" + AGENT_EXAMPLES_FOR_PROMPT + # AGENT_EXAMPLES_FOR_PROMPT_2 + ) else: prompt_template = AGENT_PROMPT_TEMPLATE + + return f'{sys_prompt}\n\n{prompt_template}' - return f'{sys_prompt} {prompt_template}' - +def get_cvss_prompt(sys_prompt: str | None = None, + prompt_examples: bool = True) -> str: + """ + Get the CVSS prompt based on the system prompt. + + Args: + sys_prompt: Optional system prompt override + prompt_examples: Whether to include few-shot examples (kept for API compatibility) + + Returns: + Complete CVSS prompt with system prompt + """ + sys_prompt = sys_prompt or CVSS_SYS_PROMPT + + # Examples are now embedded in CVSS_PROMPT_TEMPLATE + # prompt_examples parameter kept for backward compatibility but not used + prompt_template = CVSS_PROMPT_TEMPLATE + + return f'{sys_prompt}\n\n{prompt_template}' class PromptBuilder(ABC): @@ -189,6 +500,7 @@ def build_prompt(self) -> str: # IfPromptBuilder('ubuntu_notices', 'Ubuntu Priority Reason: '), # Disabling for now since its very long IfPromptBuilder('ubuntu_ubuntu_description', 'Ubuntu Security Note: '), IfPromptBuilder('vulnerable_dependencies', 'Identified Vulnerable Dependencies: '), + IfPromptBuilder('plugin_data', 'Extra information: '), ] additional_intel_prompting = '\n'.join([pb.build_prompt() for pb in additional_intel_fields]) @@ -249,11 +561,46 @@ def build_prompt(self) -> str: Given CVE Details: """ + additional_intel_prompting -MOD_FEW_SHOT = """Generate a checklist for a security analyst to use when assessing the exploitability of a specific CVE within a containerized environment. Use the provided examples as a guide to understand how to construct a checklist from a given set of CVE details, then apply this understanding to create a specific checklist for the CVE details provided below. All output should be a comma separated list enclosed in square brackets with each list item enclosed in quotes. - +MOD_FEW_SHOT = """ +Generate an investigation checklist for assessing CVE exploitability in a +containerized environment. Your output must be a comma-separated list enclosed +in square brackets, with each item enclosed in quotes. + + + +Create 3-5 checklist items that meet these requirements: + +1. STRUCTURE: Each item must be a clear, answerable question + - Start with interrogative words: Is/Does/Are/Can/Has/etc. + - Be specific and actionable + - Include relevant context from the CVE + +2. CONTENT PRIORITIES: + - If the CVE mentions a specific vulnerable function or method in a given package or library, the FIRST + checklist item must verify whether that function in that package or library is called or imported + in the codebase - function should be specified together with the package name, + for example : 'Is the function1 function from the package1 package called in the codebase?' + - Focus on exploitability factors (version presence is already confirmed) + - Include specific technical names from the CVE (functions, libraries, + configurations, cipher modes, etc.) + - Consider the attack vector (network exposure, user input, file processing, etc.) + - Address relevant security controls or mitigations + +3. INVESTIGATION TOOLS AVAILABLE: + {tool_descriptions} + + Design questions that can be answered using these analysis capabilities. + +4. COMPLETENESS: + - Cover the vulnerability chain: presence → usage → exploitability + - Each item should independently contribute to understanding exploit risk + + + {examples} + -Given CVE Details: + """ investigation_guideline = """1. Is the flagged component in the product? Determine if the container image includes the flagged package version. Verify the presence of the library or software in question by checking the container's software bill of materials (SBOM). diff --git a/src/vuln_analysis/utils/reference_fetcher.py b/src/vuln_analysis/utils/reference_fetcher.py new file mode 100644 index 000000000..d73aa4be7 --- /dev/null +++ b/src/vuln_analysis/utils/reference_fetcher.py @@ -0,0 +1,193 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Reference Fetcher for Advisory URL content retrieval. + +Fetches advisory pages (mailing lists, security bulletins) for LLM hint extraction +as part of the Reference Mining pipeline. +""" + +from __future__ import annotations + +import re + +import aiohttp +from bs4 import BeautifulSoup + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.functions.code_agent_graph_defs import AdvisoryContent +from vuln_analysis.utils.async_http_utils import request_with_retry + +logger = LoggingFactory.get_agent_logger(__name__) + +DEFAULT_FETCH_TIMEOUT_SECONDS = 30 + + +class ReferenceFetcher: + """Fetches advisory page content for Reference Mining. + + Retrieves content from advisory URLs (mailing lists, vendor security pages) + and converts HTML to plain text for LLM extraction. + """ + + def __init__( + self, + timeout_seconds: int = DEFAULT_FETCH_TIMEOUT_SECONDS, + max_content_length: int = 500_000, + ) -> None: + """Initialize the fetcher. + + Args: + timeout_seconds: HTTP request timeout in seconds. + max_content_length: Maximum content length to fetch (bytes). + """ + self._timeout_seconds = timeout_seconds + self._max_content_length = max_content_length + + async def fetch_advisory( + self, + session: aiohttp.ClientSession, + url: str, + source_type: str, + ) -> AdvisoryContent: + """Fetch advisory content from a URL. + + Args: + session: aiohttp ClientSession for making requests. + url: The advisory URL to fetch. + source_type: Classification of the source (e.g., "openwall", "vendor_advisory"). + + Returns: + AdvisoryContent with fetched text or error information. + """ + logger.info("Fetching advisory: %s (type: %s)", url, source_type) + + try: + request_kwargs = { + "method": "GET", + "url": url, + "timeout": aiohttp.ClientTimeout(total=self._timeout_seconds), + "headers": { + "User-Agent": "Mozilla/5.0 (compatible; CVE-Advisory-Fetcher/1.0)", + "Accept": "text/html,text/plain,application/xhtml+xml,*/*", + }, + } + + async with request_with_retry( + session, + request_kwargs, + max_retries=3, + retry_on_client_errors=False, + ) as response: + content_type = response.headers.get("Content-Type", "") + content_length = response.content_length or 0 + + if content_length > self._max_content_length: + logger.warning( + "Advisory content too large (%d bytes), truncating: %s", + content_length, + url, + ) + + raw_bytes = await response.content.read(self._max_content_length) + raw_text = self._decode_content(raw_bytes, content_type) + + if "html" in content_type.lower(): + text = _extract_text_from_html(raw_text) + else: + text = raw_text + + logger.info( + "Fetched advisory: %s (%d chars)", + url, + len(text), + ) + + return AdvisoryContent( + url=url, + source_type=source_type, + raw_text=text, + fetch_success=True, + error_message=None, + ) + + except aiohttp.ClientError as e: + error_msg = f"HTTP error fetching {url}: {e}" + logger.warning(error_msg) + return AdvisoryContent( + url=url, + source_type=source_type, + raw_text="", + fetch_success=False, + error_message=error_msg, + ) + except Exception as e: + error_msg = f"Unexpected error fetching {url}: {e}" + logger.error(error_msg, exc_info=True) + return AdvisoryContent( + url=url, + source_type=source_type, + raw_text="", + fetch_success=False, + error_message=error_msg, + ) + + def _decode_content(self, raw_bytes: bytes, content_type: str) -> str: + """Decode raw bytes to string, handling encoding detection. + + Args: + raw_bytes: Raw response bytes. + content_type: Content-Type header value. + + Returns: + Decoded string content. + """ + encoding = "utf-8" + if "charset=" in content_type: + charset_match = re.search(r"charset=([^\s;]+)", content_type) + if charset_match: + encoding = charset_match.group(1).strip('"\'') + + try: + return raw_bytes.decode(encoding) + except (UnicodeDecodeError, LookupError): + return raw_bytes.decode("utf-8", errors="replace") + + +def _extract_text_from_html(html: str) -> str: + """Extract readable text from HTML content using BeautifulSoup. + + Args: + html: Raw HTML content. + + Returns: + Extracted plain text with normalized whitespace. + """ + soup = BeautifulSoup(html, "html.parser") + + # Remove script and style elements + for element in soup(["script", "style", "noscript", "header", "footer", "nav"]): + element.decompose() + + # Get text with separator for block elements + text = soup.get_text(separator="\n") + + # Normalize whitespace + text = re.sub(r"[ \t]+", " ", text) + text = re.sub(r"\n\s*\n+", "\n\n", text) + text = text.strip() + + return text diff --git a/src/vuln_analysis/utils/reference_parser.py b/src/vuln_analysis/utils/reference_parser.py new file mode 100644 index 000000000..5b017c974 --- /dev/null +++ b/src/vuln_analysis/utils/reference_parser.py @@ -0,0 +1,575 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Reflective Reference Parser for CVE hint extraction. + +Uses LLM with reflection pattern to extract hints from advisory content, +with chunking support for limited context window LLMs. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from langchain_core.language_models import BaseChatModel + +from langchain_core.messages import HumanMessage, SystemMessage +from pydantic import BaseModel, Field + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.functions.code_agent_graph_defs import ( + HINT_STRENGTH_NONE, + HINT_STRENGTH_STRONG, + HINT_STRENGTH_MEDIUM, + HINT_STRENGTH_WEAK, + ReferenceHints, +) + +logger = LoggingFactory.get_agent_logger(__name__) + +# --------------------------------------------------------------------------- +# Token estimation and chunking utilities +# --------------------------------------------------------------------------- + +CHARS_PER_TOKEN_ESTIMATE = 4 +DEFAULT_CHUNK_OVERLAP_TOKENS = 200 +DEFAULT_MIN_CHUNK_TOKENS = 500 + + +def estimate_tokens(text: str) -> int: + """Estimate token count using simple character heuristic. + + Args: + text: Input text. + + Returns: + Estimated token count (chars / 4). + """ + return len(text) // CHARS_PER_TOKEN_ESTIMATE + + +def split_into_chunks( + text: str, + max_tokens: int, + overlap_tokens: int = DEFAULT_CHUNK_OVERLAP_TOKENS, + min_chunk_tokens: int = DEFAULT_MIN_CHUNK_TOKENS, +) -> list[str]: + """Split text into chunks respecting token limits and paragraph boundaries. + + Args: + text: Text to split. + max_tokens: Maximum tokens per chunk. + overlap_tokens: Token overlap between chunks for context preservation. + min_chunk_tokens: Minimum chunk size (don't create tiny chunks). + + Returns: + List of text chunks. + """ + if estimate_tokens(text) <= max_tokens: + return [text] + + max_chars = max_tokens * CHARS_PER_TOKEN_ESTIMATE + overlap_chars = overlap_tokens * CHARS_PER_TOKEN_ESTIMATE + min_chars = min_chunk_tokens * CHARS_PER_TOKEN_ESTIMATE + + paragraphs = re.split(r"\n\s*\n", text) + + chunks: list[str] = [] + current_chunk: list[str] = [] + current_length = 0 + + for para in paragraphs: + para_length = len(para) + + if current_length + para_length > max_chars and current_chunk: + chunk_text = "\n\n".join(current_chunk) + chunks.append(chunk_text) + + overlap_text = _get_overlap_text(current_chunk, overlap_chars) + current_chunk = [overlap_text] if overlap_text else [] + current_length = len(overlap_text) if overlap_text else 0 + + current_chunk.append(para) + current_length += para_length + 2 + + if current_chunk: + chunk_text = "\n\n".join(current_chunk) + if chunks and estimate_tokens(chunk_text) < min_chunk_tokens: + chunks[-1] = chunks[-1] + "\n\n" + chunk_text + else: + chunks.append(chunk_text) + + return chunks if chunks else [text] + + +def _get_overlap_text(paragraphs: list[str], max_chars: int) -> str: + """Get trailing text from paragraphs up to max_chars for overlap.""" + if not paragraphs: + return "" + + result: list[str] = [] + total = 0 + + for para in reversed(paragraphs): + if total + len(para) > max_chars: + break + result.insert(0, para) + total += len(para) + 2 + + return "\n\n".join(result) + + +def merge_chunk_hints(chunk_hints: list[ReferenceHints]) -> ReferenceHints: + """Merge hints from multiple chunks, keeping first revision and deduplicating others. + + For revision_hint: keeps the first non-None value found (earlier chunks + are more likely to contain the primary CVE information). + + For other hints: deduplicates across chunks. + + Args: + chunk_hints: List of ReferenceHints from individual chunks. + + Returns: + Merged ReferenceHints with single revision and deduplicated lists. + """ + if not chunk_hints: + return ReferenceHints() + + if len(chunk_hints) == 1: + return chunk_hints[0] + + merged = ReferenceHints() + + for hints in chunk_hints: + if merged.revision_hint is None and hints.revision_hint is not None: + merged.revision_hint = hints.revision_hint + logger.debug("merge_chunk_hints: keeping revision_hint=%s", hints.revision_hint) + + merged.function_hints = list(set(merged.function_hints + hints.function_hints)) + merged.file_hints = list(set(merged.file_hints + hints.file_hints)) + merged.branch_hints = list(set(merged.branch_hints + hints.branch_hints)) + merged.version_hints = list(set(merged.version_hints + hints.version_hints)) + + if hints.confidence > merged.confidence: + merged.confidence = hints.confidence + merged.hint_strength = hints.hint_strength + + return merged + + +# --------------------------------------------------------------------------- +# LLM response models +# --------------------------------------------------------------------------- + + +class HintWithContext(BaseModel): + """A hint value with extraction context.""" + value: str + context: str = "" + + +class ExtractionResult(BaseModel): + """LLM extraction output model.""" + revision_hint: HintWithContext | None = Field(default=None) + function_hints: list[HintWithContext] = Field(default_factory=list) + file_hints: list[HintWithContext] = Field(default_factory=list) + branch_hints: list[HintWithContext] = Field(default_factory=list) + version_hints: list[HintWithContext] = Field(default_factory=list) + extraction_notes: str = "" + + +class QualityIssue(BaseModel): + """A quality issue identified during reflection.""" + field: str + issue: str + severity: str = "medium" + + +class MissedHint(BaseModel): + """A hint that was missed in extraction.""" + type: str + value: str + evidence: str = "" + + +class ReflectionResult(BaseModel): + """LLM reflection output model.""" + quality_issues: list[QualityIssue] = Field(default_factory=list) + missed_hints: list[MissedHint] = Field(default_factory=list) + hint_strength: str = HINT_STRENGTH_NONE + confidence: float = 0.0 + needs_more_sources: bool = True + decision_reasoning: str = "" + + +# --------------------------------------------------------------------------- +# Prompts +# --------------------------------------------------------------------------- + +REFERENCE_EXTRACT_PROMPT = """You are analyzing a CVE advisory to extract hints for finding the fix commit. + +## Context + +CVE: {vuln_id} +CVE Description: {cve_description} +Affected Package: {package_name} + +## Advisory Source + +URL: {advisory_url} +Source Type: {source_type} +{chunk_info} + +## Advisory Content + +``` +{advisory_content} +``` + +## Task + +Extract hints from the advisory text to help find the fix commit: + +1. **revision_hint**: The SINGLE SVN revision (r1935008) or git commit hash for THIS CVE's fix. + - IMPORTANT: Select only ONE revision that is CLEARLY for {vuln_id} + - If multiple revisions appear, pick the one that matches the CVE description + - Return null if no revision clearly relates to this specific CVE +2. **function_hints**: Function names mentioned as vulnerable or patched +3. **file_hints**: Source file paths (.c, .h, .py, .go, etc.) +4. **branch_hints**: Branch names where the fix was applied (e.g., "2.4.x", "main", "stable") +5. **version_hints**: Version numbers that contain the fix (e.g., "2.4.68", "1.2.3") + +## Rules + +- Only extract hints that are CLEARLY related to THIS CVE ({vuln_id}) +- For revision_hint: if the advisory mentions multiple CVEs/fixes, select ONLY the revision for {vuln_id} +- Do NOT hallucinate - if no hints found, return null/empty +- Distinguish between bug IDs and revision numbers + +## Output Format + +Return valid JSON: + +```json +{{ + "revision_hint": {{"value": "r1935008", "context": "fix commit for {vuln_id}"}}, + "function_hints": [ + {{"value": "ap_proxy_cookie_reverse", "context": "vulnerable function"}} + ], + "file_hints": [ + {{"value": "proxy_util.c", "context": "file containing the fix"}} + ], + "branch_hints": [ + {{"value": "2.4.x", "context": "branch where fix was backported"}} + ], + "version_hints": [ + {{"value": "2.4.68", "context": "first fixed version"}} + ], + "extraction_notes": "Any observations about the advisory quality or ambiguity" +}} +```""" + +REFERENCE_REFLECT_PROMPT = """You are reviewing your own extraction of CVE fix hints. +Your task is to CRITIQUE the quality AND DECIDE if more sources are needed. + +## Context + +CVE: {vuln_id} +CVE Description: {cve_description} +Package: {package_name} + +## Current Advisory + +URL: {current_url} + +## Your Extraction + +```json +{extraction_json} +``` + +## Other Advisory URLs Available (not yet processed) + +{remaining_urls} + +--- + +## Part 1: Quality Critique + +Evaluate your extraction: + +1. **Relevance Check**: Is the extracted revision_hint clearly about THIS CVE, not other CVEs? +2. **Type Validation**: + - Is "revision_hint" actually a revision (not a bug ID, ticket number)? + - Are "function_hints" valid function names (not variables, types)? + - Are "file_hints" actual file paths (not directories, URLs)? +3. **Completeness Check**: Did you miss any obvious hints in the text? + +## Part 2: Hint Strength Assessment + +Rate the strength of hints for finding a git commit: + +- **strong**: Have revision/commit hash, OR specific function + file combination +- **medium**: Have function OR file hints with version info +- **weak**: Only have version hints or generic terms +- **none**: No useful hints extracted + +## Part 3: Control Flow Decision + +Based on hint strength and remaining URLs, decide: + +| Condition | Decision | +|-----------|----------| +| Hints are strong | `needs_more_sources: false` (proceed to git search) | +| Hints are weak/none + URLs remain | `needs_more_sources: true` (fetch more) | +| Hints are medium + high-quality URLs remain | `needs_more_sources: true` (try to improve) | +| Hints are medium + low-quality URLs remain | `needs_more_sources: false` (good enough) | +| No URLs remain | `needs_more_sources: false` (can't fetch more) | + +## Output Format + +Return valid JSON: + +```json +{{ + "quality_issues": [ + {{"field": "revision_hint", "issue": "might be a bug ID, not SVN revision", "severity": "medium"}} + ], + "missed_hints": [ + {{"type": "version_hints", "value": "2.4.68", "evidence": "text says 'fixed in 2.4.68'"}} + ], + "hint_strength": "strong", + "confidence": 0.85, + "needs_more_sources": false, + "decision_reasoning": "We have a revision hint (r1935008) which is strong enough to search git log directly." +}} +```""" + + +# --------------------------------------------------------------------------- +# Parser class +# --------------------------------------------------------------------------- + + +@dataclass +class ParserConfig: + """Configuration for ReflectiveReferenceParser.""" + max_chunk_tokens: int = 4400 + chunk_overlap_tokens: int = DEFAULT_CHUNK_OVERLAP_TOKENS + min_chunk_tokens: int = DEFAULT_MIN_CHUNK_TOKENS + + +class ReflectiveReferenceParser: + """Extracts hints from advisory content using LLM with reflection pattern. + + Handles large content via chunking, extracts hints, and uses reflection + to assess quality and decide if more sources are needed. + """ + + def __init__(self, llm: BaseChatModel, config: ParserConfig | None = None) -> None: + """Initialize the parser. + + Args: + llm: LangChain LLM for extraction and reflection. + config: Parser configuration (uses defaults if None). + """ + self._llm = llm + self._config = config or ParserConfig() + + async def extract_hints( + self, + advisory_content: str, + vuln_id: str, + cve_description: str, + package_name: str, + advisory_url: str, + source_type: str, + ) -> ReferenceHints: + """Extract hints from advisory content, handling chunking if needed. + + Args: + advisory_content: Raw text content from advisory page. + vuln_id: CVE identifier (e.g., "CVE-2024-1234"). + cve_description: CVE description text. + package_name: Affected package name. + advisory_url: URL of the advisory. + source_type: Classification of the source. + + Returns: + ReferenceHints with extracted values. + """ + chunks = split_into_chunks( + advisory_content, + max_tokens=self._config.max_chunk_tokens, + overlap_tokens=self._config.chunk_overlap_tokens, + min_chunk_tokens=self._config.min_chunk_tokens, + ) + + logger.info( + "Extracting hints from %s (%d chunks)", + advisory_url, + len(chunks), + ) + + chunk_hints: list[ReferenceHints] = [] + + for i, chunk in enumerate(chunks): + chunk_info = f"Chunk {i + 1}/{len(chunks)}" if len(chunks) > 1 else "" + + extraction = await self._extract_from_chunk( + chunk=chunk, + vuln_id=vuln_id, + cve_description=cve_description, + package_name=package_name, + advisory_url=advisory_url, + source_type=source_type, + chunk_info=chunk_info, + ) + + hints = self._extraction_to_hints(extraction) + chunk_hints.append(hints) + + merged = merge_chunk_hints(chunk_hints) + merged.sources_processed.append(advisory_url) + + logger.info( + "Extracted hints: revision=%s, %d functions, %d files", + merged.revision_hint, + len(merged.function_hints), + len(merged.file_hints), + ) + + return merged + + async def reflect_on_extraction( + self, + extraction: ReferenceHints, + vuln_id: str, + cve_description: str, + package_name: str, + current_url: str, + remaining_urls: list[tuple[str, str, int]], + ) -> ReflectionResult: + """Reflect on extraction quality and decide if more sources needed. + + Args: + extraction: Current extracted hints. + vuln_id: CVE identifier. + cve_description: CVE description. + package_name: Package name. + current_url: URL that was just processed. + remaining_urls: List of (url, source_type, priority) not yet processed. + + Returns: + ReflectionResult with quality assessment and control flow decision. + """ + extraction_dict = { + "revision_hint": extraction.revision_hint, + "function_hints": extraction.function_hints, + "file_hints": extraction.file_hints, + "branch_hints": extraction.branch_hints, + "version_hints": extraction.version_hints, + } + + if remaining_urls: + urls_text = "\n".join( + f"- {url} (type: {stype}, priority: {priority})" + for url, stype, priority in remaining_urls + ) + else: + urls_text = "None - all available URLs have been processed." + + prompt = REFERENCE_REFLECT_PROMPT.format( + vuln_id=vuln_id, + cve_description=cve_description[:500] if cve_description else "Not available", + package_name=package_name, + current_url=current_url, + extraction_json=json.dumps(extraction_dict, indent=2), + remaining_urls=urls_text, + ) + + try: + structured_llm = self._llm.with_structured_output(ReflectionResult) + result: ReflectionResult = await structured_llm.ainvoke([ + SystemMessage(content=prompt), + HumanMessage(content="Reflect on the extraction quality."), + ]) # type: ignore[assignment] + + logger.info( + "Reflection: strength=%s, confidence=%.2f, needs_more=%s", + result.hint_strength, + result.confidence, + result.needs_more_sources, + ) + + return result + + except Exception as e: + logger.error("Reflection failed: %s", e, exc_info=True) + return ReflectionResult( + hint_strength=HINT_STRENGTH_WEAK, + confidence=0.3, + needs_more_sources=bool(remaining_urls), + decision_reasoning=f"Reflection failed: {e}", + ) + + async def _extract_from_chunk( + self, + chunk: str, + vuln_id: str, + cve_description: str, + package_name: str, + advisory_url: str, + source_type: str, + chunk_info: str, + ) -> ExtractionResult: + """Extract hints from a single chunk using structured LLM output.""" + prompt = REFERENCE_EXTRACT_PROMPT.format( + vuln_id=vuln_id, + cve_description=cve_description[:500] if cve_description else "Not available", + package_name=package_name, + advisory_url=advisory_url, + source_type=source_type, + chunk_info=f"\n{chunk_info}" if chunk_info else "", + advisory_content=chunk, + ) + + try: + structured_llm = self._llm.with_structured_output(ExtractionResult) + result: ExtractionResult = await structured_llm.ainvoke([ + SystemMessage(content=prompt), + HumanMessage(content="Extract hints from the advisory."), + ]) # type: ignore[assignment] + return result + + except Exception as e: + logger.error("Extraction failed for chunk: %s", e, exc_info=True) + return ExtractionResult() + + def _extraction_to_hints(self, extraction: ExtractionResult) -> ReferenceHints: + """Convert ExtractionResult to ReferenceHints.""" + return ReferenceHints( + revision_hint=extraction.revision_hint.value if extraction.revision_hint else None, + function_hints=[h.value for h in extraction.function_hints], + file_hints=[h.value for h in extraction.file_hints], + branch_hints=[h.value for h in extraction.branch_hints], + version_hints=[h.value for h in extraction.version_hints], + ) diff --git a/src/vuln_analysis/utils/repo_resolver.py b/src/vuln_analysis/utils/repo_resolver.py new file mode 100644 index 000000000..35fb33a17 --- /dev/null +++ b/src/vuln_analysis/utils/repo_resolver.py @@ -0,0 +1,512 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Repository Resolver - map package names to upstream git repository URLs. + +Resolves package names (e.g., "curl", "httpd", "openssl") to their upstream +git repository URLs using multiple strategies in priority order: + +1. OSIDB upstream_purls (confidence 0.95) +2. Curated package → repo mapping (confidence 0.90, 0.85 for aliases) +3. PURL parsing from any intel source (confidence 0.80) + +Only public repositories are supported. If resolution fails, returns +a RepoResolution with method="not_found" and logs the package for +future mapping expansion. +""" + +from __future__ import annotations + +import json +import re +from functools import lru_cache +from pathlib import Path +from typing import TYPE_CHECKING + +from packageurl import PackageURL + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.functions.code_agent_graph_defs import ( + RESOLUTION_METHOD_CURATED_MAPPING, + RESOLUTION_METHOD_NOT_FOUND, + RESOLUTION_METHOD_OSIDB_PURL, + RESOLUTION_METHOD_PURL_PARSE, + RepoResolution, +) + +if TYPE_CHECKING: + from exploit_iq_commons.data_models.cve_intel import CveIntel + +logger = LoggingFactory.get_agent_logger(__name__) + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +MAPPING_FILE_PATH = Path(__file__).parent.parent.parent / "exploit_iq_commons" / "data" / "package_repo_mapping.json" + +CONFIDENCE_OSIDB_PURL = 0.95 +CONFIDENCE_CURATED_DIRECT = 0.90 +CONFIDENCE_CURATED_ALIAS = 0.85 +CONFIDENCE_PURL_PARSE = 0.80 + +PURL_TYPE_GITHUB = "github" +PURL_TYPE_GITLAB = "gitlab" +PURL_TYPE_BITBUCKET = "bitbucket" + +PLATFORM_GITHUB = "github" +PLATFORM_GITLAB = "gitlab" +PLATFORM_BITBUCKET = "bitbucket" +PLATFORM_KERNEL_ORG = "kernel.org" +PLATFORM_SOURCEWARE = "sourceware" +PLATFORM_SAVANNAH = "savannah" +PLATFORM_UNKNOWN = "unknown" + +# Prefixes to strip during package name normalization +_PACKAGE_PREFIXES = ( + "lib", + "python-", + "python3-", + "py-", + "node-", + "golang-", + "rubygem-", + "perl-", + "php-", +) + +# Pattern to strip version suffixes (e.g., libcurl4 → libcurl) +_VERSION_SUFFIX_PATTERN = re.compile(r"\d+$") + + +# --------------------------------------------------------------------------- +# Mapping Loader (with caching) +# --------------------------------------------------------------------------- + + +@lru_cache(maxsize=1) +def load_package_repo_mapping() -> dict: + """Load the curated package-to-repo mapping from JSON file. + + Uses module-level caching to avoid repeated file reads. + + Returns: + Dictionary with "packages" and "aliases" keys. + + Raises: + FileNotFoundError: If mapping file doesn't exist. + json.JSONDecodeError: If mapping file is invalid JSON. + """ + logger.debug("Loading package repo mapping from %s", MAPPING_FILE_PATH) + + if not MAPPING_FILE_PATH.exists(): + logger.error("Package repo mapping file not found: %s", MAPPING_FILE_PATH) + raise FileNotFoundError(f"Package repo mapping not found: {MAPPING_FILE_PATH}") + + with MAPPING_FILE_PATH.open("r", encoding="utf-8") as f: + mapping = json.load(f) + + package_count = len(mapping.get("packages", {})) + alias_count = len(mapping.get("aliases", {})) + logger.info( + "Loaded package repo mapping: %d packages, %d aliases", + package_count, + alias_count, + ) + + return mapping + + +def clear_mapping_cache() -> None: + """Clear the cached mapping (useful for testing).""" + load_package_repo_mapping.cache_clear() + + +# --------------------------------------------------------------------------- +# Package Name Normalization +# --------------------------------------------------------------------------- + + +def normalize_package_name(name: str) -> list[str]: + """Generate normalized variants of a package name for mapping lookup. + + Strips common prefixes (lib, python-, etc.) and version suffixes + to increase matching probability. + + Args: + name: Original package name (e.g., "python3-urllib3", "libcurl4") + + Returns: + List of name variants to try, most specific first. + + Examples: + >>> normalize_package_name("python3-urllib3") + ['python3-urllib3', 'urllib3'] + >>> normalize_package_name("libcurl4") + ['libcurl4', 'curl4', 'libcurl', 'curl'] + """ + if not name: + return [] + + variants = [name] + name_lower = name.lower() + + # Strip version suffix from original first (libcurl4 → libcurl) + no_version = _VERSION_SUFFIX_PATTERN.sub("", name_lower) + if no_version and no_version != name_lower and no_version not in variants: + variants.append(no_version) + + # Strip common prefixes from original (libcurl4 → curl4) + for prefix in _PACKAGE_PREFIXES: + if name_lower.startswith(prefix): + stripped = name_lower[len(prefix):] + if stripped and stripped not in variants: + variants.append(stripped) + # Also strip version from this (curl4 → curl) + stripped_no_ver = _VERSION_SUFFIX_PATTERN.sub("", stripped) + if stripped_no_ver and stripped_no_ver != stripped and stripped_no_ver not in variants: + variants.append(stripped_no_ver) + break + + return variants + + +# --------------------------------------------------------------------------- +# PURL Utilities +# --------------------------------------------------------------------------- + + +_PURL_TYPE_TO_PLATFORM = { + PURL_TYPE_GITHUB: (PLATFORM_GITHUB, "https://github.com"), + PURL_TYPE_GITLAB: (PLATFORM_GITLAB, "https://gitlab.com"), + PURL_TYPE_BITBUCKET: (PLATFORM_BITBUCKET, "https://bitbucket.org"), +} + + +def purl_to_repo_url(purl: str | None) -> tuple[str | None, str | None]: + """Convert Package URL to repository URL. + + Supported PURL types: + - pkg:github/owner/repo → https://github.com/owner/repo + - pkg:gitlab/group/project → https://gitlab.com/group/project + - pkg:bitbucket/owner/repo → https://bitbucket.org/owner/repo + + Args: + purl: Package URL string (e.g., "pkg:github/curl/curl") + + Returns: + Tuple of (repo_url, platform) or (None, None) if unsupported. + """ + if not purl: + return None, None + + try: + parsed = PackageURL.from_string(purl) + + platform_info = _PURL_TYPE_TO_PLATFORM.get(parsed.type) + if not platform_info: + return None, None + + platform, base_url = platform_info + + if not parsed.namespace or not parsed.name: + logger.debug("PURL missing namespace or name: %s", purl) + return None, None + + repo_url = f"{base_url}/{parsed.namespace}/{parsed.name}" + return repo_url, platform + + except Exception as e: + logger.debug("Failed to parse PURL %s: %s", purl, e) + return None, None + + +def detect_platform(repo_url: str) -> str: + """Detect the hosting platform from a repository URL. + + Args: + repo_url: Repository URL + + Returns: + Platform identifier string. + """ + if not repo_url: + return PLATFORM_UNKNOWN + + url_lower = repo_url.lower() + + if "github.com" in url_lower: + return PLATFORM_GITHUB + if "gitlab.com" in url_lower or "gitlab." in url_lower: + return PLATFORM_GITLAB + if "bitbucket.org" in url_lower: + return PLATFORM_BITBUCKET + if "kernel.org" in url_lower: + return PLATFORM_KERNEL_ORG + if "sourceware.org" in url_lower: + return PLATFORM_SOURCEWARE + if "savannah.gnu.org" in url_lower: + return PLATFORM_SAVANNAH + + return PLATFORM_UNKNOWN + + +# --------------------------------------------------------------------------- +# PURL Collection from Intel +# --------------------------------------------------------------------------- + + +def collect_all_purls(intel: CveIntel | None) -> list[str]: + """Collect PURLs from all intel sources (OSIDB, NVD, GHSA). + + Args: + intel: CveIntel object containing data from various sources + + Returns: + Deduplicated list of PURL strings found in intel. + """ + if not intel: + return [] + + purls: set[str] = set() + + # OSIDB upstream_purls + if intel.osidb and intel.osidb.upstream_purls: + for purl_info in intel.osidb.upstream_purls: + if hasattr(purl_info, "purl") and purl_info.purl: + purls.add(purl_info.purl) + + # OSIDB affects (may contain PURLs) + if intel.osidb and intel.osidb.affects: + for affect in intel.osidb.affects: + if hasattr(affect, "purl") and affect.purl: + purls.add(affect.purl) + + return list(purls) + + +# --------------------------------------------------------------------------- +# Repository Resolver +# --------------------------------------------------------------------------- + + +class RepoResolver: + """Resolve package names to upstream git repository URLs. + + Uses multiple strategies in priority order: + 1. OSIDB upstream_purls (highest confidence) + 2. Curated package → repo mapping (PRIMARY for C/C++) + 3. PURL parsing from any intel source + + Example: + resolver = RepoResolver() + result = resolver.resolve("curl", intel=vulnerability_intel) + if result.repo_url: + print(f"Found repo: {result.repo_url} via {result.resolution_method}") + """ + + def __init__(self) -> None: + """Initialize the resolver.""" + self._mapping: dict | None = None + + @property + def mapping(self) -> dict: + """Lazily load and cache the package mapping.""" + if self._mapping is None: + try: + self._mapping = load_package_repo_mapping() + except FileNotFoundError: + logger.warning("Package mapping file not found, using empty mapping") + self._mapping = {"packages": {}, "aliases": {}} + return self._mapping + + def resolve( + self, + package_name: str, + intel: CveIntel | None = None, + ) -> RepoResolution: + """Resolve package name to upstream git repository URL. + + Tries strategies in order until one succeeds: + 1. OSIDB upstream_purls + 2. Curated mapping + 3. PURL parsing + + Args: + package_name: Package name to resolve (e.g., "curl", "httpd") + intel: Optional vulnerability intel containing PURLs + + Returns: + RepoResolution with repo_url if found, or method="not_found". + """ + strategies_tried: list[str] = [] + error_messages: list[str] = [] + alternative_repos: list[str] = [] + + # Strategy 1: OSIDB upstream_purls + strategies_tried.append(RESOLUTION_METHOD_OSIDB_PURL) + result = self._try_osidb_purls(intel) + if result: + repo_url, platform = result + logger.info( + "repo_resolution: resolved %s via OSIDB PURL → %s", + package_name, + repo_url, + ) + return RepoResolution( + repo_url=repo_url, + repo_platform=platform, + resolution_method=RESOLUTION_METHOD_OSIDB_PURL, + confidence=CONFIDENCE_OSIDB_PURL, + is_alias=False, + strategies_tried=strategies_tried, + ) + logger.debug( + "repo_resolution: OSIDB PURL strategy failed for %s, trying curated mapping", + package_name, + ) + + # Strategy 2: Curated mapping (PRIMARY) + strategies_tried.append(RESOLUTION_METHOD_CURATED_MAPPING) + result = self._try_curated_mapping(package_name) + if result: + repo_url, is_alias = result + platform = detect_platform(repo_url) + confidence = CONFIDENCE_CURATED_ALIAS if is_alias else CONFIDENCE_CURATED_DIRECT + logger.info( + "repo_resolution: resolved %s via curated mapping%s → %s", + package_name, + " (alias)" if is_alias else "", + repo_url, + ) + return RepoResolution( + repo_url=repo_url, + repo_platform=platform, + resolution_method=RESOLUTION_METHOD_CURATED_MAPPING, + confidence=confidence, + is_alias=is_alias, + strategies_tried=strategies_tried, + ) + logger.debug( + "repo_resolution: curated mapping strategy failed for %s, trying PURL parse", + package_name, + ) + + # Strategy 3: PURL parsing from any intel source + strategies_tried.append(RESOLUTION_METHOD_PURL_PARSE) + result = self._try_purl_parse(intel) + if result: + repo_url, platform = result + logger.info( + "repo_resolution: resolved %s via PURL parse → %s", + package_name, + repo_url, + ) + return RepoResolution( + repo_url=repo_url, + repo_platform=platform, + resolution_method=RESOLUTION_METHOD_PURL_PARSE, + confidence=CONFIDENCE_PURL_PARSE, + is_alias=False, + strategies_tried=strategies_tried, + ) + + # All strategies failed - log for mapping expansion + logger.info( + "repo_resolution: no repo found for '%s' - consider adding to mapping", + package_name, + ) + return RepoResolution( + repo_url=None, + repo_platform=None, + resolution_method=RESOLUTION_METHOD_NOT_FOUND, + confidence=0.0, + is_alias=False, + strategies_tried=strategies_tried, + error_messages=error_messages, + alternative_repos=alternative_repos, + ) + + def _try_osidb_purls( + self, + intel: CveIntel | None, + ) -> tuple[str, str] | None: + """Strategy 1: Extract repo URL from OSIDB upstream_purls. + + Returns: + Tuple of (repo_url, platform) if found, None otherwise. + """ + if not intel or not intel.osidb: + return None + + if not intel.osidb.upstream_purls: + return None + + for purl_info in intel.osidb.upstream_purls: + purl = getattr(purl_info, "purl", None) + if not purl: + continue + + repo_url, platform = purl_to_repo_url(purl) + if repo_url and platform: + return repo_url, platform + + return None + + def _try_curated_mapping( + self, + package_name: str, + ) -> tuple[str, bool] | None: + """Strategy 2: Look up package in curated mapping. + + Returns: + Tuple of (repo_url, is_alias) if found, None otherwise. + """ + packages = self.mapping.get("packages", {}) + aliases = self.mapping.get("aliases", {}) + + # Generate name variants + variants = normalize_package_name(package_name) + + for name in variants: + # Check direct mapping + if name in packages: + return packages[name], False + + # Check aliases + if name in aliases: + canonical = aliases[name] + if canonical in packages: + return packages[canonical], True + + return None + + def _try_purl_parse( + self, + intel: CveIntel | None, + ) -> tuple[str, str] | None: + """Strategy 3: Parse PURLs from any intel source. + + Returns: + Tuple of (repo_url, platform) if found, None otherwise. + """ + purls = collect_all_purls(intel) + + for purl in purls: + repo_url, platform = purl_to_repo_url(purl) + if repo_url and platform: + return repo_url, platform + + return None diff --git a/src/vuln_analysis/utils/rpm_checker_prompts.py b/src/vuln_analysis/utils/rpm_checker_prompts.py new file mode 100644 index 000000000..80c75420e --- /dev/null +++ b/src/vuln_analysis/utils/rpm_checker_prompts.py @@ -0,0 +1,1969 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Prompt templates for the RPM Checker L1 (Code Agent) and L2 (Build Agent) phases. + +L1 prompts handle source code verification for CVE patch status. +L2 prompts handle build-time configuration and hardening flag verification. +""" + +# =========================================================================== +# L1 CODE AGENT PROMPTS +# =========================================================================== + +# --------------------------------------------------------------------------- +# L1 Verdict Extraction +# --------------------------------------------------------------------------- + +L1_VERDICT_EXTRACTION_PROMPT = """\ +Extract the security verdict from this L1 agent investigation conclusion. + +CVE: {vuln_id} +Package: {target_package} + +L1 Agent Final Answer: +{final_answer} + +Classify the conclusion into one of these categories: +- "protected": The package is protected (patch applied, fix backported, or mitigating control present) +- "not_present": The vulnerable code/function is not present in this version +- "vulnerable": The vulnerable code is confirmed present and unpatched +- "uncertain": Insufficient evidence or conflicting findings + +Provide your confidence level (0.0-1.0) based on the strength of evidence in the answer. +""" + +# --------------------------------------------------------------------------- +# L1 Vulnerability Intel Extraction +# --------------------------------------------------------------------------- + +VULNERABILITY_INTEL_EXTRACTION_PROMPT = """\ +Extract structured vulnerability intelligence from the CVE data and patch content. +Your output will be used to guide source code searches, so focus on grep-able patterns. + + +CVE ID: {vuln_id} +Package: {target_package} +CVE Description: {cve_description} + + + +{vendor_mitigations} + + + +{patch_data} + + + +1. affected_files: Extract file paths from patch headers (strip a/ b/ prefixes) +2. vulnerable_functions: Extract function names ONLY from: + - Removed lines (- lines) in patch (HIGHEST PRIORITY - these are the actual vulnerable functions) + - Explicit function names mentioned VERBATIM in CVE description (e.g., "vulnerability in parse_rockridge()") + - Functions or buffers EXPLICITLY named in VENDOR_MITIGATIONS + ANTI-HALLUCINATION: Do NOT infer or construct function names from descriptions. + - BAD: CVE says "HTML directory list generation" -> guessing "mod_proxy_ftp_html_dir_list" (WRONG - made up) + - GOOD: CVE says "vulnerability in ap_proxy_ftp_handler()" -> extract "ap_proxy_ftp_handler" (CORRECT - verbatim) + - If no explicit function names exist in patch or CVE, leave this field EMPTY rather than guessing +3. vulnerable_variables: Extract variable names from: + - Removed lines that are key to the vulnerability + - Variables/buffers explicitly mentioned in VENDOR_MITIGATIONS (e.g., "sum2 buffer") +4. vulnerable_patterns: Extract distinctive code snippets from removed lines (- lines) + - Focus on patterns that can be grepped + - Include enough context to be unique +5. fix_patterns: Extract distinctive code snippets from added lines (+ lines) + - These indicate the fix is present + - MUST be actual CODE patterns that can be grepped in source files + NO-PATCH RULE: If no patch is provided, leave fix_patterns EMPTY. + - Do NOT substitute version strings (e.g., "Apache HTTP Server 2.4.68") - these do not appear in source code + - Do NOT guess fix patterns from CVE description prose + - Empty fix_patterns is valid and expected when no patch is available +6. root_cause: Explain WHY the code is vulnerable in 1-2 sentences + - Incorporate insights from VENDOR_MITIGATIONS if provided +7. vulnerability_type: Classify as one of: buffer_overflow, integer_overflow, use_after_free, + null_deref, format_string, race_condition, path_traversal, injection, uninitialized_memory, other +8. search_keywords: List 3-5 grep patterns ordered by specificity: + - Start with most specific (unique variable/function names from patch or mitigations) + - End with broader patterns (file names, component names) +9. component_names: Extract module or component names ONLY if explicitly mentioned in CVE description or advisory. + - ONLY include names that appear verbatim (e.g., "mod_http2", "mod_proxy_http2", "libxml2_sax") + - Do NOT infer component names from context (e.g., do NOT guess "mod_ssl" just because CVE mentions TLS) + - Do NOT include the package name itself (e.g., do NOT add "httpd" for an httpd package CVE) + - If CVE mentions a broad component (like "kernel" or "glibc"), leave empty - too generic to help file discovery + - Examples of GOOD extractions: ["mod_http2", "mod_proxy_http2"], ["nf_tables"], ["bpf_verifier"] + - Examples of BAD extractions: ["httpd"] (package name), ["network"] (too vague), ["memory"] (not a component) +10. affected_bitness: Determine which bitness is affected: + - "32-bit": Look for "32-bit systems", "i386", "i686", "on 32-bit" + - "64-bit": Look for "64-bit only", "x86_64 only" (rare) + - "both": DEFAULT when not explicitly stated + NOTE: Do NOT assume bitness based on the vulnerability type. Default to "both" unless explicitly stated. +11. affected_architectures: Determine which CPU families are affected (or null for all): + - Look for: "x86", "Intel", "AMD" -> ["x86"] + - Look for: "ARM", "aarch64", "arm64" -> ["arm"] + - Look for: "PowerPC", "POWER", "ppc64" -> ["ppc"] + - Look for: "s390", "z/Architecture", "IBM Z" -> ["s390"] + - If none mentioned or "all architectures" -> null (affects all) + NOTE: Most CVEs affect all architectures. Only extract specific families if explicitly mentioned. +12. known_mitigations: Copy vendor-provided mitigations verbatim if present (e.g., compiler flags like "-ftrivial-auto-var-init=zero", configuration changes). Empty if none provided. + + + +- If no patch is provided, extract what you can from the CVE description and VENDOR_MITIGATIONS +- VENDOR_MITIGATIONS often contain specific variable names, buffers, or compiler flags that are highly relevant +- For search_keywords, prefer identifiers over natural language +- Patterns should be grep-friendly (avoid regex special chars unless escaped) + +""" + +# --------------------------------------------------------------------------- +# L1 System Prompts +# --------------------------------------------------------------------------- + +L1_AGENT_SYS_PROMPT_PATCH_AVAILABLE = ( + "You are a security analyst investigating whether a CVE fix has been applied to a package.\n" + "A downstream patch file exists and has been analyzed.\n\n" + "VULNERABILITY_INTEL contains DOWNSTREAM_PATCH_STATUS and extracted patterns from the patch.\n" + "The source code index contains the UNPATCHED tarball; the patch is applied at BUILD time.\n\n" + "YOUR TASK: Verify (1) vulnerable code exists in source, (2) fix pattern is absent.\n" + "Both outcomes are EXPECTED when DOWNSTREAM_PATCH_STATUS is APPLIED.\n\n" + "CRITICAL RULES:\n" + "- If DOWNSTREAM_PATCH_STATUS is APPLIED, the package is PATCHED (patch applied at build time).\n" + "- Finding vulnerable code in source is EXPECTED (source is unpatched tarball).\n" + "- NOT finding fix pattern in source is EXPECTED (fix is in patch file, not tarball).\n" + "- Both findings together confirm the patch will correctly fix the code at build time.\n\n" + "ANSWER QUALITY:\n" + "- Cite specific file paths and line numbers from tool results.\n" + "- Quote the actual code found, not just describe it.\n" + "- Confirm the patch addresses the vulnerable code found.\n" + "- State confidence level based on evidence quality." +) + +L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH = ( + "You are a security analyst verifying that a package is VULNERABLE to a CVE.\n" + "The TARGET package does NOT contain a CVE-specific patch file.\n" + "However, patterns have been extracted from the patch in a FIXED RPM version.\n\n" + "VULNERABILITY_INTEL contains patterns extracted from the fixed version's patch.\n\n" + "YOUR TASK: Verify the TARGET package contains the vulnerable code and LACKS the fix.\n\n" + "VERIFICATION STRATEGY:\n" + "1. FIRST search for the VULNERABLE code pattern (from VULNERABLE_PATTERNS).\n" + " - Use function names, variable names, or unique code snippets.\n" + " - The vulnerable code SHOULD exist in the target package.\n" + "2. If vulnerable code is found, search for the FIX code pattern (from FIX_PATTERNS).\n" + " - The fix code should NOT exist in the target package.\n" + "3. CONCLUSION:\n" + " - If vulnerable code EXISTS and fix is ABSENT → Package is VULNERABLE.\n" + " - If fix code IS found → Package may be patched via rebase (investigate further).\n" + " - If neither is found → Use file paths from AFFECTED_FILES to locate relevant code.\n\n" + "CRITICAL RULES:\n" + "- The patch is from a FIXED version - expect the target to have vulnerable code.\n" + "- Use file paths and function names from VULNERABILITY_INTEL to locate code.\n" + "- Search for distinctive code patterns, not generic keywords.\n" + "- Base conclusions ONLY on tool results, not assumptions.\n\n" + "ANSWER QUALITY:\n" + "- Cite specific file paths and line numbers from tool results.\n" + "- Quote the actual code found, not just describe it.\n" + "- Compare found code against both vulnerable and fix patterns.\n" + "- Clearly state whether vulnerable code exists and whether fix is absent.\n" + "- State confidence level based on evidence quality." +) + +L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH_CASE_B = ( + "You are a security analyst verifying whether a package remains VULNERABLE to a CVE.\n" + "The TARGET package does NOT contain a CVE-specific patch file.\n" + "Patterns were extracted from a FIXED RPM patch, but VULNERABLE_PATTERNS is EMPTY.\n\n" + "YOUR TASK: Determine whether the FIX is present in the target source.\n\n" + "VERIFICATION STRATEGY (CASE B - EMPTY VULNERABLE_PATTERNS):\n" + "1. FIRST search FIX_PATTERNS inside each file from AFFECTED_FILES.\n" + "2. If a fix marker is found, verify fix application at the call site.\n" + " - Definition-only matches are insufficient.\n" + " - The fix must be called and its result must be used.\n" + "3. CONCLUSION:\n" + " - If fix is applied at call site → Package is PATCHED via rebase.\n" + " - If fix is absent or unverified → investigate remaining affected files and use version fallback rules.\n\n" + "CRITICAL RULES:\n" + "- Do NOT start with vulnerable-pattern search in this mode.\n" + "- Do NOT assume vulnerability or safety without tool evidence.\n" + "- Base conclusions ONLY on tool results and version-range evidence when required.\n\n" + "ANSWER QUALITY:\n" + "- Cite specific file paths and line numbers from tool results.\n" + "- Quote the actual code found, not just describe it.\n" + "- Clearly state whether fix is present at call site, absent, or unverified.\n" + "- State confidence level based on evidence quality." +) + +L1_AGENT_SYS_PROMPT_REBASE_FIX = ( + "You are a security analyst verifying that a CVE fix is PRESENT in a rebased package.\n" + "The TARGET package was REBASED to a newer upstream version that claims to fix this CVE.\n\n" + "VULNERABILITY_INTEL contains patterns extracted from the upstream fix.\n\n" + "YOUR TASK: Verify the TARGET package contains the FIX code (proving rebase was effective).\n\n" + "VERIFICATION STRATEGY:\n" + "1. FIRST search for the FIX code pattern (from FIX_PATTERNS).\n" + " - Use function names, variable names, or unique code snippets.\n" + " - The fix code SHOULD exist in the target package (proving rebase worked).\n" + "2. If fix code is found, optionally confirm VULNERABLE code is ABSENT.\n" + " - The vulnerable code should NOT exist (was replaced by the fix).\n" + "3. CONCLUSION:\n" + " - If fix code EXISTS → Package is PATCHED via rebase.\n" + " - If vulnerable code still EXISTS and fix is ABSENT → Rebase may be incomplete.\n" + " - If neither is found → Use file paths from AFFECTED_FILES to locate relevant code.\n\n" + "CRITICAL RULES:\n" + "- The patch is from a FIXED version - expect the target to have the fix code.\n" + "- Use file paths and function names from VULNERABILITY_INTEL to locate code.\n" + "- Search for distinctive code patterns, not generic keywords.\n" + "- Base conclusions ONLY on tool results, not assumptions.\n\n" + "ANSWER QUALITY:\n" + "- Cite specific file paths and line numbers from tool results.\n" + "- Quote the actual code found, not just describe it.\n" + "- Compare found code against both vulnerable and fix patterns.\n" + "- Clearly state whether fix code exists, confirming the rebase.\n" + "- State confidence level based on evidence quality." +) + +L1_AGENT_SYS_PROMPT_REBASE_NO_PATCH = """You are a security analyst verifying that a CVE fix is PRESENT in a rebased package. +The TARGET package was REBASED to a newer upstream version that claims to fix this CVE. +NO PATCH FILE IS AVAILABLE - you must use the CVE description to guide your search. + +YOUR TASK: Verify the TARGET package contains the fix by searching for: +1. Code patterns mentioned in the CVE description +2. Defensive code that would mitigate the vulnerability +3. Function/symbol names related to the CVE + +VERIFICATION STRATEGY (No Patch Mode): +1. EXTRACT key identifiers from the CVE description: + - Function names, API calls, variable names + - Vulnerable code constructs described + - Fixed/secure code patterns described +2. SEARCH for these patterns in the target source code +3. ANALYZE the code to determine if it shows the fix behavior +4. CONCLUDE based on presence of defensive code and absence of vulnerability indicators""" + +# --------------------------------------------------------------------------- +# L1 Prompt Templates +# --------------------------------------------------------------------------- + +L1_AGENT_PROMPT_TEMPLATE = """{sys_prompt} + + +CVE ID: {vuln_id} +Target Package: {target_package} + + + +{vulnerability_intel} + + + +{tools} + + + +{tool_selection_strategy} + + +{tool_instructions} + +RESPONSE: +{{""" + +L1_AGENT_PROMPT_TEMPLATE_NO_PATCH = """{sys_prompt} + + +CVE ID: {vuln_id} +Target Package: {target_package} + + + +{vulnerability_intel} + + + +{tools} + + +{tool_selection_strategy} + +{tool_instructions}""" + +# --------------------------------------------------------------------------- +# L1 Thought Instructions +# --------------------------------------------------------------------------- + +L1_AGENT_THOUGHT_INSTRUCTIONS = """ +You will receive KNOWLEDGE (cumulative findings) and LATEST FINDINGS (most recent tool results). +BEFORE ACTING, you MUST: +1. Review KNOWLEDGE to see what tools were already called (TOOL_CALL_RECORD entries) +2. Review LATEST FINDINGS for the most recent tool output analysis +3. NEVER repeat any action already in TOOL_CALL_RECORD +4. Your next action MUST build on findings - progress the investigation + + + +PHASE 0 - CHECK PATCH STATUS (PRIORITY): + FIRST check VULNERABILITY_INTEL for DOWNSTREAM_PATCH_STATUS. + If DOWNSTREAM_PATCH_STATUS is APPLIED: + - The source code index contains the UNPATCHED tarball + - The patch file is applied at BUILD time, not in the indexed source + - Do 2 verification searches, then FINISH with verdict PATCHED + +PHASE 1 - INTELLIGENCE (PRE-COMPLETED): + Review VULNERABILITY_INTEL above. It contains: + - DOWNSTREAM_PATCH_STATUS: APPLIED means package is patched at build time + - PATCH_FILE: Name of the patch file + - AFFECTED_FILES: Files to verify + - VULNERABLE_FUNCTIONS: Functions to search for + - VULNERABLE_PATTERNS: Code patterns indicating vulnerability + - FIX_PATTERNS: Code patterns indicating the fix (will be ABSENT in source) + +PHASE 2 - SOURCE CODE INSPECTION (when DOWNSTREAM_PATCH_STATUS is APPLIED): + Do exactly 2 verification searches: + 1. Search for vulnerable function/pattern → should FIND it (source is unpatched) + 2. Search for fix pattern → should NOT find it (fix is in separate patch file) + Both outcomes are EXPECTED and confirm the patch is correct. + After both searches, FINISH immediately with PATCHED verdict. + +PHASE 3 - VERDICT: + If DOWNSTREAM_PATCH_STATUS is APPLIED: + - Found vulnerable code + fix absent = PATCHED (patch will fix it at build time) + - This is the EXPECTED outcome, not a failure + Conclude after 2 searches - do NOT keep searching. + + + +1. You MUST select a tool ONLY from . Do NOT invent or use any other tool names. +2. Output valid JSON only. thought < 100 words. final_answer < 150 words. +3. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. +4. If DOWNSTREAM_PATCH_STATUS is APPLIED, do max 2 searches then conclude PATCHED. +5. Do NOT call the same tool with the same input twice - CHECK KNOWLEDGE for prior calls. +6. When patch is APPLIED: finding vulnerable code = GOOD, not finding fix = GOOD (expected). +7. If a pattern contains special regex characters, escape them or use literal substrings. + + + +If DOWNSTREAM_PATCH_STATUS is APPLIED: +- Search 1: Find vulnerable function → EXPECTED to find (source is unpatched) +- Search 2: Check fix pattern → EXPECTED to NOT find (fix is in patch file) +- After both: FINISH with PATCHED verdict +If a search returned results: +- If vulnerable code found and patch is APPLIED, proceed to verify fix is absent +- After both checks complete, FINISH +If a pattern wasn't found: +- Try simpler substrings or partial patterns +- Try a different tool (Source Grep <-> Code Keyword Search) + + + +{{"thought": "DOWNSTREAM_PATCH_STATUS is APPLIED. Search for vulnerable function first", "mode": "act", "actions": {{"tool": "Source Grep", "query": "parse_rockridge", "reason": "Verify vulnerable function exists in unpatched source"}}, "final_answer": null}} + + +{{"thought": "Found vulnerable function. Now verify fix pattern is absent (expected since fix is in patch file)", "mode": "act", "actions": {{"tool": "Source Grep", "query": "if (file->pz_log2_bs < 15", "reason": "Confirm fix pattern is absent from source"}}, "final_answer": null}} + + +{{"thought": "Vulnerable code found, fix absent as expected. DOWNSTREAM_PATCH_STATUS is APPLIED so package is PATCHED.", "mode": "finish", "actions": null, "final_answer": "The package is PATCHED. Found vulnerable function at file.c:123. Fix pattern absent from source (expected - fix is in patch file applied at build time). DOWNSTREAM_PATCH_STATUS confirms patch is applied."}} + + +{{"thought": "KNOWLEDGE has sufficient evidence: vulnerable code at X, fix absent", "mode": "finish", "actions": null, "final_answer": "The package is [PATCHED/VULNERABLE]. Found [evidence] at [file:line]. The code [matches/differs from] the patch because [reason]."}} +""" + +L1_AGENT_THOUGHT_UPSTREAM_INSTRUCTIONS = """ +You will receive KNOWLEDGE (cumulative findings) and LATEST FINDINGS (most recent tool results). +BEFORE ACTING, you MUST: +1. Review KNOWLEDGE to see what tools were already called (TOOL_CALL_RECORD entries) +2. Review LATEST FINDINGS for the most recent tool output analysis +3. NEVER repeat any action already in TOOL_CALL_RECORD +4. Your next action MUST build on findings - progress the investigation + + + +PHASE 1 - INTELLIGENCE (PRE-COMPLETED): + Review VULNERABILITY_INTEL above. It contains: + - AFFECTED_FILES: Files to verify + - VULNERABLE_FUNCTIONS: Functions to search for + - VULNERABLE_PATTERNS: Code patterns indicating vulnerability + - FIX_PATTERNS: Code patterns indicating the fix + - SEARCH_KEYWORDS: Terms to grep for + +PHASE 2 - SOURCE CODE INSPECTION (YOUR TASK): + CASE A - VULNERABLE_PATTERNS is NOT empty: + 1. Search for vulnerable pattern - it SHOULD exist in unpatched target + 2. Search for fix pattern - it should NOT exist in unpatched target + + CASE B - VULNERABLE_PATTERNS is empty: + 1. Search for FIX_PATTERNS in AFFECTED_FILES first + 2. Verify fix is applied at call site (not definition-only) + 3. Do NOT force a vulnerable-pattern search in this case + IMPORTANT: Do NOT stop after finding the first file. Check ALL AFFECTED_FILES. + +PHASE 3 - VERDICT: + Only conclude when: + - ALL AFFECTED_FILES have been searched + - ALL VULNERABLE_FUNCTIONS have been located + - Evidence is sufficient for confident verdict + + **VERSION-BASED FALLBACK (when code search is inconclusive):** + If your code searches found NO conclusive evidence: + - CASE A: no vulnerable pattern and no fix pattern found + - CASE B: no fix pattern or no fix call-site verification found + - Check TARGET_IN_VULNERABLE_RANGE in VULNERABILITY_INTEL + - If TARGET_IN_VULNERABLE_RANGE: YES and no fix was verified in code: + → Conclude VULNERABLE based on version evidence + → Reason: "Target version is within affected range, and no fix was verified in code." + - The absence of a verified fix, combined with version range membership, + is strong evidence of vulnerability. + + + +1. You MUST select a tool ONLY from . Do NOT invent or use any other tool names. +2. Output valid JSON only. thought < 100 words. final_answer < 150 words. +3. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. +4. Source Grep: use query field with pattern from VULNERABILITY_INTEL (function name, variable, or code snippet). +5. Code Keyword Search: use query field for broader searches. +6. Do NOT call the same tool with the same input twice - CHECK KNOWLEDGE for prior calls. +7. IF VULNERABLE_PATTERNS is non-empty: search VULNERABLE code first. +8. IF VULNERABLE_PATTERNS is empty: search FIX_PATTERNS first in AFFECTED_FILES. +9. Do NOT classify as vulnerable solely because VULNERABLE_PATTERNS is empty. +10. If a pattern contains special regex characters, escape them or use literal substrings. +11. Before PATCHED: verify FIX_APPLIED_AT_CALL_SITE in ALL AFFECTED_FILES. FIX_DEFINITION_FOUND alone is insufficient. +12. Fix must be CALLED and result USED (assigned/in condition). Called but unused = not applied. + + + +If a search returned results: +- Narrow down by searching within that specific file (e.g., "pattern,filename.c") +- Search for related symbols or variables from the code found +If a pattern wasn't found: +- Try simpler substrings or partial patterns +- Try a different tool (Source Grep <-> Code Keyword Search) +- Search for file paths from VULNERABILITY_INTEL AFFECTED_FILES +If KNOWLEDGE shows partial evidence: +- Investigate other files mentioned in VULNERABILITY_INTEL AFFECTED_FILES +- Search for key variables from the fix pattern +If FIX_DEFINITION_FOUND: search AFFECTED_FILES for actual usage before concluding PATCHED. + + + +{{"thought": "No prior searches in KNOWLEDGE. Search for the vulnerable code pattern from the patch", "mode": "act", "actions": {{"tool": "Source Grep", "query": "", "reason": "Locate vulnerable code that should exist in unpatched target"}}, "final_answer": null}} + + +{{"thought": "VULNERABLE_PATTERNS is empty. Start with FIX_PATTERNS in affected files.", "mode": "act", "actions": {{"tool": "Source Grep", "query": ",", "reason": "CASE B: verify additive fix markers are present at call site"}}, "final_answer": null}} + + +{{"thought": "KNOWLEDGE shows function found at iso9660.c:2074. Now verify the fix is NOT present", "mode": "act", "actions": {{"tool": "Source Grep", "query": "", "reason": "Check if fix code is absent (confirms vulnerability)"}}, "final_answer": null}} + + +{{"thought": "KNOWLEDGE shows fix pattern not found but need more evidence. Search for key variable in the found file", "mode": "act", "actions": {{"tool": "Source Grep", "query": ",", "reason": "Examine how the vulnerable variable is handled"}}, "final_answer": null}} + + +{{"thought": "KNOWLEDGE shows Source Grep failed. Try Code Keyword Search for the file from patch", "mode": "act", "actions": {{"tool": "Code Keyword Search", "query": "", "reason": "Verify we are looking at the correct file"}}, "final_answer": null}} + + +{{"thought": "Vulnerable code found, fix absent", "mode": "finish", "actions": null, "final_answer": "VULNERABLE. Found vulnerable code at [file:line]. Fix not present."}} + + +{{"thought": "FIX_APPLIED_AT_CALL_SITE found despite no CVE patch", "mode": "finish", "actions": null, "final_answer": "PATCHED via rebase. Fix applied at [file:line]. Included via upstream update."}} + + +{{"thought": "FIX_DEFINITION_FOUND but no call site evidence", "mode": "finish", "actions": null, "final_answer": "UNCERTAIN - fix function exists at [file:line] but usage in AFFECTED_FILES unverified. Manual review required."}} + + +{{"thought": "VULNERABLE_PATTERNS is empty and fix application is unverified", "mode": "finish", "actions": null, "final_answer": "UNCERTAIN - vulnerable patterns were unavailable and fix call-site evidence is insufficient. Manual review required."}} + + +{{"thought": "Code searches found no vulnerable or fix patterns - affected module not located in source. However, TARGET_IN_VULNERABLE_RANGE is YES. No fix was verified in code. Version evidence indicates vulnerability.", "mode": "finish", "actions": null, "final_answer": "VULNERABLE (version-based). Target version is within the affected range per VULNERABILITY_INTEL. Code search could not locate the affected module or verify a fix. Based on version evidence and absence of verified fix, the package is vulnerable."}} +""" + +L1_AGENT_THOUGHT_UPSTREAM_CASE_B_INSTRUCTIONS = """ +You will receive KNOWLEDGE (cumulative findings) and LATEST FINDINGS (most recent tool results). +BEFORE ACTING, you MUST: +1. Review KNOWLEDGE to see what tools were already called (TOOL_CALL_RECORD entries) +2. Review LATEST FINDINGS for the most recent tool output analysis +3. NEVER repeat any action already in TOOL_CALL_RECORD +4. Your next action MUST build on findings - progress the investigation + + + +PHASE 1 - INTELLIGENCE (PRE-COMPLETED): + Review VULNERABILITY_INTEL above. It contains: + - AFFECTED_FILES: Files to verify + - FIX_PATTERNS: Code patterns indicating the fix + - SEARCH_KEYWORDS: Terms to grep for + +PHASE 2 - SOURCE CODE INSPECTION (CASE B ONLY): + VULNERABLE_PATTERNS is empty: + 1. Search FIX_PATTERNS in AFFECTED_FILES first + 2. Verify fix is applied at call site (not definition-only) + 3. Do NOT force a vulnerable-pattern search in this case + IMPORTANT: Do NOT stop after finding the first file. Check ALL AFFECTED_FILES. + +PHASE 3 - VERDICT: + Only conclude when: + - ALL AFFECTED_FILES have been searched + - Evidence is sufficient for confident verdict + + **VERSION-BASED FALLBACK (when code search is inconclusive):** + If code searches found no fix pattern or no fix call-site verification: + - Check TARGET_IN_VULNERABLE_RANGE in VULNERABILITY_INTEL + - If TARGET_IN_VULNERABLE_RANGE: YES and no fix was verified in code: + → Conclude VULNERABLE based on version evidence + → Reason: "Target version is within affected range, and no fix was verified in code." + + + +1. You MUST select a tool ONLY from . Do NOT invent or use any other tool names. +2. Output valid JSON only. thought < 100 words. final_answer < 150 words. +3. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. +4. Source Grep: use query field with pattern from VULNERABILITY_INTEL (function name, variable, or code snippet). +5. Code Keyword Search: use query field for broader searches. +6. Do NOT call the same tool with the same input twice - CHECK KNOWLEDGE for prior calls. +7. FIRST action in this mode MUST search FIX_PATTERNS in AFFECTED_FILES. +8. Do NOT perform vulnerable-pattern-first search in this mode. +9. If a pattern contains special regex characters, escape them or use literal substrings. +10. Before PATCHED: verify FIX_APPLIED_AT_CALL_SITE in ALL AFFECTED_FILES. FIX_DEFINITION_FOUND alone is insufficient. +11. Fix must be CALLED and result USED (assigned/in condition). Called but unused = not applied. + + + +If a search returned results: +- Narrow down by searching within that specific file (e.g., "pattern,filename.c") +- Search for related symbols or variables from the code found +If a pattern wasn't found: +- Try simpler substrings or partial patterns +- Try a different tool (Source Grep <-> Code Keyword Search) +- Search for file paths from VULNERABILITY_INTEL AFFECTED_FILES +If KNOWLEDGE shows partial evidence: +- Investigate other files mentioned in VULNERABILITY_INTEL AFFECTED_FILES +- Search for key variables from the fix pattern +If FIX_DEFINITION_FOUND: search AFFECTED_FILES for actual usage before concluding PATCHED. + + + +{{"thought": "No prior searches in KNOWLEDGE. CASE B requires fix-first verification in affected file.", "mode": "act", "actions": {{"tool": "Source Grep", "query": ",", "reason": "CASE B: verify whether fix marker exists in affected file"}}, "final_answer": null}} + + +{{"thought": "Fix marker search completed for one file. Continue to remaining affected files and verify call-site usage where found.", "mode": "act", "actions": {{"tool": "Source Grep", "query": ",", "reason": "Ensure all affected files are checked before verdict"}}, "final_answer": null}} + + +{{"thought": "Fix was verified at call site in affected code", "mode": "finish", "actions": null, "final_answer": "PATCHED via rebase. Fix pattern is present and applied at call site in affected files."}} + + +{{"thought": "Fix not verified in code and target remains in affected version range", "mode": "finish", "actions": null, "final_answer": "VULNERABLE (version-based). Target version is within the affected range and no fix was verified at call site in affected files."}} +""" + +def select_upstream_prompt_and_instructions(vulnerable_patterns: list[str]) -> tuple[str, str]: + """Select upstream patch prompts based on vulnerable-pattern availability.""" + if vulnerable_patterns: + return L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH, L1_AGENT_THOUGHT_UPSTREAM_INSTRUCTIONS + return L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH_CASE_B, L1_AGENT_THOUGHT_UPSTREAM_CASE_B_INSTRUCTIONS + +L1_AGENT_THOUGHT_REBASE_INSTRUCTIONS = """ +You will receive KNOWLEDGE (cumulative findings) and LATEST FINDINGS (most recent tool results). +BEFORE ACTING, you MUST: +1. Review KNOWLEDGE to see what tools were already called (TOOL_CALL_RECORD entries) +2. Review LATEST FINDINGS for the most recent tool output analysis +3. NEVER repeat any action already in TOOL_CALL_RECORD +4. Your next action MUST build on findings - progress the investigation + + + +PHASE 1 - INTELLIGENCE (PRE-COMPLETED): + Review VULNERABILITY_INTEL above. It contains: + - AFFECTED_FILES: Files to verify + - VULNERABLE_FUNCTIONS: Functions to search for + - VULNERABLE_PATTERNS: Code patterns indicating vulnerability + - FIX_PATTERNS: Code patterns indicating the fix + - SEARCH_KEYWORDS: Terms to grep for + +PHASE 2 - SOURCE CODE INSPECTION (YOUR TASK): + For EACH item in VULNERABLE_FUNCTIONS and AFFECTED_FILES: + 1. Search for fix pattern - it SHOULD exist in rebased target + 2. Verify vulnerable pattern is ABSENT from target + IMPORTANT: Do NOT stop after finding the first file. Check ALL AFFECTED_FILES. + +PHASE 3 - VERDICT: + Only conclude when: + - ALL AFFECTED_FILES have been searched + - ALL VULNERABLE_FUNCTIONS have been located + - Evidence is sufficient for confident verdict + + + +1. You MUST select a tool ONLY from . Do NOT invent or use any other tool names. +2. Output valid JSON only. thought < 100 words. final_answer < 150 words. +3. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. +4. Source Grep: use query field with pattern from VULNERABILITY_INTEL (function name, variable, or code snippet). +5. Code Keyword Search: use query field for broader searches. +6. Do NOT call the same tool with the same input twice - CHECK KNOWLEDGE for prior calls. +7. FIRST search for FIX code - it SHOULD exist in rebased target. +8. THEN verify VULNERABLE code is ABSENT from target. +9. If a pattern contains special regex characters, escape them or use literal substrings. +10. Before PATCHED: verify FIX_APPLIED_AT_CALL_SITE in ALL AFFECTED_FILES. FIX_DEFINITION_FOUND alone is insufficient. +11. Fix must be CALLED and result USED (assigned/in condition). Called but unused = not applied. + + + +If a search returned results: +- Narrow down by searching within that specific file (e.g., "pattern,filename.c") +- Search for related symbols or variables from the code found +If a pattern wasn't found: +- Try simpler substrings or partial patterns +- Try a different tool (Source Grep <-> Code Keyword Search) +- Search for file paths from VULNERABILITY_INTEL AFFECTED_FILES +If KNOWLEDGE shows partial evidence: +- Investigate other files mentioned in VULNERABILITY_INTEL AFFECTED_FILES +- Search for key variables from the fix pattern +If FIX_DEFINITION_FOUND: search AFFECTED_FILES for actual usage before concluding PATCHED. + + + +{{"thought": "No prior searches in KNOWLEDGE. Search for the fix code pattern from the patch", "mode": "act", "actions": {{"tool": "Source Grep", "query": "", "reason": "Locate fix code that should exist after rebase"}}, "final_answer": null}} + + +{{"thought": "KNOWLEDGE shows fix pattern not found. Try searching for key variable from the fix", "mode": "act", "actions": {{"tool": "Source Grep", "query": "", "reason": "Find how the fix-related variable is handled"}}, "final_answer": null}} + + +{{"thought": "KNOWLEDGE shows variable found at file.c:100. Search for the full fix pattern in that file", "mode": "act", "actions": {{"tool": "Source Grep", "query": ",file.c", "reason": "Check if fix exists in the located file"}}, "final_answer": null}} + + +{{"thought": "KNOWLEDGE shows fix confirmed. Now verify the vulnerable code is absent", "mode": "act", "actions": {{"tool": "Source Grep", "query": "", "reason": "Check if vulnerable code was removed (confirms fix)"}}, "final_answer": null}} + + +{{"thought": "FIX_APPLIED_AT_CALL_SITE confirmed, vulnerable code absent", "mode": "finish", "actions": null, "final_answer": "PATCHED via rebase. Fix at [file:line]. Vulnerable code absent."}} + + +{{"thought": "Vulnerable code still present despite rebase", "mode": "finish", "actions": null, "final_answer": "INCOMPLETE rebase. Vulnerable code at [file:line]. Manual review required."}} + + +{{"thought": "FIX_DEFINITION_FOUND but no call site evidence", "mode": "finish", "actions": null, "final_answer": "UNCERTAIN - fix function exists at [file:line] but usage in AFFECTED_FILES unverified. Manual review required."}} +""" + +L1_AGENT_THOUGHT_CVE_DESC_INSTRUCTIONS = """ +You will receive KNOWLEDGE (cumulative findings) and LATEST FINDINGS (most recent tool results). +BEFORE ACTING, you MUST: +1. Review KNOWLEDGE to see what tools were already called (TOOL_CALL_RECORD entries) +2. Review LATEST FINDINGS for the most recent tool output analysis +3. NEVER repeat any action already in TOOL_CALL_RECORD +4. Your next action MUST build on findings - progress the investigation + + + +PHASE 1 - INTELLIGENCE (PRE-COMPLETED): + Review VULNERABILITY_INTEL above. It contains: + - COMPONENT_NAMES: Module/component names to use for file discovery (PRIORITY - use these FIRST) + - AFFECTED_FILES: Files to verify (may be empty in no-patch mode) + - VULNERABLE_FUNCTIONS: Functions to search for + - VULNERABLE_PATTERNS: Code patterns indicating vulnerability + - FIX_PATTERNS: Code patterns indicating the fix + - SEARCH_KEYWORDS: Terms to grep for (includes COMPONENT_NAMES at start) + - ROOT_CAUSE: Description of the vulnerability mechanism + +PHASE 2 - SOURCE CODE INSPECTION (YOUR TASK): + **STEP 1 - FILE DISCOVERY (DO THIS FIRST):** + If COMPONENT_NAMES is available (e.g., "mod_http2", "nf_tables"): + - Search for files containing the component name: "mod_http2" or "mod_http2.c" + - This locates the relevant source files for the CVE + - Record discovered file paths for STEP 2 + + If COMPONENT_NAMES is empty or search returns no results: + - Use SEARCH_KEYWORDS to find related files + - Try function names from VULNERABLE_FUNCTIONS + - Try combinations like ".c" or header file names + + **STEP 2 - CODE ANALYSIS (AFTER finding files):** + Once you have found relevant files from STEP 1: + - Search WITHIN those files for VULNERABLE_PATTERNS + - Search WITHIN those files for FIX_PATTERNS or defensive code + - Use file-scoped searches: "pattern,discovered_file.c" + + IMPORTANT: Do NOT skip STEP 1. File discovery FIRST, then code analysis. + +PHASE 3 - VERDICT: + Only conclude when: + - Relevant files have been discovered (STEP 1 complete) + - Vulnerable/fix patterns searched within those files (STEP 2 complete) + - Evidence is sufficient for confident verdict + + **VERSION-BASED FALLBACK (when code search is inconclusive):** + If your code searches found NO conclusive evidence (no vulnerable pattern, no fix pattern): + - Check TARGET_IN_VULNERABLE_RANGE in VULNERABILITY_INTEL + - If TARGET_IN_VULNERABLE_RANGE: YES and no fix was verified in code: + → Conclude VULNERABLE based on version evidence + → Reason: "Target version is within affected range, and no fix was verified in code." + - The absence of a verified fix, combined with version range membership, + is strong evidence of vulnerability. + + +RESPONSE FORMAT (JSON): +You must respond with a JSON object with these fields: +- thought: Your reasoning based on KNOWLEDGE and VULNERABILITY_INTEL (reference what was already found) +- mode: "act" (to use a tool) or "finish" (to provide final answer) +- actions: (only if mode="act") {{"tool": "Tool Name", "query": "search term", "reason": "why this search"}} +- final_answer: (only if mode="finish") Your conclusion about patch status + + +1. Do NOT call the same tool with the same input twice - CHECK KNOWLEDGE for prior calls. +2. If KNOWLEDGE shows a search was done, your next action must be DIFFERENT. +3. Output valid JSON only. thought < 100 words. + + + +**STEP 1 - FILE DISCOVERY:** +If COMPONENT_NAMES available (e.g., "mod_http2"): +- First search: "mod_http2" to find files containing this component +- This returns file paths like modules/http2/mod_http2.c +If COMPONENT_NAMES empty or no results: +- Try SEARCH_KEYWORDS from VULNERABILITY_INTEL +- Try function names with .c suffix: "function_name.c" + +**STEP 2 - CODE ANALYSIS (after files found):** +Once you have file paths from STEP 1: +- Search WITHIN those files: "vulnerable_pattern,mod_http2.c" +- Search for fix patterns: "fix_pattern,mod_http2.c" + +**MANDATORY FALLBACK - If Source Grep returns NO MATCHES:** +When Source Grep returns empty for a function name or pattern search: +1. MUST try Code Keyword Search with semantic terms from ROOT_CAUSE or CVE description + - Example: CVE describes "XSS in HTML directory listing" → try "HTML directory" or "directory listing" +2. MUST try simpler substrings within the discovered file + - Example: instead of "mod_proxy_ftp_html_dir_list", try just "directory" or "html" in mod_proxy_ftp.c +3. Do NOT give up after 2-3 empty Source Grep searches - switch tools first! + + + +{{"thought": "COMPONENT_NAMES shows mod_http2. Start with file discovery - search for files containing this module", "mode": "act", "actions": {{"tool": "Source Grep", "query": "mod_http2", "reason": "FILE DISCOVERY: Locate source files for the mod_http2 component"}}, "final_answer": null}} + + +{{"thought": "KNOWLEDGE shows mod_http2.c found at modules/http2/. Now search within that file for vulnerable pattern", "mode": "act", "actions": {{"tool": "Source Grep", "query": "h2_stream_reset,mod_http2.c", "reason": "CODE ANALYSIS: Search for vulnerable function within discovered file"}}, "final_answer": null}} + + +{{"thought": "COMPONENT_NAMES is empty. Use SEARCH_KEYWORDS for file discovery", "mode": "act", "actions": {{"tool": "Source Grep", "query": "", "reason": "FILE DISCOVERY: Locate relevant files using CVE keywords"}}, "final_answer": null}} + + +{{"thought": "Source Grep for function name returned empty. MANDATORY FALLBACK: Try Code Keyword Search with semantic terms from CVE description (XSS in HTML directory listing)", "mode": "act", "actions": {{"tool": "Code Keyword Search", "query": "HTML directory listing", "reason": "FALLBACK: Source Grep failed, using semantic search for vulnerability-related code"}}, "final_answer": null}} + + +{{"thought": "Source Grep for specific function returned empty. Try simpler pattern within discovered file", "mode": "act", "actions": {{"tool": "Source Grep", "query": "directory,mod_proxy_ftp.c", "reason": "FALLBACK: Broader search within known-relevant file"}}, "final_answer": null}} + + +When concluding, follow this STRICT priority order: +1. If fix code found in target → LIKELY PATCHED +2. If vulnerable code found and no fix → VULNERABLE +3. If code search inconclusive AND TARGET_IN_VULNERABLE_RANGE=YES → VULNERABLE (version-based) +4. ONLY use INCONCLUSIVE when TARGET_IN_VULNERABLE_RANGE=NO or UNKNOWN +IMPORTANT: You MUST check TARGET_IN_VULNERABLE_RANGE before concluding INCONCLUSIVE. + + + +{{"thought": "KNOWLEDGE shows defensive code found in discovered file, no vulnerability indicators", "mode": "finish", "actions": null, "final_answer": "The package is LIKELY PATCHED. Found defensive code at [file:line]: [quote code]. The fix behavior described in the CVE appears to be present."}} + + +{{"thought": "Code searches found no vulnerable or fix patterns - affected module not located in source. However, TARGET_IN_VULNERABLE_RANGE is YES. No fix was verified in code. Version evidence indicates vulnerability.", "mode": "finish", "actions": null, "final_answer": "VULNERABLE (version-based). Target version is within the affected range per VULNERABILITY_INTEL. Code search could not locate the affected module or verify a fix. Based on version evidence and absence of verified fix, the package is vulnerable."}} + + +{{"thought": "KNOWLEDGE shows insufficient evidence AND TARGET_IN_VULNERABLE_RANGE is NO/UNKNOWN - cannot determine patch status from version alone", "mode": "finish", "actions": null, "final_answer": "INCONCLUSIVE. Could not find definitive evidence of fix or vulnerability, and version range is unknown. Manual review recommended."}} +""" + +# --------------------------------------------------------------------------- +# L1 Empty Result Classification Prompt +# --------------------------------------------------------------------------- + +L1_EMPTY_RESULT_CLASSIFICATION_PROMPT = """Classify an empty grep result for CVE patch verification. + +The grep search returned NO MATCHES. Determine what this means for patch verification. + +TOOL USED: {tool_used} +AGENT THOUGHT (why the search was performed): +{last_thought} + +SEARCHED PATTERN: {tool_input} + +RAW_PATCH_DIFF (for reference): +{raw_patch_diff} + +CLASSIFICATION RULES: +1. Read AGENT THOUGHT to understand what the agent was searching for +2. Determine if the agent was searching for VULNERABLE code or FIX code: + - Keywords like "verify vulnerable", "check if vulnerable", "removed", "absent" → searching for VULNERABLE code + - Keywords like "search for fix", "find fix", "added", "patched" → searching for FIX code +3. Classify the empty result: + - Searching for VULNERABLE code + NOT FOUND → VULNERABLE_CODE_ABSENT (good - fix removed it) + - Searching for FIX code + NOT FOUND → FIX_CODE_ABSENT (bad - fix not applied) + - Intent unclear → INCONCLUSIVE + +OUTPUT FORMAT (JSON - CodeFindings): +{{ + "findings": [": not found - "], + "tool_outcome": "{tool_used} [{tool_input}] -> NO MATCHES ()" +}} + +CLASSIFICATION VALUES for findings[0]: +- VULNERABLE_CODE_ABSENT: Agent searched for vulnerable code, it's not found = fix likely applied +- FIX_CODE_ABSENT: Agent searched for fix code, it's not found = fix NOT applied +- INCONCLUSIVE: Cannot determine search intent from agent thought + +EXAMPLE OUTPUTS: +If searching for vulnerable code: +{{"findings": ["VULNERABLE_CODE_ABSENT: switch (data->verdict.code & NF_VERDICT_MASK) not found - vulnerable pattern removed by fix"], "tool_outcome": "{tool_used} [pattern] -> NO MATCHES (vulnerable code absent = fix applied)"}} + +If searching for fix code: +{{"findings": ["FIX_CODE_ABSENT: safe_function() not found - fix pattern missing"], "tool_outcome": "{tool_used} [pattern] -> NO MATCHES (fix code absent = fix NOT applied)"}} + +RESPONSE: +{{""" + +# --------------------------------------------------------------------------- +# L1 Observation Prompts (Comprehension + Memory Update) +# --------------------------------------------------------------------------- + +L1_COMPREHENSION_PROMPT = """Analyze the tool output and extract key findings for CVE patch verification. +GOAL: Verify whether {vuln_id} fix is applied to {target_package} + +**CRITICAL FIRST CHECK - DO THIS BEFORE ANYTHING ELSE:** +Examine NEW OUTPUT below. If it is EMPTY, contains only whitespace, or shows an error: +- findings MUST be: ["FAILED: {tool_used} '{tool_input}' returned empty/no matches"] +- tool_outcome MUST be: "{tool_used} [{tool_input}] -> NO MATCHES" +- DO NOT fabricate, infer, or assume any results. STOP HERE. + + +{vulnerability_intel} + + + +{raw_patch_diff} + + + +If RAW_PATCH_DIFF contains content: +- Lines starting with "-" are REMOVED by the fix (vulnerable code) +- Lines starting with "+" are ADDED by the fix (patched code) + +PATTERN CLASSIFICATION (do this for EACH grep match): +1. Check if the matched pattern appears ONLY in "-" lines → VULNERABLE_ONLY pattern +2. Check if the matched pattern appears ONLY in "+" lines → FIX_ONLY pattern +3. Check if the matched pattern appears in BOTH "-" and "+" lines → AMBIGUOUS pattern (not a unique indicator) + +DECISION RULES: +- Finding a VULNERABLE_ONLY pattern in source = code is VULNERABLE (fix removed this, but it still exists) +- Finding a FIX_ONLY pattern in source = fix IS APPLIED (fix added this, and it exists) +- Finding an AMBIGUOUS pattern = INCONCLUSIVE (exists in both versions, proves nothing) +- CRITICAL: If you find a VULNERABLE_ONLY pattern, the code is VULNERABLE even if you also see AMBIGUOUS patterns nearby + +WORKED EXAMPLE - Follow this exact reasoning: +Given RAW_PATCH_DIFF: + - vulnerable_call(); + + safe_call(); + - shared_code(); + + shared_code(); + +Step 1: Grep finds 'vulnerable_call()' in source + → Check RAW_PATCH_DIFF: In "-" lines? YES. In "+" lines? NO. + → Classification: VULNERABLE_ONLY (the fix REMOVES this line) + → Correct finding: "VULNERABLE_ONLY_FOUND: vulnerable_call()" + → Meaning: Code is VULNERABLE because this line should have been removed + +Step 2: Grep finds 'shared_code()' in source + → Check RAW_PATCH_DIFF: In "-" lines? YES. In "+" lines? YES. + → Classification: AMBIGUOUS (exists in both versions) + → Correct finding: "AMBIGUOUS_PATTERN: shared_code() - not evidence" + → Meaning: This proves nothing about vulnerability status + +Step 3: Grep does NOT find 'safe_call()' in source + → This would be FIX_ONLY (only in "+" lines) + → Its absence confirms fix is NOT applied + +REMEMBER: Lines with "-" are REMOVED by fix. If you find them, the vulnerable code still exists! + +If RAW_PATCH_DIFF is empty: +- Fall back to using VULNERABLE_PATTERNS and FIX_PATTERNS from VULNERABILITY_INTEL + + +TOOL USED: {tool_used} +TOOL INPUT: {tool_input} +THOUGHT: {last_thought} +NEW OUTPUT: +{tool_output} + +**ANTI-HALLUCINATION RULES:** +1. You can ONLY report findings based on text that ACTUALLY APPEARS in NEW OUTPUT above. +2. Every finding claiming code was "found" MUST include a direct quote from NEW OUTPUT. +3. If NEW OUTPUT is empty, you CANNOT claim any code was found - report FAILED. +4. The tool_outcome MUST accurately reflect what NEW OUTPUT shows, not what you expect. + +CODE ANALYSIS RULES (only if NEW OUTPUT has content): +1. MANDATORY CLASSIFICATION - For EACH pattern found in NEW OUTPUT, classify against RAW_PATCH_DIFF: + a. Pattern appears ONLY in "-" lines → VULNERABLE_ONLY (fix removed this) + b. Pattern appears ONLY in "+" lines → FIX_ONLY (fix added this) + c. Pattern appears in BOTH "-" and "+" lines → AMBIGUOUS (exists in both versions) + +2. LABELING RULES (strict): + - VULNERABLE_ONLY → report as "VULNERABLE_ONLY_FOUND: [pattern] at [file:line]" + - FIX_ONLY → report as "FIX_ONLY_FOUND: [pattern] at [file:line]" + - AMBIGUOUS → report as "AMBIGUOUS_PATTERN: [pattern] - exists in both versions, not evidence" + - NEVER label an AMBIGUOUS pattern as "fix pattern found" or "vulnerable pattern found" + +3. DECISION PRIORITY: + - If ANY VULNERABLE_ONLY pattern is found → code is VULNERABLE (definitive) + - AMBIGUOUS patterns do NOT contradict a VULNERABLE_ONLY finding + - Only FIX_ONLY patterns (without any VULNERABLE_ONLY) indicate fix is applied + +4. Quote actual lines from NEW OUTPUT and record file:line for all matches. +5. FIX LOCATION: Tag FIX_DEFINITION_FOUND if found outside AFFECTED_FILES. Tag FIX_APPLIED_AT_CALL_SITE only when fix is CALLED and result USED in an AFFECTED_FILE. + + +Based on VULNERABILITY_INTEL above, assess investigation completeness: +- Have you searched in ALL files listed in AFFECTED_FILES? +- Have you found ALL instances of VULNERABLE_FUNCTIONS? +- Are there OTHER files containing the same vulnerable pattern? +If coverage is incomplete, note which files/functions remain unchecked. + + +OUTPUT RULES: +- findings: 2-4 observations. Each positive finding MUST quote actual content from NEW OUTPUT. +- tool_outcome: "{tool_used} [pattern] -> found in file.c:123" OR "{tool_used} [pattern] -> NO MATCHES" +RESPONSE: +{{""" + +L1_MEMORY_UPDATE_PROMPT = """Merge new findings into the CVE patch investigation memory. +GOAL: Verify whether {vuln_id} fix is applied to {target_package} +PREVIOUS MEMORY: {previous_memory} +NEW FINDINGS (from tool analysis): +{findings} +TOOL CALL RECORD: {tool_outcome} + +**CRITICAL: HANDLE FAILURES CORRECTLY** +If NEW FINDINGS contains "FAILED:" or TOOL CALL RECORD shows "NO MATCHES": +- Add the failure/no-match to memory verbatim +- Do NOT convert a failed search into a positive finding +- "NO MATCHES" for a fix pattern means FIX_CODE_ABSENT, not FIX_CODE_FOUND + +MEMORY RULES: +1. Start from PREVIOUS MEMORY. Append new facts from NEW FINDINGS. No duplicates. +2. Add TOOL CALL RECORD verbatim so future steps know what was already searched. +3. If NEW FINDINGS report a failure or no matches, record it as-is. Do NOT infer positive findings. + +PATCH VERIFICATION TRACKING: +- Vulnerable code FOUND: add "VULNERABLE_CODE_FOUND: [pattern] in [file:line]" +- Fix at definition only: add "FIX_DEFINITION_FOUND: [pattern] in [file:line]" +- Fix called+used in AFFECTED_FILE: add "FIX_APPLIED_AT_CALL_SITE: [pattern] in [file:line]" +- No vulnerable code matches: add "VULNERABLE_CODE_ABSENT: [pattern] not found" +- No fix code matches: add "FIX_CODE_ABSENT: [pattern] not found" + +VERDICT EVIDENCE: +- PATCHED: FIX_APPLIED_AT_CALL_SITE in AFFECTED_FILES or vulnerable code absent +- VULNERABLE: vulnerable code found, fix absent from call sites +- UNCERTAIN: FIX_DEFINITION_FOUND only, no call site evidence +- INCONCLUSIVE: neither pattern found, or conflicting evidence + +- results: copy the NEW FINDINGS as-is. +- memory: updated cumulative findings with search results and evidence tags. +RESPONSE: +{{""" + +# --------------------------------------------------------------------------- +# L1 Observation Prompts (CVE-Description Mode - No Patch Available) +# --------------------------------------------------------------------------- + +L1_COMPREHENSION_PROMPT_CVE_DESC = """Analyze the tool output for CVE patch verification using CVE description context. +GOAL: Verify whether {vuln_id} fix is applied to {target_package} + +**CRITICAL FIRST CHECK - DO THIS BEFORE ANYTHING ELSE:** +Examine NEW OUTPUT below. If it is EMPTY, contains only whitespace, or shows an error: +- findings MUST be: ["FAILED: {tool_used} '{tool_input}' returned empty/no matches"] +- tool_outcome MUST be: "{tool_used} [{tool_input}] -> NO MATCHES" +- DO NOT fabricate, infer, or assume any results. STOP HERE. + +CVE DESCRIPTION: +{cve_description} + +SPEC CHANGELOG (rebase info): +{spec_log_change} + +NOTE: No patch file available. Extract search terms from CVE description. + +TOOL USED: {tool_used} +TOOL INPUT: {tool_input} +THOUGHT: {last_thought} +NEW OUTPUT: +{tool_output} + +**ANTI-HALLUCINATION RULES:** +1. You can ONLY report findings based on text that ACTUALLY APPEARS in NEW OUTPUT above. +2. Every finding claiming code was "found" MUST include a direct quote from NEW OUTPUT. +3. If NEW OUTPUT is empty, you CANNOT claim any code was found - report FAILED. +4. The tool_outcome MUST accurately reflect what NEW OUTPUT shows, not what you expect. + +CODE ANALYSIS RULES (only if NEW OUTPUT has content): +1. EXTRACT key identifiers from the CVE description: + - Function names, variable names, API calls + - File paths or component names mentioned + +2. For each code match in NEW OUTPUT: + - Quote the actual line from NEW OUTPUT + - Does it relate to the vulnerability described? + - Does it show defensive patterns (bounds checking, null validation)? + - Record file path and line number as evidence + +3. DEFENSIVE PATTERNS indicating a fix: + - Input validation, bounds checking, null guards + - Resource cleanup, error handling + +OUTPUT: +- findings: 2-4 observations. Each positive finding MUST quote actual content from NEW OUTPUT. +- tool_outcome: "{tool_used} [pattern] -> found in file.c:123" OR "{tool_used} [pattern] -> NO MATCHES" +RESPONSE: +{{""" + +L1_MEMORY_UPDATE_PROMPT_CVE_DESC = """Merge findings into CVE patch investigation memory. +GOAL: Verify whether {vuln_id} fix is applied to {target_package} +MODE: CVE-description based (no patch patterns) + +PREVIOUS MEMORY: {previous_memory} +NEW FINDINGS: {findings} +TOOL CALL RECORD: {tool_outcome} + +MEMORY RULES: +1. Append new facts from NEW FINDINGS to PREVIOUS MEMORY. No duplicates. +2. Add TOOL CALL RECORD verbatim. + +CVE-BASED TRACKING: +- CVE-related code FOUND: "CVE_CODE_FOUND: [symbol] in [file:line]" +- Defensive pattern FOUND: "DEFENSIVE_CODE_FOUND: [pattern] in [file:line]" +- Search no match: "SEARCH_NO_MATCH: [pattern]" + +VERDICT (CVE-description mode): +- LIKELY_PATCHED: defensive code found, no vulnerability indicators +- LIKELY_VULNERABLE: vulnerability patterns found, no defensive code +- INCONCLUSIVE: insufficient evidence + +- results: copy the NEW FINDINGS as-is. +- memory: updated cumulative findings with evidence tags. +RESPONSE: +{{""" + + +# =========================================================================== +# L2 BUILD AGENT PROMPTS +# =========================================================================== + +# --------------------------------------------------------------------------- +# L2 Configuration Investigation Prompts +# --------------------------------------------------------------------------- + +L2_CONFIG_SYS_PROMPT = ( + "You are an L2 Build Agent investigating whether VULNERABLE CODE is DISABLED at build time.\n\n" + "GOAL: Determine if the CVE-affected feature/component is compiled into the binary.\n\n" + "EVIDENCE SOURCES:\n" + "1. BUILD_HARVEST section below - disabled/enabled features ALREADY extracted (analyze in thought)\n" + "2. Build log (searchable with 'logs:' prefix) - verify affected source files were compiled\n\n" + "INVESTIGATION FLOW:\n" + "1. Analyze BUILD_HARVEST in your thought (no tool needed - just read and decide)\n" + "2. If feature IS DISABLED -> mode='finish' with NOT_COMPILED verdict immediately\n" + "3. If feature IS ENABLED -> mode='act' with 'logs:' prefix to verify affected files in build log\n" + "4. If feature NOT listed -> mode='act' with 'logs:' prefix to search build log for compilation evidence\n\n" + "VERDICTS:\n" + "- NOT_COMPILED: Feature is disabled OR affected files not in build log\n" + "- COMPILED: Feature is enabled AND affected files are compiled\n" + "- UNKNOWN: Cannot determine from available evidence" +) + +L2_CONFIG_PROMPT_TEMPLATE = """{sys_prompt} + + +CVE ID: {vuln_id} +Target Package: {target_package} + + + +{vulnerability_intel} + +L1 Preliminary Verdict: {l1_preliminary_verdict} + + + +** CHECK THESE FIRST - No tool call needed! ** + +Disabled Features (from build log -D defines): +{disabled_features} + +Disabled Features (from spec configure flags): +{spec_disabled_features} + +Enabled Features (from build log -D defines): +{enabled_features} + +Enabled Features (from spec configure flags): +{spec_enabled_features} + +Linked Libraries (from build -l flags): +{linked_libraries} + +Built Subpackages (from spec %package): +{built_subpackages} + +Excluded Subpackages (from spec %bcond_without, ExcludeArch): +{excluded_subpackages} + +Kernel Config (CONFIG_*=y/n/m, kernel packages only): +{kernel_config} + +DECISION GUIDE: +- If CVE-affected feature in DISABLED lists -> verdict NOT_COMPILED (no tool needed) +- If CVE-affected feature in ENABLED lists -> strong signal for COMPILED (verify affected files in build log) +- If CVE-affected module in EXCLUDED subpackages -> verdict NOT_COMPILED +- If CVE-affected library NOT in linked_libraries -> check for alternative backend +- For kernel CVEs: if CONFIG_X=n in kernel_config -> NOT_COMPILED +- If lists are empty or feature not listed -> search build log with 'logs:' prefix + +IMPORTANT - Library vs Capability: +- Disabled features often name a specific LIBRARY, not the capability itself +- Example: 'ssl' disabled = OpenSSL library disabled, NOT TLS capability +- Always check if an ALTERNATIVE library for the same capability is ENABLED +- If alternative enabled, the capability IS compiled (just different implementation) +- Common patterns: ssl->nss/gnutls (TLS), zlib->zstd (compression), libidn->libidn2 (IDN) +- Check linked_libraries for actual library linkage evidence + + + +{tools} + + +{tool_instructions} + +RESPONSE: +{{""" + +L2_CONFIG_THOUGHT_INSTRUCTIONS = """ +You will receive KNOWLEDGE (cumulative findings) and LATEST FINDINGS (most recent tool results). +BEFORE ACTING, you MUST: +1. Review KNOWLEDGE to see what tools were already called (TOOL_CALL_RECORD entries) +2. Review LATEST FINDINGS for the most recent tool output analysis +3. NEVER repeat any action already in TOOL_CALL_RECORD +4. Your next action MUST build on findings - progress the investigation + + + +1. You MUST select a tool ONLY from . Do NOT invent tool names. +2. Output valid JSON only. thought < 100 words. final_answer < 150 words. +3. mode="act" REQUIRES actions with a tool. mode="finish" REQUIRES final_answer. NEVER use mode="act" with null actions. +4. Analyze BUILD_HARVEST in your thought, then immediately decide: finish (if disabled) or act with tool (to verify). +5. If feature in DISABLED lists -> check if alternative library for same capability is ENABLED first. +6. If DISABLED library has alternative ENABLED (e.g., ssl disabled but nss enabled) -> capability IS compiled. +7. If feature in ENABLED lists -> strong signal for COMPILED, verify affected files in build log. +8. If feature NOT in BUILD_HARVEST, search build log using 'logs:' prefix (e.g., 'logs:filename.c'). +9. NEVER grep source code - use 'logs:' prefix to search build logs for compilation evidence. +10. Do NOT call the same tool with the same input twice. + + + +{{"thought": "CVE affects SM2 crypto. BUILD_HARVEST shows 'sm2' in spec_disabled_features. SM2 is disabled at build time.", "mode": "finish", "actions": null, "final_answer": "NOT_COMPILED. The spec file configures with 'no-sm2' flag, which disables SM2 cryptographic functions. The vulnerable code in crypto/sm2/ is not compiled into the binary."}} + + + +{{"thought": "CVE affects zisofs. BUILD_HARVEST disabled features are empty - zisofs not disabled. Need to verify affected file was compiled.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "logs:archive_read_support_format_iso9660", "reason": "Check if affected file appears in build compilation log"}}, "final_answer": null}} + + + +{{"thought": "CVE affects LDAP. BUILD_HARVEST shows 'ldap' in spec_enabled_features. LDAP explicitly enabled - strong signal feature is compiled. Verifying affected files in build log.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "logs:ldap.c", "reason": "Verify LDAP-related files appear in build compilation log"}}, "final_answer": null}} + + + +{{"thought": "CVE affects TLS. 'ssl' in DISABLED but 'nss' in ENABLED. ssl=OpenSSL library, nss=NSS library - both provide TLS. Alternative library enabled, so TLS capability IS compiled. Verifying TLS files.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "logs:vtls", "reason": "Verify TLS abstraction layer files compiled in build log"}}, "final_answer": null}} + + + +{{"thought": "Feature not disabled. Now verify affected file crypto/sm2/sm2.c was compiled.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "logs:sm2.c", "reason": "Check if affected source file appears in build log"}}, "final_answer": null}} + + + +{{"thought": "Found no-sm2 in spec_disabled_features. SM2 code is not compiled.", "mode": "finish", "actions": null, "final_answer": "NOT_COMPILED. The spec file configures with 'no-sm2' flag, which disables SM2 cryptographic functions. The vulnerable code in crypto/sm2/ is not compiled into the binary."}} + + + +{{"thought": "SM2 not disabled. Found sm2.c compilation in build log.", "mode": "finish", "actions": null, "final_answer": "COMPILED. SM2 is not in disabled features. Build log shows 'gcc -c crypto/sm2/sm2_crypt.c -o sm2_crypt.o', confirming the vulnerable code is compiled into the binary."}} + + + +{{"thought": "Cannot find evidence either way. Affected files not in build log but feature not disabled.", "mode": "finish", "actions": null, "final_answer": "UNKNOWN. The feature is not explicitly disabled, but the affected files do not appear in the build log. Cannot determine compilation status."}} +""" + +# --------------------------------------------------------------------------- +# L2 Configuration Investigation Prompts - SPEC-ONLY MODE (no build log) +# --------------------------------------------------------------------------- + +L2_CONFIG_SPEC_ONLY_SYS_PROMPT = ( + "You are an L2 Build Agent investigating whether VULNERABLE CODE is DISABLED at build time.\n\n" + "GOAL: Determine if the CVE-affected feature/component is compiled into the binary.\n\n" + "** SPEC-ONLY MODE - No build log available **\n" + "Confidence is LOWER because we cannot verify actual compilation output.\n" + "Conclusions are based on BUILD INTENT from spec configure flags, not actual build output.\n\n" + "EVIDENCE SOURCES:\n" + "1. BUILD_HARVEST section below - disabled/enabled features from spec %build section\n" + "2. Source/build system files (CMakeLists.txt, Makefile.am, configure.ac) - searchable with Source Grep\n\n" + "INVESTIGATION FLOW:\n" + "1. Analyze BUILD_HARVEST in your thought (no tool needed - just read and decide)\n" + "2. If feature IS DISABLED -> mode='finish' with NOT_COMPILED verdict immediately\n" + "3. If feature IS ENABLED -> mode='finish' with COMPILED verdict (lower confidence)\n" + "4. If feature NOT listed -> mode='act' to search build system files (CMakeLists.txt, Makefile.am)\n\n" + "VERDICTS:\n" + "- NOT_COMPILED: Feature explicitly disabled in spec configure flags\n" + "- COMPILED: Feature not disabled (lower confidence - based on spec, not build output)\n" + "- UNKNOWN: Cannot determine from available evidence" +) + +L2_CONFIG_PROMPT_SPEC_ONLY_TEMPLATE = """{sys_prompt} + + +CVE ID: {vuln_id} +Target Package: {target_package} + + + +{vulnerability_intel} + +L1 Preliminary Verdict: {l1_preliminary_verdict} + + + +** SPEC-ONLY MODE - No build log available ** +Confidence: Lower - conclusions based on spec configure flags only + +Disabled Features (from spec configure flags): +{spec_disabled_features} + +Enabled Features (from spec configure flags): +{spec_enabled_features} + +Built Subpackages (from spec %package): +{built_subpackages} + +Excluded Subpackages (from spec %bcond_without, ExcludeArch): +{excluded_subpackages} + +Kernel Config (CONFIG_*=y/n/m, kernel packages only): +{kernel_config} + +DECISION GUIDE (spec-only): +- If CVE-affected feature in DISABLED list -> verdict NOT_COMPILED +- If CVE-affected feature in ENABLED list -> strong signal for COMPILED +- If CVE-affected module in EXCLUDED subpackages -> verdict NOT_COMPILED +- For kernel CVEs: if CONFIG_X=n in kernel_config -> NOT_COMPILED +- If feature not listed -> search build system files (CMakeLists.txt, Makefile.am, configure.ac) +- Cannot verify actual compilation - verdicts are based on BUILD INTENT, not build output + +IMPORTANT - Library vs Capability: +- Disabled features often name a specific LIBRARY, not the capability itself +- Example: 'ssl' disabled = OpenSSL library disabled, NOT TLS capability +- Always check if an ALTERNATIVE library for the same capability is ENABLED +- If alternative enabled, the capability IS compiled (just different implementation) +- Common patterns: ssl->nss/gnutls (TLS), zlib->zstd (compression), libidn->libidn2 (IDN) + + + +{tools} + + +{tool_instructions} + +RESPONSE: +{{""" + +L2_CONFIG_SPEC_ONLY_THOUGHT_INSTRUCTIONS = """ +You will receive KNOWLEDGE (cumulative findings) and LATEST FINDINGS (most recent tool results). +BEFORE ACTING, you MUST: +1. Review KNOWLEDGE to see what tools were already called (TOOL_CALL_RECORD entries) +2. Review LATEST FINDINGS for the most recent tool output analysis +3. NEVER repeat any action already in TOOL_CALL_RECORD +4. Your next action MUST build on findings - progress the investigation + + + +1. You MUST select a tool ONLY from . Do NOT invent tool names. +2. Output valid JSON only. thought < 100 words. final_answer < 150 words. +3. mode="act" REQUIRES actions with a tool. mode="finish" REQUIRES final_answer. NEVER use mode="act" with null actions. +4. Analyze BUILD_HARVEST in your thought, then immediately decide: finish (if disabled) or act with tool (to verify). +5. If feature in DISABLED lists -> check if alternative library for same capability is ENABLED first. +6. If DISABLED library has alternative ENABLED (e.g., ssl disabled but nss enabled) -> capability IS compiled. +7. If feature in ENABLED lists -> strong signal for COMPILED. Can finish or search build system files for confirmation. +8. If feature NOT in BUILD_HARVEST, search build system files (CMakeLists.txt, Makefile.am, configure.ac). +9. Use Source Grep to search source code and build system files. Do NOT use 'logs:' prefix (no build log available). +10. Do NOT call the same tool with the same input twice. + + + +{{"thought": "CVE affects SM2 crypto. BUILD_HARVEST shows 'sm2' in spec_disabled_features. SM2 is disabled at build time.", "mode": "finish", "actions": null, "final_answer": "NOT_COMPILED. The spec file configures with 'no-sm2' flag, which disables SM2 cryptographic functions. The vulnerable code in crypto/sm2/ is not compiled into the binary."}} + + + +{{"thought": "CVE affects zisofs. BUILD_HARVEST disabled features are empty - zisofs not disabled. Searching build system for zisofs configuration.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "zisofs", "reason": "Search CMakeLists.txt or Makefile.am for zisofs feature configuration"}}, "final_answer": null}} + + + +{{"thought": "CVE affects LDAP. BUILD_HARVEST shows 'ldap' in spec_enabled_features. LDAP explicitly enabled in spec - feature is compiled (lower confidence, spec-only mode).", "mode": "finish", "actions": null, "final_answer": "COMPILED. The spec file explicitly enables LDAP with '--with-ldap' configure flag. Based on spec configure flags (no build log available), the vulnerable code is likely compiled into the binary."}} + + + +{{"thought": "CVE affects TLS. 'ssl' in DISABLED but 'nss' in ENABLED. ssl=OpenSSL library, nss=NSS library - both provide TLS. Alternative library enabled, so TLS capability IS compiled.", "mode": "finish", "actions": null, "final_answer": "COMPILED. While OpenSSL (ssl) is disabled, NSS is enabled as an alternative TLS library. The TLS capability is compiled with a different implementation."}} + + + +{{"thought": "Feature not in spec flags. Searching CMakeLists.txt for feature configuration.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "option.*SM2|SM2.*OFF|SM2.*ON", "reason": "Search CMakeLists.txt for SM2 feature toggle"}}, "final_answer": null}} + + + +{{"thought": "Found no-sm2 in spec_disabled_features. SM2 code is not compiled.", "mode": "finish", "actions": null, "final_answer": "NOT_COMPILED. The spec file configures with 'no-sm2' flag, which disables SM2 cryptographic functions. The vulnerable code in crypto/sm2/ is not compiled into the binary."}} + + + +{{"thought": "SM2 not disabled in spec. No evidence of disabling in build system files either.", "mode": "finish", "actions": null, "final_answer": "COMPILED. SM2 is not in disabled features in the spec file. No build system configuration disables it. Based on spec configure flags (no build log available), the vulnerable code is likely compiled into the binary."}} + + + +{{"thought": "Cannot find evidence either way. Feature not mentioned in spec flags or build system files.", "mode": "finish", "actions": null, "final_answer": "UNKNOWN. The feature is not explicitly configured in the spec file or build system files. Cannot determine compilation status without build log evidence."}} +""" + +# --------------------------------------------------------------------------- +# L2 Hardening Investigation Prompts +# --------------------------------------------------------------------------- + +L2_HARDENING_SYS_PROMPT = ( + "You are an L2 Build Agent investigating COMPILER HARDENING mitigations.\n\n" + "GOAL: Determine if hardening flags relevant to this CVE's vulnerability class are present.\n\n" + "CONTEXT: Investigation 1 determined the vulnerable code IS compiled. Now check if\n" + "compiler/linker hardening makes exploitation significantly harder.\n\n" + "CRITICAL - CWE-SPECIFIC MATCHING:\n" + "- ONLY flags listed in EXPECTED_HARDENING can justify a MITIGATED verdict\n" + "- General hardening flags (stack protector, FORTIFY_SOURCE) do NOT mitigate all CWEs\n" + "- Example: -fstack-protector helps CWE-121 (stack overflow), NOT CWE-190 (integer overflow)\n" + "- You MUST match the EXACT flags from EXPECTED_HARDENING table to the build output\n\n" + "EVIDENCE SOURCES:\n" + "1. EXPECTED_HARDENING table (CWE-specific flags from knowledge base) - THIS IS YOUR CHECKLIST\n" + "2. Build log (searchable with 'logs:' prefix) - contains CFLAGS/CXXFLAGS/LDFLAGS definitions\n\n" + "EFFICIENT SEARCH STRATEGY:\n" + "- Search 'logs:FLAGS=' to get ALL compiler/linker flags in ONE call (matches CFLAGS=, LDFLAGS=, etc.)\n" + "- Grep supports regex OR: 'logs:CFLAGS\\|LDFLAGS' combines patterns\n" + "- Analyze the output to check for expected hardening flags - avoid multiple individual searches\n\n" + "IMPORTANT - RHEL/Fedora Specs Files:\n" + "When you see these specs files in build logs, hardening flags are IMPLICITLY enabled:\n" + "- '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1' => -fPIE (position-independent code for ASLR)\n" + "- '-specs=/usr/lib/rpm/redhat/redhat-hardened-ld' => -pie + -z now (PIE linking + BIND_NOW/Full RELRO)\n" + "These flags will NOT appear explicitly in the build log - the specs file injects them.\n" + "If you see these specs files, count the corresponding protections as PRESENT.\n\n" + "INVESTIGATION STEPS:\n" + "1. Review EXPECTED_HARDENING table - these are the ONLY flags that matter for this CWE\n" + "2. Search 'logs:FLAGS=' to get all compiler/linker flag definitions at once\n" + "3. For EACH flag in EXPECTED_HARDENING, check if present in build output\n" + "4. Verdict based ONLY on EXPECTED_HARDENING flags (ignore unrelated hardening)\n\n" + "VERDICTS:\n" + "- MITIGATED: One or more flags from EXPECTED_HARDENING are present in build\n" + "- NOT_MITIGATED: NONE of the EXPECTED_HARDENING flags are present (even if other hardening exists)\n" + "- UNKNOWN: Cannot determine from available evidence" +) + +L2_HARDENING_PROMPT_TEMPLATE = """{sys_prompt} + + +CVE ID: {vuln_id} +Target Package: {target_package} +CWE: {cwe_id} + + + +The following compiler/linker flags mitigate this vulnerability class: + +{expected_hardening_table} + + + +{tools} + + +{tool_instructions} + +RESPONSE: +{{""" + +L2_HARDENING_THOUGHT_INSTRUCTIONS = """ +You will receive KNOWLEDGE (cumulative findings) and LATEST FINDINGS (most recent tool results). +BEFORE ACTING, you MUST: +1. Review KNOWLEDGE to see what tools were already called (TOOL_CALL_RECORD entries) +2. Review LATEST FINDINGS for the most recent tool output analysis +3. NEVER repeat any action already in TOOL_CALL_RECORD +4. Your next action MUST build on findings - progress the investigation + + + +1. You MUST select a tool ONLY from . Do NOT invent tool names. +2. Output valid JSON only. thought < 100 words. final_answer < 150 words. +3. mode="act" REQUIRES actions. mode="finish" REQUIRES final_answer. +4. EFFICIENT SEARCH: First search for 'logs:FLAGS=' to get ALL compiler/linker flags in one call. +5. Grep supports regex OR: use 'logs:CFLAGS\\|LDFLAGS' to combine patterns. +6. Analyze the FLAGS output to check for expected hardening flags - avoid multiple individual searches. +7. Do NOT call the same tool with the same input twice. +8. STRICT MATCHING: Only flags from EXPECTED_HARDENING justify MITIGATED verdict. +9. IGNORE UNRELATED HARDENING: -fstack-protector, -D_FORTIFY_SOURCE do NOT mitigate all CWEs. Check the table! + + + +{{"thought": "Need to check hardening flags for this CWE. Search for all compiler/linker flag definitions at once.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "logs:FLAGS=", "reason": "Get all CFLAGS/CXXFLAGS/LDFLAGS definitions in one search"}}, "final_answer": null}} + + + +{{"thought": "CWE-121 stack overflow. EXPECTED_HARDENING lists -fstack-protector-strong. Found -fstack-protector-strong in CFLAGS. This matches EXPECTED_HARDENING.", "mode": "finish", "actions": null, "final_answer": "MITIGATED. Found -fstack-protector-strong which is listed in EXPECTED_HARDENING for CWE-121. Stack buffer overflows will be detected at runtime."}} + + + +{{"thought": "CWE-190 integer overflow. EXPECTED_HARDENING lists -ftrapv, -fsanitize=signed-integer-overflow. Found -fstack-protector-strong and -D_FORTIFY_SOURCE=2, but these are for memory safety NOT integer overflow. Checking for expected flags: -ftrapv MISSING, -fsanitize=signed-integer-overflow MISSING.", "mode": "finish", "actions": null, "final_answer": "NOT_MITIGATED. Build has general hardening (-fstack-protector-strong, -D_FORTIFY_SOURCE=2) but NONE of the CWE-190 specific flags from EXPECTED_HARDENING (-ftrapv, -fsanitize=signed-integer-overflow) are present. Integer overflow is not mitigated."}} + + + +{{"thought": "CWE-693 control flow. EXPECTED_HARDENING lists PIE. Found -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 which implies -fPIE. This matches EXPECTED_HARDENING.", "mode": "finish", "actions": null, "final_answer": "MITIGATED. Build uses RHEL hardened specs files which implicitly enable PIE (listed in EXPECTED_HARDENING for CWE-693). Control flow exploitation is significantly harder."}} + + + +{{"thought": "CWE-190 integer overflow. EXPECTED_HARDENING lists -ftrapv, -fsanitize=*-integer-overflow. None found in FLAGS output.", "mode": "finish", "actions": null, "final_answer": "NOT_MITIGATED. None of the EXPECTED_HARDENING flags for CWE-190 (-ftrapv, -fsanitize=signed-integer-overflow, -fsanitize=unsigned-integer-overflow) are present in the build."}} + + + +{{"thought": "Build log does not contain FLAGS= definitions. Cannot determine if EXPECTED_HARDENING flags are present.", "mode": "finish", "actions": null, "final_answer": "UNKNOWN. Build log does not contain CFLAGS/LDFLAGS definitions. Cannot determine if EXPECTED_HARDENING mitigations are present."}} +""" + +# --------------------------------------------------------------------------- +# L2 Kernel-Specific Configuration Prompts +# --------------------------------------------------------------------------- + +L2_KERNEL_CONFIG_SYS_PROMPT = ( + "You are an L2 Build Agent investigating whether VULNERABLE CODE is COMPILED in a KERNEL package.\n\n" + "KERNEL BUILD CHARACTERISTICS:\n" + "- Kernel builds use 'make -s' (silent) - individual .c file compilation is NOT visible in build logs\n" + "- Compilation is controlled by CONFIG_* options in kernel config files (kernel-{arch}-rhel.config)\n" + "- CONFIG_X=y means built-in (COMPILED), CONFIG_X=m means module (COMPILED), CONFIG_X=n means NOT COMPILED\n" + "- Source-to-module mapping requires parsing Makefile to find obj-$(CONFIG_X) directives\n\n" + "GOAL: Determine if the CVE-affected kernel code is compiled by checking CONFIG_* settings.\n\n" + "INVESTIGATION STRATEGY:\n" + "1. Identify the affected source file(s) from VULNERABILITY_INTEL (e.g., net/netfilter/nf_tables_api.c)\n" + "2. Read the Makefile in the same directory to find which CONFIG_* controls compilation\n" + " - Look for patterns like: nf_tables-objs := ... nf_tables_api.o\n" + " - Then find: obj-$(CONFIG_NF_TABLES) += nf_tables.o\n" + "3. Grep the kernel config file for that CONFIG symbol to get its value (y/m/n)\n" + "4. Determine compilation status based on CONFIG value\n\n" + "VERDICTS:\n" + "- NOT_COMPILED: CONFIG_X=n or CONFIG symbol not defined (feature disabled)\n" + "- COMPILED: CONFIG_X=y (built-in) or CONFIG_X=m (loadable module)\n" + "- UNKNOWN: Cannot determine CONFIG symbol or config file unavailable" +) + +L2_KERNEL_CONFIG_PROMPT_TEMPLATE = """{sys_prompt} + + +CVE ID: {vuln_id} +Target Package: {target_package} + + + +{vulnerability_intel} + +L1 Preliminary Verdict: {l1_preliminary_verdict} + + + +Target Architecture Config File: {kernel_config_path} +Kernel Source Root: {kernel_source_root} + +IMPORTANT: Different architectures (x86_64, aarch64, s390x, ppc64le) have DIFFERENT config files. +A CONFIG option may be enabled on x86_64 but disabled on s390x. +Always search the TARGET ARCHITECTURE config file specified above. + +INVESTIGATION STEPS: +1. From VULNERABILITY_INTEL, identify affected source file(s) + Example: net/netfilter/nf_tables_api.c + +2. Grep the Makefile in the same directory as the affected .c file to find the CONFIG symbol + Source Grep format is pattern[,file_glob] — it searches FILE CONTENTS, not file paths. + WRONG: "drivers/nvme/host/Makefile" (searches for that path string inside files → no matches) + RIGHT: "nvme-tcp,drivers/nvme/host/Makefile" (search for "nvme-tcp" only in that Makefile) + Look for: nf_tables-objs := ... nf_tables_api.o OR nvme-tcp-y += tcp.o + obj-$(CONFIG_NF_TABLES) += nf_tables.o OR obj-$(CONFIG_NVME_TCP) += nvme-tcp.o + → This tells you which CONFIG_* controls the affected .c file + +3. Grep the TARGET ARCHITECTURE kernel config file for that CONFIG symbol + Use the basename from "Target Architecture Config File" above (e.g. kernel-x86_64-rhel.config). + Example: "CONFIG_NF_TABLES,kernel-x86_64-rhel.config" + Result: CONFIG_NF_TABLES=m + +4. Interpret the result: + - CONFIG_X=y → Built into kernel (COMPILED) + - CONFIG_X=m → Built as module (COMPILED) + - CONFIG_X=n or "# CONFIG_X is not set" → NOT_COMPILED + - Symbol not found → NOT_COMPILED (feature effectively disabled/not configured) + + + +{tools} + + +{tool_instructions} + +RESPONSE: +{{""" + +L2_KERNEL_THOUGHT_INSTRUCTIONS = """ +You will receive KNOWLEDGE (cumulative findings) and LATEST FINDINGS (most recent tool results). +BEFORE ACTING, you MUST: +1. Review KNOWLEDGE to see what tools were already called (TOOL_CALL_RECORD entries) +2. Review LATEST FINDINGS for the most recent tool output analysis +3. NEVER repeat any action already in TOOL_CALL_RECORD +4. Your next action MUST build on findings - progress the investigation + + + +Source Grep query: [target:]pattern[,file_glob] +- pattern = text to find INSIDE files (function name, CONFIG_*, module name like nvme-tcp) +- file_glob = optional path/filename filter after a comma (Makefile, *.c, kernel-x86_64-rhel.config) +- NEVER pass a bare file path as the pattern (e.g. "drivers/nvme/host/Makefile" will NOT open that file) +- Use specific patterns, not short strings like "tcp" (matches unrelated files like kernel.spec / mptcp) + + + +1. You MUST select a tool ONLY from . Do NOT invent tool names. +2. Output valid JSON only. thought < 100 words. final_answer < 150 words. +3. mode="act" REQUIRES actions with a tool. mode="finish" REQUIRES final_answer. +4. For kernel packages: FIRST grep the directory Makefile for CONFIG/obj-* lines, THEN grep the arch config file. +5. Makefile step: use pattern[,relative/path/Makefile] — e.g. "nf_tables,net/netfilter/Makefile" or "nvme-tcp,drivers/nvme/host/Makefile". +6. Config step: use CONFIG_SYMBOL,config-basename — e.g. "CONFIG_NVME_TCP,kernel-x86_64-rhel.config" (basename only, not full path). +7. Do NOT search build logs for .c compilation - kernel uses make -s (silent). +8. Do NOT call the same tool with the same input twice. + + + +{{"thought": "CVE affects net/netfilter/nf_tables_api.c. Grep the directory Makefile for nf_tables / CONFIG lines.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "nf_tables,net/netfilter/Makefile", "reason": "Find obj-$(CONFIG_*) controlling nf_tables in that Makefile only"}}, "final_answer": null}} + + + +{{"thought": "CVE affects drivers/nvme/host/tcp.c. Grep the host Makefile for nvme-tcp and CONFIG_NVME_TCP.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "nvme-tcp,drivers/nvme/host/Makefile", "reason": "Find obj-$(CONFIG_NVME_TCP) and nvme-tcp-y lines linking tcp.c to a CONFIG symbol"}}, "final_answer": null}} + + + +{{"thought": "Makefile shows obj-$(CONFIG_NF_TABLES) += nf_tables.o. Grep arch config for CONFIG_NF_TABLES value.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "CONFIG_NF_TABLES,kernel-x86_64-rhel.config", "reason": "Check if CONFIG_NF_TABLES is y/m (compiled), '# CONFIG_* is not set', or missing (not compiled)"}}, "final_answer": null}} + + + +{{"thought": "Makefile shows obj-$(CONFIG_NVME_TCP) += nvme-tcp.o and nvme-tcp-y += tcp.o. Check CONFIG_NVME_TCP in arch config.", "mode": "act", "actions": {{"tool": "Source Grep", "query": "CONFIG_NVME_TCP,kernel-x86_64-rhel.config", "reason": "Confirm CONFIG_NVME_TCP=y/m or '# CONFIG_* is not set'/missing for compilation verdict"}}, "final_answer": null}} + + + +{{"thought": "CONFIG_NF_TABLES=m found in kernel config. Module is compiled.", "mode": "finish", "actions": null, "final_answer": "COMPILED. CONFIG_NF_TABLES=m in kernel config means nf_tables is built as a loadable module. The vulnerable code in net/netfilter/nf_tables_api.c IS compiled into the kernel."}} + + + +{{"thought": "CONFIG_NF_TABLES=y found in kernel config. Built-in to kernel.", "mode": "finish", "actions": null, "final_answer": "COMPILED. CONFIG_NF_TABLES=y in kernel config means nf_tables is built into the kernel. The vulnerable code in net/netfilter/nf_tables_api.c IS compiled into the kernel."}} + + + +{{"thought": "CONFIG_NF_TABLES is not set (n) in kernel config. Feature disabled.", "mode": "finish", "actions": null, "final_answer": "NOT_COMPILED. CONFIG_NF_TABLES=n (or not set) in kernel config means nf_tables is disabled. The vulnerable code in net/netfilter/nf_tables_api.c is NOT compiled into the kernel."}} + + + +{{"thought": "Could not find CONFIG symbol in Makefile. Cannot determine compilation status.", "mode": "finish", "actions": null, "final_answer": "UNKNOWN. Could not determine which CONFIG symbol controls the affected code. Manual review of kernel configuration required."}} +""" + +# --------------------------------------------------------------------------- +# L2 Verdict Extraction Prompts +# --------------------------------------------------------------------------- + +L2_COMPILATION_VERDICT_PROMPT = """Extract the compilation verdict from the L2 Configuration investigation. + + +{final_answer} + + +Extract: +1. compilation_status: "compiled", "not_compiled", or "unknown" +2. confidence: 0.0 to 1.0 based on evidence strength +3. reasoning: Brief explanation (1-2 sentences) + +Output JSON only: +{{"compilation_status": "...", "confidence": 0.X, "reasoning": "..."}}""" + +L2_HARDENING_VERDICT_PROMPT = """Extract the hardening verdict from the L2 Hardening investigation. + + +{final_answer} + + +Extract: +1. hardening_status: "mitigated", "not_mitigated", "not_applicable", or "unknown" + - "not_applicable": This CWE class has no compiler-level mitigations available +2. hardening_flags: List of specific compiler/linker flags that provide protection (e.g., ["-fstack-protector-strong", "-D_FORTIFY_SOURCE=2", "RELRO", "PIE"]) + - Extract the actual flag names mentioned in the investigation + - Empty list if no relevant flags found +3. confidence: 0.0 to 1.0 based on evidence strength +4. reasoning: Brief explanation (1-2 sentences) + +Output JSON only: +{{"hardening_status": "...", "hardening_flags": ["..."], "confidence": 0.X, "reasoning": "..."}}""" + +# --------------------------------------------------------------------------- +# L2 Observation Prompts (Comprehension + Memory Update) +# --------------------------------------------------------------------------- + +L2_COMPREHENSION_PROMPT = """Analyze the tool output for L2 build/compilation verification. +GOAL: Determine whether {vuln_id} vulnerable code is COMPILED in {target_package} + + +{vulnerability_intel} + + + +Disabled Features (build log): {disabled_features} +Disabled Features (spec file): {spec_disabled_features} +Enabled Features (build log): {enabled_features} +Enabled Features (spec file): {spec_enabled_features} + + +TOOL USED: {tool_used} +TOOL INPUT: {tool_input} +THOUGHT: {last_thought} +NEW OUTPUT: +{tool_output} + +BUILD ANALYSIS RULES: +1. CHECK if tool output shows: + - Compilation commands for AFFECTED_FILES (e.g., gcc -c file.c -o file.o) + - Feature-disable flags that match the CVE-affected component + - Patching or processing of files containing VULNERABLE_FUNCTIONS + NOTE: Function names do NOT appear in build logs. Only FILE names appear. + If the file containing a function is compiled/patched, the function is compiled. + +2. COMPILATION EVIDENCE: + - COMPILED: Found gcc/compile commands for affected files + - NOT_COMPILED: Feature is disabled OR affected files not in build + - UNKNOWN: Insufficient evidence + +3. RECORD specific file paths, compile commands, or flag matches. + +TOOL-SPECIFIC RULES: +- If NEW OUTPUT is empty or error: "FAILED: [tool] [input] - [reason]" +- Source Grep: Check if matches show compilation or feature disabling +- Build log search: Look for compile commands and disabled features + +OUTPUT: +- findings: 2-4 key observations about compilation status +- tool_outcome: "Source Grep [pattern] -> found in build.log:123" +RESPONSE: +{{""" + +L2_MEMORY_UPDATE_PROMPT = """Merge findings into L2 build investigation memory. +GOAL: Determine whether {vuln_id} vulnerable code is COMPILED in {target_package} + +PREVIOUS MEMORY: {previous_memory} +NEW FINDINGS: {findings} +TOOL CALL RECORD: {tool_outcome} + +MEMORY RULES: +1. Append NEW FINDINGS to PREVIOUS MEMORY. No duplicates. +2. Add TOOL CALL RECORD verbatim. +3. If NEW FINDINGS report a failure, add the failure to memory. + +COMPILATION TRACKING: +- Affected file COMPILED: "FILE_COMPILED: [file] - evidence: [compile command]" +- Affected file NOT_COMPILED: "FILE_NOT_COMPILED: [file] - evidence: [disabled feature]" +- Feature DISABLED: "FEATURE_DISABLED: [feature] in [build_log/spec]" +- Feature ENABLED: "FEATURE_ENABLED: [feature] - no disable flag found" + +VERDICT EVIDENCE: +- NOT_COMPILED evidence: feature disabled OR affected files not compiled +- COMPILED evidence: affected files appear in compile commands +- UNKNOWN: conflicting evidence or no compilation info found + +- results: copy the NEW FINDINGS as-is. +- memory: updated cumulative findings with evidence tags. +RESPONSE: +{{""" + +# --------------------------------------------------------------------------- +# L2 Hardening Observation Prompts (Comprehension + Memory Update) +# --------------------------------------------------------------------------- + +L2_HARDENING_COMPREHENSION_PROMPT = """Analyze the tool output for L2 hardening flag verification. +GOAL: Determine whether {vuln_id} has HARDENING mitigations in {target_package} + + +CWE: {cwe_id} +Expected Hardening Flags: +{expected_hardening} + + +TOOL USED: {tool_used} +TOOL INPUT: {tool_input} +THOUGHT: {last_thought} +NEW OUTPUT: +{tool_output} + +HARDENING ANALYSIS RULES: +1. FIRST CHECK Expected Hardening Flags above: + - If "None" or empty: This CWE has NO known compiler-level mitigations + - Mark findings as "NO_RELEVANT_HARDENING: {cwe_id} has no compiler mitigations" + - Skip searching for generic flags - they won't help this vulnerability class + +2. IF expected hardening flags exist, CHECK tool output for: + - Compiler hardening flags (e.g., -fstack-protector, -fPIE, -fstack-clash-protection) + - Preprocessor defines (e.g., -D_FORTIFY_SOURCE=2, -D_GLIBCXX_ASSERTIONS) + - Linker hardening flags (e.g., -Wl,-z,relro, -Wl,-z,now) + +3. HARDENING EVIDENCE: + - FLAG_PRESENT: Found expected hardening flag in build commands + - FLAG_ABSENT: Searched but did not find expected flag + - NOT_APPLICABLE: No compiler mitigations exist for this CWE class + - UNKNOWN: Insufficient evidence + +4. RECORD specific flags found and their context (compilation line). + +TOOL-SPECIFIC RULES: +- If NEW OUTPUT is empty or error: "FAILED: [tool] [input] - [reason]" +- Source Grep: Check if matches show hardening flags in gcc/clang commands +- Build log search: Look for -f*, -D*, -Wl,* patterns + +OUTPUT: +- findings: 2-4 key observations about hardening flags +- tool_outcome: "Source Grep [pattern] -> found in build.log:123" +RESPONSE: +{{""" + +L2_HARDENING_MEMORY_UPDATE_PROMPT = """Merge findings into L2 hardening investigation memory. +GOAL: Determine whether {vuln_id} has HARDENING mitigations in {target_package} + +CWE: {cwe_id} +PREVIOUS MEMORY: {previous_memory} +NEW FINDINGS: {findings} +TOOL CALL RECORD: {tool_outcome} + +MEMORY RULES: +1. Append NEW FINDINGS to PREVIOUS MEMORY. No duplicates. +2. Add TOOL CALL RECORD verbatim. +3. If NEW FINDINGS report a failure, add the failure to memory. + +HARDENING TRACKING: +- No relevant hardening: "NO_RELEVANT_HARDENING: [CWE] has no compiler mitigations" +- Flag FOUND: "HARDENING_PRESENT: [flag] - evidence: [build command excerpt]" +- Flag NOT FOUND: "HARDENING_ABSENT: [flag] - searched but not found" +- Critical mitigation: "CRITICAL_MITIGATION: [flag] for [CWE] - [present/absent]" + +VERDICT EVIDENCE: +- NOT_APPLICABLE evidence: this CWE class has no compiler-level mitigations +- MITIGATED evidence: key hardening flags present that reduce exploitability +- NOT_MITIGATED evidence: expected hardening flags absent +- UNKNOWN: build log incomplete or no compilation commands found + +- results: copy the NEW FINDINGS as-is. +- memory: updated cumulative findings with hardening evidence tags. +RESPONSE: +{{""" + + +# =========================================================================== +# FINAL REPORT PROMPTS +# =========================================================================== + +CODE_AGENT_REPORT_PROMPT = """\ + +You are a security analyst generating the final checker investigation report. +Synthesize the results from the target package analysis, additional intel (target rebase check + reference package), code agent analysis (source code), +and optionally build agent analysis (compilation, linking, hardening) into a comprehensive, auditable report with a clear +justification and supporting evidence. + + + +CVE: {vuln_id} +Target Package: {target_package} +CVE Description: {cve_description} + + +{policy_context_section} + +## Target Package Analysis +(Checked the target package being scanned for CVE-specific patch files) +{downstream_section} + +## Additional Intel (Target Rebase + Reference Package) +This section contains TWO distinct checks: +- TARGET REBASE CHECK: Searched the target package's spec file for CVE mentions indicating a rebase fix +- REFERENCE PACKAGE: Downloaded a known-fixed package version from intel to extract patch patterns +{upstream_section} + +## Code Agent Analysis +{l1_agent_section} + + +{build_agent_status_section} +{l2_context_section} +{override_notice_section} + +Generate a structured report following these requirements: + +1. JUSTIFICATION LABEL: + + FIRST: Check BUILD_AGENT_STATUS above. + - If status is "ran_with_override": You MUST use the DECISION TREE below to select the label. + The code agent may report "vulnerable" in source, but the build agent determines final exploitability. + IGNORE the code agent's "vulnerable" verdict when an override is present. + - If status is "ran_no_override": Use code agent verdict with build context. + - If status is "not_run": Use CODE AGENT PRECEDENCE RULES at the end of this section. + + LABEL SELECTION DECISION TREE (MANDATORY when BUILD_AGENT_STATUS is "ran_with_override"): + +-----------------------------------------------------------------------------+ + | Step 1: Is compilation_status == "not_compiled"? | + | YES -> STOP. Label = "code_not_present" | + | NO -> Continue to Step 2 | + +-----------------------------------------------------------------------------+ + | Step 2: Does build agent evidence mention "architecture mismatch" or | + | architecture-specific condition (32-bit/64-bit, endianness)? | + | YES -> STOP. Label = "requires_environment" | + | NO -> Continue to Step 3 | + +-----------------------------------------------------------------------------+ + | Step 3: Is build agent verdict "vulnerable_mitigated" with hardening flags? | + | YES -> STOP. Label = "protected_by_compiler" | + | NO -> Continue to Step 4 | + +-----------------------------------------------------------------------------+ + | Step 4: Is build agent verdict "not_vulnerable" for other reasons? | + | YES -> Label = "code_not_reachable" | + | NO -> Use code-agent-based rules below | + +-----------------------------------------------------------------------------+ + + CRITICAL: When the decision tree applies: + - compilation_status="not_compiled" ALWAYS means "code_not_present" + - Do NOT output "vulnerable" if build agent has an override verdict + - The build agent verdict supersedes code agent source analysis + + AVAILABLE LABELS: + - code_not_present: Vulnerable code/function is NOT COMPILED into the binary + (e.g., disabled via configure flags like --no-ssl, --without-feature, or + conditionally excluded at build time) + - code_not_reachable: Code IS compiled but cannot be reached/executed in this context + - protected_by_mitigating_control: Downstream patch or backport mitigates the vulnerability + - protected_by_compiler: Compiler hardening flags (FORTIFY_SOURCE, stack protector, etc.) + mitigate the vulnerability + - requires_environment: Vulnerability requires specific RUNTIME/ARCHITECTURE conditions + not present (e.g., 32-bit integer overflow impossible on 64-bit, big-endian specific + bug on little-endian system). NOT for build-time configuration. + - vulnerable: Package is actually vulnerable and needs patching + - uncertain: Insufficient information to determine exploitability + + When Override verdict is "none", the build agent ran but did not change the code agent + label—still apply compilation/hardening facts in the executive summary. + + Do NOT state "vulnerable" if build agent evidence contradicts it. Instead explain: + "While source contains vulnerable patterns, the build is not affected due to [build agent reason]." + + EXAMPLES (build agent scenarios): + + Example A - SSL disabled via configure: + Build agent verdict: not_vulnerable + compilation_status: not_compiled + Evidence: "configured with --no-ssl flag" + -> CORRECT: "code_not_present" (Step 1: not_compiled -> code_not_present) + -> WRONG: "vulnerable" (build agent override supersedes code agent) + -> WRONG: "requires_environment" (this is build config, not runtime/arch) + + Example B - 64-bit architecture prevents overflow: + Build agent verdict: not_vulnerable + compilation_status: compiled + Evidence: "Architecture mismatch - 32-bit overflow impossible on 64-bit" + -> CORRECT: "requires_environment" (Step 2: architecture condition) + -> WRONG: "code_not_present" (code IS compiled, just not exploitable) + + Example C - Hardening flags mitigate: + Build agent verdict: vulnerable_mitigated + compilation_status: compiled + Evidence: "FORTIFY_SOURCE=2 prevents buffer overflow exploitation" + -> CORRECT: "protected_by_compiler" (Step 3: hardening mitigation) + + CODE AGENT PRECEDENCE RULES (ONLY when BUILD_AGENT_STATUS is "not_run"): + WARNING: Skip this section if BUILD_AGENT_STATUS contains "ran". + - If a CVE-specific patch file exists AND is applied in build, use "protected_by_mitigating_control". + - If the code agent found the fix code in source, use "protected_by_mitigating_control". + - If the code agent found vulnerable code pattern still present, use "vulnerable". + - If upstream shows rebase fixed the issue, use "protected_by_mitigating_control". + - Only use "uncertain" when evidence is conflicting or insufficient. + +2. EVIDENCE CHAIN: + - Start with target package patch availability + - Include target rebase findings (if CVE mentioned in target's spec changelog, this is from the TARGET package) + - Include reference package findings (if a known-fixed package from intel was used for comparison) + - Include code analysis findings (patch targets, vulnerable vs fix patterns) + - Reference specific files, line numbers, and code snippets + - Summarize findings; the rendered report places an "Extracted facts" section **after** the Evidence chain with verbatim spec Patch lines, changelog hits, and build log lines (when available)—do not invent `PatchN:` numbers or spec quotes; only state patch indices you could derive from the investigation text below, or point readers to *Extracted facts* for exact lines + + PHRASING GUIDANCE for code analysis findings: + - GOOD: "Code analysis verified that the patch modifies `filename.c` to address the vulnerability" + - GOOD: "Patch targets the `function_name()` function in `filename.c`" + - BAD: "L1 agent found the fix code in the source" (use "code agent" — do not say L1/L2) + - BAD: "L2 says not vulnerable" (use "build agent") + - BAD: "Found fix in source" (unclear what was found) + - Use active voice: "The patch adds validation..." not "Validation was found..." + +3. EXECUTIVE SUMMARY (scenario-aware; use "code agent" and "build agent", never L1/L2): + + Read BUILD_AGENT_STATUS first: + + A) Build agent status: ran_with_override (Override verdict is not "none"): + - Write 4-5 sentences. + - Sentence 1 (Verdict): State final posture clearly + - Sentences 2-3 (Technical Context): Nature of the flaw and why the build agent negates or mitigates it + - Sentences 4-5 (Reconciliation): Why the code agent found patterns in source but the build agent changes exploitability + + B) Build agent status: ran_no_override: + - Write 3-5 sentences. + - Sentence 1: State verdict (code agent label stands) + - Include 1-2 sentences on compilation status and/or hardening from BUILD_AGENT_CONTEXT when present + - Do NOT say the build agent is absent, missing, or "context is not present" + + C) Build agent status: not_run: + - Write 3-4 sentences on code agent and patch intel only + - Do NOT mention the build agent at all + + FORBIDDEN phrases (never write these): + - "build agent context is not present" + - "L2 context is not present" + - "verdict is based solely on the code agent analysis" (when BUILD_AGENT_STATUS is ran_no_override) + - "because the build agent did not run" (when BUILD_AGENT_STATUS is ran_no_override) + + Do NOT invent RHSA IDs, function names, or technical details not present in the context. + +4. PATCH ANALYSIS (semantic fix narrative): + - When target package patch or reference patch evidence exists, briefly describe **what** the fix does: name the function(s) or file(s) and the nature of the change (e.g. "adds range validation 15–17 in parse_rockridge_ZF1"). + - Derive this from Target Package Analysis, Reference Intel, patch file names, or code agent excerpts—do NOT invent code or function names absent from investigation results. + +5. DELIVERY MODEL: + - When a CVE-named patch file is present, explicitly note that the fix is carried as a separate `%patch` directive while the upstream tarball (`Source0`) version may remain unchanged. + - Encourage citing "Extracted facts" for exact spec `PatchN:` and `Source0`/`Version` lines when shown below. + +6. CAVEATS (optional): + - Note any missing data (no patch file, no build log, etc.) + - Flag low-confidence findings that may need manual review + - Leave empty if no significant gaps exist + + + +Provide a structured JSON response with: +- justification_label: one of the labels above +- executive_summary: 3-5 sentence summary (see Instruction #3 for structure) +- evidence_chain: list of evidence items in logical order +- affected_files: list of source files involved +- patch_analysis: analysis of patches (or null if none) +- caveats: list of investigation gaps or uncertainties (empty list if none) + +Ensure all code snippets and special characters within JSON string values are properly escaped +(e.g., quotes as \", backslashes as \\, newlines as \\n) to maintain valid JSON format. + +""" diff --git a/src/vuln_analysis/utils/serp_api_wrapper.py b/src/vuln_analysis/utils/serp_api_wrapper.py index a412cc82d..eecac3514 100644 --- a/src/vuln_analysis/utils/serp_api_wrapper.py +++ b/src/vuln_analysis/utils/serp_api_wrapper.py @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import threading +from typing import ClassVar + import aiohttp from langchain.utils.env import get_from_env from langchain_community.utilities import SerpAPIWrapper @@ -23,10 +26,38 @@ class MorpheusSerpAPIWrapper(SerpAPIWrapper): - + """Custom SerpAPI wrapper with multi-key rotation support. + + This wrapper extends the standard SerpAPIWrapper to support multiple API keys + for increased request capacity. When a key encounters rate limiting (429) or + payment issues (402), the wrapper automatically rotates to the next available key. + + Key rotation state is shared across all instances to ensure that when one instance + encounters a rate limit, all other instances benefit from the rotation without making + redundant failing requests. + + Attributes: + base_url: Base URL for SerpAPI service + max_retries: Maximum retry attempts for network errors + serp_api_keys: Pool of API keys loaded from SERPAPI_API_KEY env variable + serp_api_key_index: Index of currently active key in the pool + """ + _key_rotation_lock: ClassVar[threading.Lock] = threading.Lock() + _serp_api_keys: ClassVar[list[str]] = [] + _serp_api_key_index: ClassVar[int] = 0 + base_url: str = "https://serpapi.com" max_retries: int = 10 - + + @property + def serp_api_keys(self) -> list[str]: + """Shared API keys pool.""" + return self.__class__._serp_api_keys + + @property + def serp_api_key_index(self) -> int: + """Shared current key index.""" + return self.__class__._serp_api_key_index @model_validator(mode="after") def validate_base_url(self) -> "MorpheusSerpAPIWrapper": """Validate the base URL from the environment.""" @@ -36,23 +67,90 @@ def validate_base_url(self) -> "MorpheusSerpAPIWrapper": # Update the base URL for search_engine self.search_engine.BACKEND = self.base_url return self - - @retry_async() + @model_validator(mode="after") + def validate_serp_api_keys(self) -> "MorpheusSerpAPIWrapper": + """Initialize API keys pool from SERPAPI_API_KEY environment variable. + + Parses comma-separated keys from the serpapi_api_key field (populated by parent class) + and initializes the key pool. Sets the first key as the active key. + + Returns: + Self for method chaining + + Raises: + ValueError: If SERPAPI_API_KEY is empty or not set + """ + if not self.serpapi_api_key: + raise ValueError("SERPAPI_API_KEY must not be empty") + keys = [k.strip() for k in self.serpapi_api_key.split(",") if k.strip()] + + cls = self.__class__ + with cls._key_rotation_lock: + # Initialize class-level keys (first time or if keys changed) + if not cls._serp_api_keys or cls._serp_api_keys != keys: + cls._serp_api_keys = keys + cls._serp_api_key_index = 0 + # Set instance's active key to current class-level key + self.serpapi_api_key = cls._serp_api_keys[cls._serp_api_key_index] + return self + def _rotate_next_key(self) -> str: + """Rotate to the next API key in the pool. + + Advances key index cyclically through the pool and updates the active key. + This method is called when the current key encounters rate limiting (429) or + payment errors (402). All instances share the same rotation state. + + Returns: + The new active API key (returned under lock to ensure atomicity) + """ + cls = self.__class__ + with cls._key_rotation_lock: + cls._serp_api_key_index = (cls._serp_api_key_index + 1) % len(cls._serp_api_keys) + self.serpapi_api_key = cls._serp_api_keys[cls._serp_api_key_index] + return self.serpapi_api_key + @retry_async(exclude_http_statuses=[402, 429]) async def _session_get_with_retry(self, session: aiohttp.ClientSession, url: str, params: dict) -> dict: - + """Execute HTTP GET request with retry logic. + + Uses retry_async decorator to retry on network errors (timeout, 5xx), but excludes + 402 and 429 status codes from retry (these trigger key rotation in aresults). + + Args: + session: Active aiohttp client session + url: Target URL for the request + params: Query parameters including API key + + Returns: + JSON response as dictionary + + Raises: + aiohttp.ClientResponseError: For HTTP errors (including 402/429) + """ async with session.get(url, params=params) as response: - res = await response.json() - return res + response.raise_for_status() + return await response.json() - # Override the method with hardcoded URL async def aresults(self, query: str) -> dict: - """Use aiohttp to run query through SerpAPI and return the results async.""" + """Execute async SerpAPI search with automatic key rotation on rate limits. + + Implements two-level retry strategy: + 1. Network errors (timeout, 5xx) are retried by @retry_async decorator + 2. Rate limit errors (402, 429) trigger rotation to next API key + + Args: + query: Search query string + + Returns: + SerpAPI response as dictionary + + Raises: + Exception: If all API keys are exhausted + aiohttp.ClientResponseError: For other HTTP errors (400, 500, etc.) + """ def construct_url_and_params() -> tuple[str, dict[str, str]]: params = self.get_params(query) params["source"] = "python" - if self.serpapi_api_key: - params["serp_api_key"] = self.serpapi_api_key params["output"] = "json" # Use the base path for the URL (add a "/" to ensure they get joined) @@ -60,14 +158,26 @@ def construct_url_and_params() -> tuple[str, dict[str, str]]: return url, params url, params = construct_url_and_params() - if not self.aiosession: - async with aiohttp.ClientSession() as session: - res = await self._session_get_with_retry(session, url, params) - else: - res = await self._session_get_with_retry(self.aiosession, url, params) - - return res - + session = self.aiosession if self.aiosession else aiohttp.ClientSession() + close_session_after = not self.aiosession + attempts = 0 + max_attempts = len(self.serp_api_keys) + try: + while attempts < max_attempts: + try: + res = await self._session_get_with_retry(session, url, params) + return res + except aiohttp.ClientResponseError as e: + if e.status in [402, 429]: + attempts += 1 + if attempts < max_attempts: + params["api_key"] = self._rotate_next_key() + else: + raise + raise Exception("All API keys exhausted") + finally: + if close_session_after: + await session.close() @staticmethod def _process_response(res: dict) -> str: """Catch the ValueError and return a message if no good search result found.""" diff --git a/src/vuln_analysis/utils/source_classification.py b/src/vuln_analysis/utils/source_classification.py new file mode 100644 index 000000000..6a74fd12c --- /dev/null +++ b/src/vuln_analysis/utils/source_classification.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, TypeVar + +from exploit_iq_commons.utils.dep_tree import ECOSYSTEM_DEP_DIRS + +_VENDORS = list(ECOSYSTEM_DEP_DIRS.values()) + +T = TypeVar("T") + + +def is_dependency_path(file_path: str) -> bool: + """Return True if file_path starts with any ecosystem dependency directory prefix.""" + return any(file_path.startswith(vendor) for vendor in _VENDORS) + + +def filter_by_source_scope(dep_items: list[T], source_scope: list[str] | None, + path_fn: Callable[[T], str]) -> list[T]: + """Filter dependency items to those whose path contains any source_scope substring. + + Args: + dep_items: Dependency result items to filter. + source_scope: Path substrings to match. None means no filtering. + path_fn: Extracts the file path string from an item. + """ + if not source_scope: + return dep_items + return [item for item in dep_items if any(scope in path_fn(item) for scope in source_scope)] + + +def format_app_dep_output(app_items: list[str], dep_items: list[str], + total_app: int, total_dep: int, + no_results_msg: str) -> str: + """Format results with app/dep section headers matching Code Keyword Search output. + + Output format: + Main application (N of M results) + + Application library dependencies (N of M results) + + """ + if total_app == 0 and total_dep == 0: + return no_results_msg + + app_header = f"Main application ({len(app_items)} of {total_app} results)" + dep_header = f"Application library dependencies ({len(dep_items)} of {total_dep} results)" + + parts = [app_header] + if app_items: + parts.append("\n\n".join(app_items)) + parts.append(dep_header) + if dep_items: + parts.append("\n\n".join(dep_items)) + return "\n".join(parts) diff --git a/src/vuln_analysis/utils/source_code_git_loader.py b/src/vuln_analysis/utils/source_code_git_loader.py deleted file mode 100644 index 61d27293d..000000000 --- a/src/vuln_analysis/utils/source_code_git_loader.py +++ /dev/null @@ -1,192 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import os -import typing -from pathlib import Path - -from git import Blob as GitBlob -from git import Repo -from langchain_community.document_loaders.blob_loaders.schema import BlobLoader -from langchain_core.document_loaders.blob_loaders import Blob -from tqdm import tqdm - -PathLike = typing.Union[str, os.PathLike] - -logger = logging.getLogger(__name__) - - -class SourceCodeGitLoader(BlobLoader): - """ - Load `Git` repository files. - - The Repository can be local on disk available at `repo_path`, - or remote at `clone_url` that will be cloned to `repo_path`. - Currently, supports only text files. - - Each document represents one file in the repository. The `path` points to - the local Git repository, and the `ref` specifies the git reference to load - files from. By default, it loads from the `main` branch. - """ - - def __init__( - self, - repo_path: PathLike, - clone_url: str | None = None, - ref: typing.Optional[str] = "main", - include: typing.Optional[typing.Iterable[str]] = None, - exclude: typing.Optional[typing.Iterable[str]] = None, - ): - """ - Initialize the Git loader. - - Parameters - ---------- - repo_path : PathLike - Path to the local Git repository. - clone_url : str | None, optional - URL to the remote Git repository, by default None - ref : typing.Optional[str], optional - Commit ID to load files from, by default "main" - include : typing.Optional[typing.Iterable[str]], optional - A list of file patterns to include. Uses the glob syntax, by default None - exclude : typing.Optional[typing.Iterable[str]], optional - A list of file patterns to exclude. Uses the glob syntax, by default None - """ - - self.repo_path = Path(repo_path) - self.clone_url = clone_url - self.ref = ref - - self.include = include - self.exclude = exclude - - self._repo: Repo | None = None - - def load_repo(self): - """ - Load the Git repository and return the GitPython `Repo` object. - - Returns - ------- - Repo - GitPython `Repo` object representing the Git repository. - - Raises - ------ - ValueError - If the repository path does not exist. - ValueError - If a different repository is already cloned at the path. - """ - - if (self._repo is not None): - return self._repo - - if not os.path.exists(self.repo_path) and self.clone_url is None: - raise ValueError(f"Path {self.repo_path} does not exist") - elif self.clone_url: - # If the repo_path already contains a git repository, verify that it's the - # same repository as the one we're trying to clone. - if os.path.isdir(os.path.join(self.repo_path, ".git")): - repo = Repo(self.repo_path) - # If the existing repository is not the same as the one we're trying to - # clone, raise an error. - if repo.remotes.origin.url != self.clone_url: - raise ValueError("A different repository is already cloned at this path.") - - logger.info("Updating existing Git repo for URL '%s' @ '%s'", self.clone_url, self.ref) - else: - logger.info("Cloning repository from URL: '%s' @ '%s'", self.clone_url, self.ref) - # Create a shallow clone of the repository without checking out - repo = Repo.clone_from(self.clone_url, self.repo_path, depth=1, no_checkout=True) - - # Set repo as git safe directory to avoid errors if directory ownership is changed outside the pipeline - # https://git-scm.com/docs/git-config#Documentation/git-config.txt-safedirectory - with repo.config_writer(config_level="global") as config: - config.add_value("safe", "directory", str(self.repo_path.absolute())) - - else: - repo = Repo(self.repo_path) - logger.info("Using existing Git repo at path: '%s' @ '%s'", self.repo_path, self.ref) - - # Reliable way to check out the ref using a shallow clone - repo.git.fetch("origin", self.ref, depth=1) - repo.git.checkout("FETCH_HEAD") - - logger.info("Loaded Git repository at path: '%s' @ '%s'", self.repo_path, self.ref) - - self._repo = repo - - return repo - - def yield_blobs(self) -> typing.Iterator[Blob]: - """ - Yield the blobs from the Git repository. One blob will be generated for each file in the repo which passes the - include and exclude filters. - - Returns - ------- - typing.Iterator[Blob] - An iterator of `Blob` objects representing the files in the repository. - - Yields - ------ - Iterator[typing.Iterator[Blob]] - An iterator of `Blob` objects representing the files in the repository. - """ - - repo = self.load_repo() - - logger.info("Scanning documents for Git repository at path: '%s'", self.repo_path) - - all_files_in_repo = [str(item.path) for item in repo.tree().traverse() if isinstance(item, GitBlob)] - - base_path = Path(self.repo_path) - - include_files: set[str] = set() - exclude_files: set[str] = set() - - for inc in self.include or ["**/*"]: - include_files = include_files.union(set(str(x.relative_to(base_path)) for x in base_path.glob(inc))) - - for exc in self.exclude or {}: - exclude_files = exclude_files.union(set(str(x.relative_to(base_path)) for x in base_path.glob(exc))) - - # Filter out files that are not in the repo - include_files = include_files.intersection(all_files_in_repo) - - # Take the include files and remove the exclude files. - final_files = include_files - exclude_files - - logger.info("Processing %d files in the Git repository at path: '%s'", len(final_files), self.repo_path) - - for f in tqdm(final_files): - - file_path = Path(f) - - abs_file_path = base_path / file_path - - rel_file_path = str(file_path) - - metadata = { - "source": rel_file_path, - "file_path": rel_file_path, - "file_name": file_path.name, - "file_type": file_path.suffix, - } - - yield Blob.from_path(abs_file_path, metadata=metadata) diff --git a/src/vuln_analysis/utils/tests/test_checklist_prompt_generator.py b/src/vuln_analysis/utils/tests/test_checklist_prompt_generator.py new file mode 100644 index 000000000..76d17a893 --- /dev/null +++ b/src/vuln_analysis/utils/tests/test_checklist_prompt_generator.py @@ -0,0 +1,93 @@ +import pytest +import logging +logger = logging.getLogger(__name__) + +from vuln_analysis.utils.checklist_prompt_generator import _parse_list + +line1 = '"Is the `USER $USERNAME` Dockerfile instruction used in the container image, potentially allowing an attacker to manipulate supplementary group access?",' +line2 = '"Are there any instances of `ENTRYPOINT ["su", "-", "user"]` in the Dockerfile, which could indicate a safer alternative to setting up supplementary groups?",' +line3 = '"Does the container image run with a user that has elevated privileges, potentially increasing the risk of exploiting supplementary group mishandling?",' +line4 = '"Are there any sensitive files or directories within the container that could be accessed by an attacker if they were to bypass primary group restrictions using supplementary group access?",' +line5 = '"Does the container\'s configuration or Dockerfile explicitly set or modify supplementary groups, which could impact the vulnerability\'s exploitability?"' +list1_as_string = f"[{line1}{line2}{line3}{line4}{line5}]" +list2_as_string_with_newlines = f"[{line1}\n{line2}\n{line3}\n{line4}\n{line5}]" + +expected_line1 = "Is the `USER $USERNAME` Dockerfile instruction used in the container image, potentially allowing an attacker to manipulate supplementary group access?" +expected_line2 = "Are there any instances of `ENTRYPOINT [\"su\", \"-\", \"user\"]` in the Dockerfile, which could indicate a safer alternative to setting up supplementary groups?" +expected_line3 = "Does the container image run with a user that has elevated privileges, potentially increasing the risk of exploiting supplementary group mishandling?" +expected_line4 = "Are there any sensitive files or directories within the container that could be accessed by an attacker if they were to bypass primary group restrictions using supplementary group access?" +expected_line5 = "Does the container\'s configuration or Dockerfile explicitly set or modify supplementary groups, which could impact the vulnerability\'s exploitability?" + +@pytest.mark.asyncio +async def test_parse_single_with_none_escaped_quotes_without_newlines(): + result = await _parse_list([list1_as_string]) + assert result[0][0] == expected_line1 + assert result[0][1] == expected_line2 + assert result[0][2] == expected_line3 + assert result[0][3] == expected_line4 + assert result[0][4] == expected_line5 + +@pytest.mark.asyncio +async def test_parse_single_with_none_escaped_quotes_with_newlines(): + result = await _parse_list([list2_as_string_with_newlines]) + assert result[0][0] == expected_line1 + assert result[0][1] == expected_line2 + assert result[0][2] == expected_line3 + assert result[0][3] == expected_line4 + assert result[0][4] == expected_line5 + +@pytest.mark.asyncio +async def test_parse_single_with_backslashes(): + input_backslashes = r""" + [ + "Are there any instances of `ENTRYPOINT ["su", "-", "user"]` in the Dockerfile, which could indicate a safer alternative to setting up supplementary groups?", + "Does \ the container's configuration or Dockerfile explicitly set or modify supplementary groups, which could impact the vulnerability's exploitability?" + ] + """ + expected_line2 = "Does \\ the container's configuration or Dockerfile explicitly set or modify supplementary groups, which could impact the vulnerability's exploitability?" + result = await _parse_list([input_backslashes]) + assert result[0][1] == expected_line2 + + +@pytest.mark.asyncio +async def test_parse_single_with_backslashes_n(): + input_backslashes_with_newline = """ + [ + "line1 with \nnewlines" + ] + """ + expected_line1 = "line1 with newlines" + result = await _parse_list([input_backslashes_with_newline]) + assert result[0][0] == expected_line1 + +@pytest.mark.asyncio +async def test_parse_single_with_backslashes_t(): + char_line = r"line1 with \tchar" + char_line_none_raw = "line1 with \tchar" + input_backslashes_list = "[\"" + char_line + "\"]" + result = await _parse_list([input_backslashes_list]) + assert repr(result[0][0]) == repr(char_line_none_raw) + +@pytest.mark.asyncio +async def test_parse_single_with_backslashes_f(): + char_line = r"line1 with \fchar" + char_line_none_raw = "line1 with \fchar" + input_backslashes_list = "[\"" + char_line + "\"]" + result = await _parse_list([input_backslashes_list]) + assert repr(result[0][0]) == repr(char_line_none_raw) + +@pytest.mark.asyncio +async def test_parse_single_with_backslashes_b(): + char_line = r"line1 with \bchar" + char_line_none_raw = "line1 with \bchar" + input_backslashes_list = "[\"" + char_line + "\"]" + result = await _parse_list([input_backslashes_list]) + assert repr(result[0][0]) == repr(char_line_none_raw) + +@pytest.mark.asyncio +async def test_parse_single_with_backslashes_backslash(): + char_line = r"line1 with \ char" + char_line_none_raw = "line1 with \\ char" + input_backslashes_list = "[\"" + char_line + "\"]" + result = await _parse_list([input_backslashes_list]) + assert repr(result[0][0]) == repr(char_line_none_raw) diff --git a/src/vuln_analysis/utils/tests/test_function_name_locator_go.py b/src/vuln_analysis/utils/tests/test_function_name_locator_go.py new file mode 100644 index 000000000..755c5a3af --- /dev/null +++ b/src/vuln_analysis/utils/tests/test_function_name_locator_go.py @@ -0,0 +1,157 @@ +import pytest +from unittest.mock import MagicMock +from exploit_iq_commons.utils.dep_tree import Ecosystem +from exploit_iq_commons.utils.functions_parsers.golang_functions_parsers import GoLanguageFunctionsParser +from vuln_analysis.utils.function_name_locator import FunctionNameLocator + + +class TestFLGoPackageMatching: + def _make_locator(self, supported_packages): + """Build a FunctionNameLocator with a mocked Go retriever.""" + parser = GoLanguageFunctionsParser() + retriever = MagicMock() + retriever.language_parser = parser + retriever.ecosystem = Ecosystem.GO + retriever.supported_packages = supported_packages + retriever.documents_of_functions = [] + return FunctionNameLocator(retriever) + + def test_exact_match(self): + locator = self._make_locator(["github.com/golang-jwt/jwt/v5"]) + assert locator.search_in_third_party_packages("github.com/golang-jwt/jwt/v5") is True + + def test_fqdn_to_module_prefix_match(self): + """Sub-package path should resolve to its parent module.""" + locator = self._make_locator(["google.golang.org/protobuf"]) + assert locator.search_in_third_party_packages( + "google.golang.org/protobuf/encoding/protojson" + ) is True + + def test_fqdn_to_module_picks_longest_prefix(self): + """When multiple modules share a prefix, pick the longest match.""" + locator = self._make_locator([ + "google.golang.org/protobuf", + "google.golang.org/protobuf/encoding", + ]) + result = locator._resolve_go_fqdn_to_module( + "google.golang.org/protobuf/encoding/protojson" + ) + assert result == "google.golang.org/protobuf/encoding" + + def test_sibling_submodule_not_matched_by_prefix(self): + """_resolve_go_fqdn_to_module alone can't match siblings — only prefix matches.""" + locator = self._make_locator([ + "google.golang.org/protobuf/jsonpb", + "google.golang.org/grpc", + ]) + result = locator._resolve_go_fqdn_to_module( + "google.golang.org/protobuf/encoding/protojson" + ) + assert result is None + + def test_short_go_package_name_match(self): + """Short name lookup should find the full package.""" + locator = self._make_locator(["github.com/hashicorp/go-retryablehttp"]) + assert locator.search_in_third_party_packages("go-retryablehttp") is True + + def test_no_false_match(self): + locator = self._make_locator(["github.com/hashicorp/go-retryablehttp"]) + assert locator.search_in_third_party_packages("github.com/some/other/package") is False + + def test_is_same_package_substring(self): + """Go is_same_package uses substring matching: input IN tree_entry.""" + locator = self._make_locator(["github.com/golang-jwt/jwt/v5"]) + assert locator.search_in_third_party_packages("jwt") is True + assert locator.search_in_third_party_packages( + "github.com/golang-jwt/jwt/v5/extra/subpackage" + ) is True + + +class TestFLGoCommonRootAutoResolve: + """Tests for _find_go_common_root_auto_resolve — the sibling sub-module fix.""" + + def test_sibling_submodule_resolves_to_common_root(self): + """Core bug scenario: protobuf/encoding/protojson should resolve to + protobuf (the common root), NOT protobuf/jsonpb (the sibling).""" + result = FunctionNameLocator._find_go_common_root_auto_resolve( + "google.golang.org/protobuf/encoding/protojson", + ["google.golang.org/protobuf/jsonpb"], + ) + assert result == "google.golang.org/protobuf" + + def test_common_root_not_sibling(self): + """Returned value must be the common root, never the sibling match itself.""" + result = FunctionNameLocator._find_go_common_root_auto_resolve( + "github.com/hashicorp/vault/api/sub", + ["github.com/hashicorp/vault/sdk"], + ) + assert result == "github.com/hashicorp/vault" + assert result != "github.com/hashicorp/vault/sdk" + + def test_only_domain_in_common_no_match(self): + """Sharing only the domain (1 segment) should not resolve.""" + result = FunctionNameLocator._find_go_common_root_auto_resolve( + "google.golang.org/grpc/codes", + ["google.golang.org/protobuf/jsonpb"], + ) + assert result is None + + def test_different_github_orgs_no_match(self): + """Different GitHub orgs share only github.com (1 segment) — should not resolve.""" + result = FunctionNameLocator._find_go_common_root_auto_resolve( + "github.com/foo/bar/sub", + ["github.com/baz/bar/sub"], + ) + assert result is None + + def test_versioned_modules_common_root(self): + """jwt/v5/parser with jwt/v4 in tree should resolve to the unversioned root.""" + result = FunctionNameLocator._find_go_common_root_auto_resolve( + "github.com/golang-jwt/jwt/v5/parser", + ["github.com/golang-jwt/jwt/v4"], + ) + assert result == "github.com/golang-jwt/jwt" + + def test_multiple_siblings_picks_longest_root(self): + """When multiple close matches exist, pick the one with the longest common root.""" + result = FunctionNameLocator._find_go_common_root_auto_resolve( + "google.golang.org/protobuf/encoding/protojson", + [ + "google.golang.org/protobuf/jsonpb", + "google.golang.org/protobuf/encoding/prototext", + ], + ) + # encoding/prototext shares 3 segments (google.golang.org/protobuf/encoding) + # jsonpb shares 2 segments (google.golang.org/protobuf) + assert result == "google.golang.org/protobuf/encoding" + + def test_empty_matches_returns_none(self): + result = FunctionNameLocator._find_go_common_root_auto_resolve( + "google.golang.org/protobuf/encoding/protojson", + [], + ) + assert result is None + + def test_case_insensitive_comparison(self): + """Comparison should be case-insensitive but return original case.""" + result = FunctionNameLocator._find_go_common_root_auto_resolve( + "Google.Golang.Org/Protobuf/Encoding/Protojson", + ["google.golang.org/protobuf/jsonpb"], + ) + assert result == "google.golang.org/protobuf" + + def test_exact_package_in_matches(self): + """If the exact package is somehow in matches, all segments match.""" + result = FunctionNameLocator._find_go_common_root_auto_resolve( + "github.com/foo/bar", + ["github.com/foo/bar"], + ) + assert result == "github.com/foo/bar" + + def test_single_segment_package_no_match(self): + """Single-segment packages should never resolve (below 2-segment minimum).""" + result = FunctionNameLocator._find_go_common_root_auto_resolve( + "protobuf", + ["protobuf-lite"], + ) + assert result is None diff --git a/src/vuln_analysis/utils/tests/test_function_name_locator_python.py b/src/vuln_analysis/utils/tests/test_function_name_locator_python.py new file mode 100644 index 000000000..5cd6f0ca5 --- /dev/null +++ b/src/vuln_analysis/utils/tests/test_function_name_locator_python.py @@ -0,0 +1,43 @@ +import pytest +from unittest.mock import MagicMock +from exploit_iq_commons.utils.dep_tree import Ecosystem +from exploit_iq_commons.utils.functions_parsers.python_functions_parser import PythonLanguageFunctionsParser +from vuln_analysis.utils.function_name_locator import FunctionNameLocator + + +class TestFLPythonPackageMatching: + def _make_locator(self, supported_packages, import_to_pypi=None): + """Build a FunctionNameLocator with a mocked Python retriever.""" + parser = PythonLanguageFunctionsParser() + if import_to_pypi: + builder = MagicMock() + builder._import_to_pypi = import_to_pypi + parser.set_dependency_builder(builder) + + retriever = MagicMock() + retriever.language_parser = parser + retriever.ecosystem = Ecosystem.PYTHON + retriever.supported_packages = supported_packages + retriever.documents_of_functions = [] + return FunctionNameLocator(retriever) + + def test_exact_match(self): + locator = self._make_locator(["flask", "requests"]) + assert locator.search_in_third_party_packages("flask") is True + + def test_hyphen_underscore_match(self): + """my-package should match my_package via PEP 503 normalization.""" + locator = self._make_locator(["my_package"]) + assert locator.search_in_third_party_packages("my-package") is True + + def test_pypi_to_import_mapping_match(self): + """python-dateutil should match 'dateutil' via mapping.""" + locator = self._make_locator( + ["dateutil"], + import_to_pypi={"dateutil": "python-dateutil"} + ) + assert locator.search_in_third_party_packages("python-dateutil") is True + + def test_no_false_match(self): + locator = self._make_locator(["flask"]) + assert locator.search_in_third_party_packages("django") is False \ No newline at end of file diff --git a/src/vuln_analysis/utils/tests/test_intel_utils_exports.py b/src/vuln_analysis/utils/tests/test_intel_utils_exports.py new file mode 100644 index 000000000..f407a17f2 --- /dev/null +++ b/src/vuln_analysis/utils/tests/test_intel_utils_exports.py @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for intel_utils module.""" + +import vuln_analysis.utils.intel_utils as intel_utils + + +class TestTestFileRePublic: + """Tests for TEST_FILE_RE being public (fix 6).""" + + def test_exported_as_public(self): + assert hasattr(intel_utils, "TEST_FILE_RE") + assert not intel_utils.TEST_FILE_RE.pattern.startswith("_") + + def test_matches_go_test_file(self): + assert intel_utils.TEST_FILE_RE.search("pkg/handler_test.go") + + def test_matches_java_test_file(self): + assert intel_utils.TEST_FILE_RE.search("src/test/java/FooTest.java") + + def test_no_match_production_file(self): + assert not intel_utils.TEST_FILE_RE.search("src/main/java/Foo.java") + + def test_matches_python_test_file(self): + assert intel_utils.TEST_FILE_RE.search("tests/test_utils.py") + + def test_matches_js_spec_file(self): + assert intel_utils.TEST_FILE_RE.search("src/handler.spec.ts") + + +class TestDeadConstantsRemoved: + """Tests for dead constants removal (fix 8).""" + + def test_github_commit_re_removed(self): + assert not hasattr(intel_utils, "_GITHUB_COMMIT_RE") + + def test_github_api_timeout_removed(self): + assert not hasattr(intel_utils, "_GITHUB_API_TIMEOUT") + + def test_patch_max_commits_removed(self): + assert not hasattr(intel_utils, "_PATCH_MAX_COMMITS") \ No newline at end of file diff --git a/src/vuln_analysis/utils/tests/test_llm_engine_utils.py b/src/vuln_analysis/utils/tests/test_llm_engine_utils.py new file mode 100644 index 000000000..e32aad192 --- /dev/null +++ b/src/vuln_analysis/utils/tests/test_llm_engine_utils.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for llm_engine_utils module.""" + +import vuln_analysis.utils.llm_engine_utils as llm_engine_utils +from vuln_analysis.functions.code_agent_graph_defs import PatchHunk, PatchFile, ParsedPatch +from vuln_analysis.utils.web_patch_fetcher import WebPatchResult + + +class TestNonCodeExtensionsRemoved: + """Tests for _NON_CODE_EXTENSIONS removal (fix 5).""" + + def test_no_non_code_extensions_set(self): + assert not hasattr(llm_engine_utils, "_NON_CODE_EXTENSIONS") + + +class TestFilterBeforeSlice: + """Tests for filter-before-slice in _build_full_pipeline_details_md (fix 2).""" + + @staticmethod + def _make_patch_result(files: list[PatchFile]) -> WebPatchResult: + return WebPatchResult( + cve_id="CVE-2024-0001", + fixed_commit="abc123", + repo_url="https://github.com/example/repo", + patch_url="https://example.com/commit/abc.patch", + source="test", + parsed_patch=ParsedPatch(patch_filename="test.patch", files=files), + ) + + @staticmethod + def _make_file(path: str) -> PatchFile: + return PatchFile( + source_path=f"a/{path}", + target_path=f"b/{path}", + hunks=[PatchHunk( + source_start=1, source_length=1, target_start=1, target_length=1, + added_lines=["+ fix"], + )], + ) + + def test_non_code_files_filtered_before_limit(self): + """Files with unrecognized extensions should not count toward the 5-file limit.""" + non_code = [self._make_file(f"data/blob{i}.dat") for i in range(10)] + code = [self._make_file(f"src/fix{i}.java") for i in range(3)] + result = self._make_patch_result(non_code + code) + md = llm_engine_utils._build_full_pipeline_details_md(result) + assert md is not None + for i in range(3): + assert f"src/fix{i}.java" in md + for i in range(10): + assert f"data/blob{i}.dat" not in md + + def test_markdown_and_yaml_files_filtered(self): + """Non-code files like .md and .yml should be excluded even though + infer_language_from_path returns a truthy language name for them.""" + files = [ + self._make_file("README.md"), + self._make_file("config.yml"), + self._make_file("data.json"), + self._make_file("src/Fix.java"), + ] + result = self._make_patch_result(files) + md = llm_engine_utils._build_full_pipeline_details_md(result) + assert md is not None + assert "src/Fix.java" in md + assert "README.md" not in md + assert "config.yml" not in md + assert "data.json" not in md + + def test_test_files_filtered(self): + """Test files should be excluded from the details output.""" + files = [ + self._make_file("src/Foo.java"), + self._make_file("src/FooTest.java"), + self._make_file("tests/test_bar.py"), + ] + result = self._make_patch_result(files) + md = llm_engine_utils._build_full_pipeline_details_md(result) + assert "src/Foo.java" in md + assert "FooTest.java" not in md + assert "test_bar.py" not in md + + def test_all_non_code_returns_none(self): + """If all files have unrecognized extensions, result should be None.""" + files = [self._make_file("data.dat"), self._make_file("image.png")] + result = self._make_patch_result(files) + md = llm_engine_utils._build_full_pipeline_details_md(result) + assert md is None or "data.dat" not in md + + def test_overflow_count_uses_code_files_not_parsed_files(self): + """The 'more files' count should reflect code files, not total parsed files.""" + non_code = [self._make_file(f"docs/page{i}.md") for i in range(20)] + code = [self._make_file(f"src/fix{i}.java") for i in range(7)] + result = self._make_patch_result(non_code + code) + md = llm_engine_utils._build_full_pipeline_details_md(result) + assert md is not None + assert "+2 more files" in md + assert "+22 more files" not in md \ No newline at end of file diff --git a/src/vuln_analysis/utils/tests/test_vulnerability_intel_sanitizer.py b/src/vuln_analysis/utils/tests/test_vulnerability_intel_sanitizer.py new file mode 100644 index 000000000..5185ccacf --- /dev/null +++ b/src/vuln_analysis/utils/tests/test_vulnerability_intel_sanitizer.py @@ -0,0 +1,176 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for VulnerabilityIntelSanitizer v1.""" + +from exploit_iq_commons.data_models.checker_status import VulnerabilityIntel + +from vuln_analysis.functions.code_agent_graph_defs import ParsedPatch, PatchFile, PatchHunk +from vuln_analysis.utils.vulnerability_intel_sanitizer import VulnerabilityIntelSanitizer + + +def _patch_with_util_c() -> ParsedPatch: + return ParsedPatch( + patch_filename="cve.patch", + files=[ + PatchFile( + source_path="tar/util.c", + target_path="tar/util.c", + hunks=[], + ) + ], + ) + + +def _additive_only_patch() -> ParsedPatch: + return ParsedPatch( + patch_filename="additive.patch", + files=[ + PatchFile( + source_path="a/net/sched/act_ct.c", + target_path="b/net/sched/act_ct.c", + hunks=[ + PatchHunk( + source_start=100, + source_length=0, + target_start=100, + target_length=4, + section_header="tcf_ct_init", + context_lines=[], + removed_lines=[], + added_lines=[ + "if (bind && !(flags & TCA_ACT_FLAGS_AT_INGRESS_OR_CLSACT)) {", + "return -EOPNOTSUPP;", + "}", + ], + ) + ], + ) + ], + ) + + +def _patch_with_removed_lines() -> ParsedPatch: + return ParsedPatch( + patch_filename="mixed.patch", + files=[ + PatchFile( + source_path="a/foo.c", + target_path="b/foo.c", + hunks=[ + PatchHunk( + source_start=10, + source_length=1, + target_start=10, + target_length=1, + section_header="foo", + context_lines=[], + removed_lines=["unsafe_call();"], + added_lines=["safe_call();"], + ) + ], + ) + ], + ) + + +class TestSanitizeAffectedFiles: + def test_clears_affected_files_when_no_patch(self): + raw = VulnerabilityIntel(affected_files=["generator.c", "tar/util.c"]) + result = VulnerabilityIntelSanitizer(None).apply(raw) + assert result.affected_files == [] + + def test_clears_affected_files_when_empty_files(self): + patch = ParsedPatch(patch_filename="empty.patch", files=[]) + raw = VulnerabilityIntel(affected_files=["util.c"]) + result = VulnerabilityIntelSanitizer(patch).apply(raw) + assert result.affected_files == [] + + def test_keeps_matching_basename_with_patch(self): + raw = VulnerabilityIntel(affected_files=["util.c"]) + result = VulnerabilityIntelSanitizer(_patch_with_util_c()).apply(raw) + assert result.affected_files == ["util.c"] + + def test_keeps_full_path_when_basename_matches_patch(self): + raw = VulnerabilityIntel(affected_files=["tar/util.c"]) + result = VulnerabilityIntelSanitizer(_patch_with_util_c()).apply(raw) + assert result.affected_files == ["tar/util.c"] + + def test_drops_non_patch_file_with_patch(self): + raw = VulnerabilityIntel(affected_files=["util.c", "generator.c"]) + result = VulnerabilityIntelSanitizer(_patch_with_util_c()).apply(raw) + assert result.affected_files == ["util.c"] + + +class TestFilterVulnerableFunctions: + def test_drops_function_with_spaces(self): + raw = VulnerabilityIntel( + vulnerable_functions=["parse_header", "rsync compares file checksums"], + ) + result = VulnerabilityIntelSanitizer(_patch_with_util_c()).apply(raw) + assert result.vulnerable_functions == ["parse_header"] + + def test_drops_function_with_spaces_without_patch(self): + raw = VulnerabilityIntel(vulnerable_functions=["rsync compares file checksums"]) + result = VulnerabilityIntelSanitizer(None).apply(raw) + assert result.vulnerable_functions == [] + + +class TestFilterSearchKeywords: + def test_drops_keyword_with_spaces_no_boolean(self): + raw = VulnerabilityIntel( + search_keywords=["s2length", "sum2", "rsync compares file checksums"], + ) + result = VulnerabilityIntelSanitizer(None).apply(raw) + assert result.search_keywords == ["s2length", "sum2"] + + def test_keeps_keyword_with_or(self): + raw = VulnerabilityIntel(search_keywords=["s2length OR sum2"]) + result = VulnerabilityIntelSanitizer(None).apply(raw) + assert result.search_keywords == ["s2length OR sum2"] + + def test_keeps_keyword_with_and(self): + raw = VulnerabilityIntel(search_keywords=["foo AND bar"]) + result = VulnerabilityIntelSanitizer(None).apply(raw) + assert result.search_keywords == ["foo AND bar"] + + +class TestSanitizeAdditiveOnlyPatchIntel: + def test_clears_vulnerable_fields_for_additive_only_patch(self): + raw = VulnerabilityIntel( + vulnerable_functions=["classify"], + vulnerable_variables=["skb"], + vulnerable_patterns=["TC_ACT_CONSUMED"], + fix_patterns=["TCA_ACT_FLAGS_AT_INGRESS_OR_CLSACT"], + ) + result = VulnerabilityIntelSanitizer(_additive_only_patch()).apply(raw) + assert result.vulnerable_functions == [] + assert result.vulnerable_variables == [] + assert result.vulnerable_patterns == [] + assert result.fix_patterns == ["TCA_ACT_FLAGS_AT_INGRESS_OR_CLSACT"] + + def test_keeps_vulnerable_fields_when_patch_has_removed_lines(self): + raw = VulnerabilityIntel( + vulnerable_functions=["classify"], + vulnerable_variables=["skb"], + vulnerable_patterns=["TC_ACT_CONSUMED"], + ) + result = VulnerabilityIntelSanitizer(_patch_with_removed_lines()).apply(raw) + assert result.vulnerable_functions == ["classify"] + assert result.vulnerable_variables == ["skb"] + assert result.vulnerable_patterns == ["TC_ACT_CONSUMED"] + + +class TestRsyncStyleNoPatch: + def test_strips_hallucinated_paths_and_prose(self): + raw = VulnerabilityIntel( + affected_files=["generator.c", "src/foo.c"], + vulnerable_functions=["rsync compares file checksums"], + vulnerable_variables=["s2length", "sum2"], + search_keywords=["s2length", "sum2", "rsync compares file checksums"], + ) + result = VulnerabilityIntelSanitizer(None).apply(raw) + assert result.affected_files == [] + assert result.vulnerable_functions == [] + assert result.vulnerable_variables == ["s2length", "sum2"] + assert result.search_keywords == ["s2length", "sum2"] diff --git a/src/vuln_analysis/utils/tests/test_web_patch_fetcher.py b/src/vuln_analysis/utils/tests/test_web_patch_fetcher.py new file mode 100644 index 000000000..a9900cf8a --- /dev/null +++ b/src/vuln_analysis/utils/tests/test_web_patch_fetcher.py @@ -0,0 +1,557 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for web_patch_fetcher module.""" + +import pytest +from unittest.mock import MagicMock, patch + +from vuln_analysis.functions.code_agent_graph_defs import PatchHunk, PatchFile, ParsedPatch +from vuln_analysis.utils.intel_utils import extract_functions_from_parsed_patch +from vuln_analysis.utils.web_patch_fetcher import ( + WebPatchFetcher, + build_patch_url_from_repo, + _GITHUB_COMMIT_PATTERN, + _GITHUB_PR_PATTERN, + _GITWEB_COMMIT_PATTERN, + _KERNEL_CGIT_COMMIT_PATTERN, + _KERNEL_SHORT_PATTERN, +) + + +class TestUrlPatterns: + """Test URL pattern matching.""" + + def test_github_commit_pattern(self): + """Test GitHub commit URL pattern matching.""" + url = "https://github.com/libarchive/libarchive/commit/a2a73a8f14b3208c7f6acbbc93265254a7c1efd0" + match = _GITHUB_COMMIT_PATTERN.match(url) + assert match is not None + assert match.group(1) == "libarchive/libarchive" + assert match.group(2) == "a2a73a8f14b3208c7f6acbbc93265254a7c1efd0" + + def test_github_commit_pattern_short_sha(self): + """Test GitHub commit URL with short SHA.""" + url = "https://github.com/curl/curl/commit/39d1976b7f" + match = _GITHUB_COMMIT_PATTERN.match(url) + assert match is not None + assert match.group(1) == "curl/curl" + assert match.group(2) == "39d1976b7f" + + def test_github_pr_pattern(self): + """Test GitHub PR URL pattern matching.""" + url = "https://github.com/libarchive/libarchive/pull/2934" + match = _GITHUB_PR_PATTERN.match(url) + assert match is not None + assert match.group(1) == "libarchive/libarchive" + assert match.group(2) == "2934" + + def test_kernel_cgit_commit_pattern(self): + """Test kernel.org cgit commit URL pattern.""" + url = ( + "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" + "/commit/?id=f342de4e2f33e0e39165d8639387aa6c19dff660" + ) + match = _KERNEL_CGIT_COMMIT_PATTERN.match(url) + assert match is not None + expected_repo = "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" + assert match.group(1) == expected_repo + assert match.group(2) == "f342de4e2f33e0e39165d8639387aa6c19dff660" + + def test_kernel_short_pattern(self): + """Test kernel.org short URL pattern.""" + url = "https://git.kernel.org/stable/c/096bb5b43edf755bc4477e64004fa3a20539ec2f" + match = _KERNEL_SHORT_PATTERN.match(url) + assert match is not None + assert match.group(1) == "stable" + assert match.group(2) == "096bb5b43edf755bc4477e64004fa3a20539ec2f" + + def test_gitweb_commit_pattern(self): + """Test gitweb commit URL pattern.""" + url = "https://git.samba.org/?p=rsync.git;a=commit;h=6c8ca91c731b7bf2b081694bda85b7dadc2b7aff" + match = _GITWEB_COMMIT_PATTERN.match(url) + assert match is not None + assert match.group(1) == "https://git.samba.org" + assert match.group(2) == "rsync.git" + assert match.group(3) == "6c8ca91c731b7bf2b081694bda85b7dadc2b7aff" + + +class TestWebPatchFetcherUrlResolution: + """Test URL resolution in WebPatchFetcher.""" + + @pytest.fixture + def fetcher(self): + """Create a WebPatchFetcher with a mock session.""" + mock_session = MagicMock() + return WebPatchFetcher(session=mock_session) + + def test_resolve_github_commit_url(self, fetcher): + """Test resolving GitHub commit URL to patch URL.""" + url = "https://github.com/curl/curl/commit/39d1976b7f" + resolved = fetcher._resolve_github_url(url) + assert resolved is not None + assert resolved.patch_url == "https://github.com/curl/curl/commit/39d1976b7f.patch" + assert resolved.platform == "github" + assert resolved.url_type == "commit" + assert resolved.repo_url == "https://github.com/curl/curl" + assert resolved.commit_sha == "39d1976b7f" + + def test_resolve_github_pr_url(self, fetcher): + """Test resolving GitHub PR URL to patch URL.""" + url = "https://github.com/libarchive/libarchive/pull/2934" + resolved = fetcher._resolve_github_url(url) + assert resolved is not None + assert resolved.patch_url == "https://github.com/libarchive/libarchive/pull/2934.patch" + assert resolved.platform == "github" + assert resolved.url_type == "pull" + assert resolved.repo_url == "https://github.com/libarchive/libarchive" + assert resolved.commit_sha == "PR-2934" + + def test_resolve_ubuntu_style_url(self, fetcher): + """Test resolving Ubuntu-style 'upstream:' URL.""" + url = "upstream: https://github.com/curl/curl/commit/39d1976b7f" + resolved = fetcher._resolve_github_url(url) + assert resolved is not None + assert resolved.patch_url == "https://github.com/curl/curl/commit/39d1976b7f.patch" + assert resolved.platform == "github" + assert resolved.url_type == "commit" + + def test_resolve_kernel_cgit_commit_url(self, fetcher): + """Test resolving kernel.org cgit commit URL.""" + url = ( + "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" + "/commit/?id=f342de4e2f33" + ) + resolved = fetcher._resolve_kernel_org_url(url) + assert resolved is not None + expected_patch = ( + "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" + "/patch/?id=f342de4e2f33" + ) + assert resolved.patch_url == expected_patch + assert resolved.platform == "kernel.org" + assert resolved.url_type == "commit" + assert resolved.commit_sha == "f342de4e2f33" + + def test_resolve_kernel_short_stable_url(self, fetcher): + """Test resolving kernel.org short stable URL.""" + url = "https://git.kernel.org/stable/c/096bb5b43edf" + resolved = fetcher._resolve_kernel_org_url(url) + assert resolved is not None + expected_patch = ( + "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git" + "/patch/?id=096bb5b43edf" + ) + assert resolved.patch_url == expected_patch + assert resolved.platform == "kernel.org" + assert resolved.commit_sha == "096bb5b43edf" + + def test_resolve_kernel_short_torvalds_url(self, fetcher): + """Test resolving kernel.org short torvalds URL.""" + url = "https://git.kernel.org/torvalds/c/abc123def456" + resolved = fetcher._resolve_kernel_org_url(url) + assert resolved is not None + expected_patch = ( + "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" + "/patch/?id=abc123def456" + ) + assert resolved.patch_url == expected_patch + assert resolved.platform == "kernel.org" + + def test_resolve_unknown_url_returns_none(self, fetcher): + """Test that unknown URLs return None.""" + url = "https://example.com/some/path" + resolved = fetcher._resolve_to_patch_url(url) + assert resolved is None + + def test_resolve_gitweb_commit_url(self, fetcher): + """Test resolving git.samba.org gitweb commit URL.""" + url = ( + "https://git.samba.org/?p=rsync.git;a=commit;" + "h=6c8ca91c731b7bf2b081694bda85b7dadc2b7aff" + ) + resolved = fetcher._resolve_gitweb_url(url) + assert resolved is not None + assert resolved.platform == "gitweb" + assert resolved.patch_url == ( + "https://git.samba.org/?p=rsync.git;a=patch;" + "h=6c8ca91c731b7bf2b081694bda85b7dadc2b7aff" + ) + assert resolved.repo_url == "https://git.samba.org/rsync.git" + assert resolved.commit_sha == "6c8ca91c731b7bf2b081694bda85b7dadc2b7aff" + + def test_resolve_gitweb_patch_url(self, fetcher): + """Test resolving gitweb patch URL directly.""" + url = ( + "https://git.samba.org/?p=rsync.git;a=patch;" + "h=6c8ca91c731b7bf2b081694bda85b7dadc2b7aff" + ) + resolved = fetcher._resolve_gitweb_url(url) + assert resolved is not None + assert resolved.platform == "gitweb" + assert resolved.patch_url == url + + +class TestBuildPatchUrlFromRepo: + """Test patch URL construction from OSV repo + commit.""" + + def test_samba_rsync_gitweb(self): + repo = "https://git.samba.org/rsync.git" + sha = "6c8ca91c731b7bf2b081694bda85b7dadc2b7aff" + patch_url = build_patch_url_from_repo(repo, sha) + assert patch_url == f"https://git.samba.org/?p=rsync.git;a=patch;h={sha}" + + def test_kernel_cgit_path_info(self): + repo = "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" + sha = "f342de4e2f33" + patch_url = build_patch_url_from_repo(repo, sha) + assert patch_url == f"{repo}/patch/?id={sha}" + + def test_github_repo(self): + repo = "https://github.com/openssl/openssl" + sha = "abc123def456" + patch_url = build_patch_url_from_repo(repo, sha) + assert patch_url == "https://github.com/openssl/openssl/commit/abc123def456.patch" + + +class TestPrioritizeAndDedupe: + """Test URL prioritization and deduplication.""" + + @pytest.fixture + def fetcher(self): + """Create a WebPatchFetcher with a mock session.""" + mock_session = MagicMock() + return WebPatchFetcher(session=mock_session) + + def test_prioritize_commit_over_pull(self, fetcher): + """Test that commit URLs are prioritized over PR URLs.""" + candidates = { + "ghsa": [ + "https://github.com/foo/bar/pull/123", + "https://github.com/foo/bar/commit/abc123", + ], + } + result = fetcher._prioritize_and_dedupe(candidates) + assert len(result) == 2 + assert result[0][2] == "commit" # commit first + assert result[1][2] == "pull" # PR second + + def test_prioritize_ubuntu_patches_first(self, fetcher): + """Test that ubuntu_patches source is prioritized for commits.""" + candidates = { + "ghsa": ["https://github.com/foo/bar/commit/ghsa123"], + "ubuntu_patches": ["upstream: https://github.com/foo/bar/commit/ubuntu123"], + } + result = fetcher._prioritize_and_dedupe(candidates) + assert len(result) == 2 + assert result[0][1] == "ubuntu_patches" # ubuntu_patches first + assert result[1][1] == "ghsa" + + def test_deduplicate_same_url(self, fetcher): + """Test that duplicate URLs are deduplicated.""" + candidates = { + "ghsa": ["https://github.com/foo/bar/commit/abc123"], + "nvd": ["https://github.com/foo/bar/commit/abc123"], + "rhsa": ["https://github.com/foo/bar/commit/abc123"], + } + result = fetcher._prioritize_and_dedupe(candidates) + assert len(result) == 1 # Only one unique URL + + def test_deduplicate_with_trailing_slash(self, fetcher): + """Test deduplication handles trailing slashes.""" + candidates = { + "ghsa": ["https://github.com/foo/bar/commit/abc123"], + "nvd": ["https://github.com/foo/bar/commit/abc123/"], + } + result = fetcher._prioritize_and_dedupe(candidates) + assert len(result) == 1 + + def test_kernel_org_urls_prioritized_as_commits(self, fetcher): + """Test that kernel.org URLs are recognized as commit URLs.""" + candidates = { + "nvd": [ + "https://git.kernel.org/stable/c/abc123", + "https://github.com/foo/bar/pull/456", + ], + } + result = fetcher._prioritize_and_dedupe(candidates) + assert len(result) == 2 + assert result[0][2] == "commit" # kernel.org first (commit) + assert result[1][2] == "pull" # PR second + + def test_empty_candidates(self, fetcher): + """Test handling of empty candidates.""" + candidates = {} + result = fetcher._prioritize_and_dedupe(candidates) + assert result == [] + + def test_skip_non_matching_urls(self, fetcher): + """Test that non-commit/PR URLs are skipped.""" + candidates = { + "ghsa": [ + "https://github.com/advisories/GHSA-xxxx-xxxx-xxxx", + "https://example.com/something", + ], + } + result = fetcher._prioritize_and_dedupe(candidates) + assert result == [] # Advisory URLs don't match commit/PR patterns + + +class TestNormalizeUrl: + """Test URL normalization for deduplication.""" + + @pytest.fixture + def fetcher(self): + mock_session = MagicMock() + return WebPatchFetcher(session=mock_session) + + def test_normalize_removes_trailing_slash(self, fetcher): + url = "https://github.com/foo/bar/commit/abc123/" + normalized = fetcher._normalize_url_for_dedupe(url) + assert not normalized.endswith("/") + + def test_normalize_lowercases(self, fetcher): + url = "https://GitHub.com/Foo/Bar/Commit/ABC123" + normalized = fetcher._normalize_url_for_dedupe(url) + assert normalized == normalized.lower() + + def test_normalize_removes_patch_suffix(self, fetcher): + url = "https://github.com/foo/bar/commit/abc123.patch" + normalized = fetcher._normalize_url_for_dedupe(url) + assert not normalized.endswith(".patch") + + +class TestIsCommitUrl: + """Test commit URL detection.""" + + @pytest.fixture + def fetcher(self): + mock_session = MagicMock() + return WebPatchFetcher(session=mock_session) + + def test_github_commit_is_commit_url(self, fetcher): + assert fetcher._is_commit_url("https://github.com/foo/bar/commit/abc123") + + def test_kernel_cgit_is_commit_url(self, fetcher): + assert fetcher._is_commit_url("https://git.kernel.org/pub/scm/linux.git/commit/?id=abc123") + + def test_kernel_short_is_commit_url(self, fetcher): + assert fetcher._is_commit_url("https://git.kernel.org/stable/c/abc123") + + def test_github_pr_is_not_commit_url(self, fetcher): + assert not fetcher._is_commit_url("https://github.com/foo/bar/pull/123") + + def test_advisory_is_not_commit_url(self, fetcher): + assert not fetcher._is_commit_url("https://github.com/advisories/GHSA-xxxx") + + def test_gitweb_commit_is_commit_url(self, fetcher): + url = "https://git.samba.org/?p=rsync.git;a=commit;h=6c8ca91c731b7bf2b081694bda85b7dadc2b7aff" + assert fetcher._is_commit_url(url) + + +class TestIsPrUrl: + """Test PR URL detection.""" + + @pytest.fixture + def fetcher(self): + mock_session = MagicMock() + return WebPatchFetcher(session=mock_session) + + def test_github_pr_is_pr_url(self, fetcher): + assert fetcher._is_pr_url("https://github.com/foo/bar/pull/123") + + def test_github_commit_is_not_pr_url(self, fetcher): + assert not fetcher._is_pr_url("https://github.com/foo/bar/commit/abc123") + + +@pytest.mark.asyncio +class TestFetchFromIntelRefs: + """Test fetch_from_intel_refs integration.""" + + @pytest.fixture + def mock_session(self): + return MagicMock() + + async def test_fetch_from_intel_refs_empty_candidates(self, mock_session): + """Test handling of empty candidates.""" + fetcher = WebPatchFetcher(session=mock_session) + result = await fetcher.fetch_from_intel_refs({}, "CVE-2024-1234") + assert result is None + + async def test_fetch_from_intel_refs_returns_first_success(self, mock_session): + """Test that first successful fetch is returned.""" + fetcher = WebPatchFetcher(session=mock_session) + + # Mock the fetch_from_url method + with patch.object(fetcher, 'fetch_from_url') as mock_fetch: + mock_result = MagicMock() + mock_result.parsed_patch = MagicMock() + mock_fetch.return_value = mock_result + + candidates = { + "ubuntu_patches": ["upstream: https://github.com/foo/bar/commit/abc123"], + } + result = await fetcher.fetch_from_intel_refs(candidates, "CVE-2024-1234") + + assert result is mock_result + mock_fetch.assert_called_once() + + async def test_fetch_from_intel_refs_tries_multiple_urls(self, mock_session): + """Test that multiple URLs are tried on failure.""" + fetcher = WebPatchFetcher(session=mock_session) + + with patch.object(fetcher, 'fetch_from_url') as mock_fetch: + # First call returns None (failure), second returns success + mock_result = MagicMock() + mock_result.parsed_patch = MagicMock() + mock_fetch.side_effect = [None, mock_result] + + candidates = { + "ubuntu_patches": ["upstream: https://github.com/foo/bar/commit/abc123"], + "ghsa": ["https://github.com/foo/bar/commit/def456"], + } + result = await fetcher.fetch_from_intel_refs(candidates, "CVE-2024-1234") + + assert result is mock_result + assert mock_fetch.call_count == 2 + + +class TestPatchHunkSectionHeader: + """Tests for PatchHunk.section_header field (fix 1).""" + + def test_section_header_default_empty(self): + hunk = PatchHunk(source_start=1, source_length=5, target_start=1, target_length=5) + assert hunk.section_header == "" + + def test_section_header_populated(self): + hunk = PatchHunk( + source_start=10, source_length=5, target_start=10, target_length=5, + section_header="void processRequest(HttpServletRequest req)", + ) + assert hunk.section_header == "void processRequest(HttpServletRequest req)" + + def test_section_header_in_extract_functions(self): + """section_header containing a function signature should be picked up by extract_functions_from_parsed_patch.""" + patch = ParsedPatch( + patch_filename="fix.patch", + files=[PatchFile( + source_path="a/src/main.go", + target_path="b/src/main.go", + hunks=[PatchHunk( + source_start=10, source_length=3, target_start=10, target_length=3, + section_header="func handleRequest(w http.ResponseWriter, r *http.Request)", + added_lines=[" return nil"], + )], + )], + ) + funcs = extract_functions_from_parsed_patch(patch, ecosystem="go") + assert "handleRequest" in funcs + + def test_section_header_only_no_added_lines(self): + """Function name should be extracted from section_header even when added_lines have no definitions.""" + patch = ParsedPatch( + patch_filename="fix.patch", + files=[PatchFile( + source_path="a/Foo.java", + target_path="b/Foo.java", + hunks=[PatchHunk( + source_start=50, source_length=2, target_start=50, target_length=2, + section_header="public String getVersion()", + added_lines=[" return VERSION;"], + )], + )], + ) + funcs = extract_functions_from_parsed_patch(patch, ecosystem="java") + assert "getVersion" in funcs + + def test_section_header_skipped_for_test_files(self): + """Test files should be skipped even if section_header has a function.""" + patch = ParsedPatch( + patch_filename="fix.patch", + files=[PatchFile( + source_path="a/FooTest.java", + target_path="b/FooTest.java", + hunks=[PatchHunk( + source_start=10, source_length=2, target_start=10, target_length=2, + section_header="public void shouldParseInput()", + added_lines=[" assertEquals(1, 1);"], + )], + )], + ) + funcs = extract_functions_from_parsed_patch(patch, ecosystem="java") + assert len(funcs) == 0 + + def test_section_header_empty_string_not_matched(self): + """Empty section_header should not contribute any function names.""" + patch = ParsedPatch( + patch_filename="fix.patch", + files=[PatchFile( + source_path="a/main.c", + target_path="b/main.c", + hunks=[PatchHunk( + source_start=1, source_length=1, target_start=1, target_length=1, + section_header="", + added_lines=["int x = 0;"], + )], + )], + ) + funcs = extract_functions_from_parsed_patch(patch, ecosystem="c") + assert len(funcs) == 0 + + def test_python_section_header(self): + patch = ParsedPatch( + patch_filename="fix.patch", + files=[PatchFile( + source_path="a/utils.py", + target_path="b/utils.py", + hunks=[PatchHunk( + source_start=20, source_length=3, target_start=20, target_length=3, + section_header="def validate_input(data):", + added_lines=[" if not data:"], + )], + )], + ) + funcs = extract_functions_from_parsed_patch(patch, ecosystem="python") + assert "validate_input" in funcs + + +class TestCleanTargetPath: + """Tests for PatchFile.clean_target_path property (fix 3).""" + + def test_strip_b_prefix(self): + pf = PatchFile(source_path="a/foo.c", target_path="b/foo.c", hunks=[]) + assert pf.clean_target_path == "foo.c" + + def test_strip_a_prefix(self): + pf = PatchFile(source_path="a/foo.c", target_path="a/foo.c", hunks=[]) + assert pf.clean_target_path == "foo.c" + + def test_no_prefix(self): + pf = PatchFile(source_path="foo.c", target_path="foo.c", hunks=[]) + assert pf.clean_target_path == "foo.c" + + def test_nested_path_b_prefix(self): + pf = PatchFile(source_path="a/src/main/Bar.java", target_path="b/src/main/Bar.java", hunks=[]) + assert pf.clean_target_path == "src/main/Bar.java" + + def test_b_in_path_not_prefix(self): + """A 'b/' that is part of the path (not a prefix) should not be stripped.""" + pf = PatchFile(source_path="a/lib/b/file.py", target_path="lib/b/file.py", hunks=[]) + assert pf.clean_target_path == "lib/b/file.py" + + def test_devnull_path(self): + """New files have /dev/null as source — target should still strip prefix.""" + pf = PatchFile(source_path="/dev/null", target_path="b/new_file.go", hunks=[], is_new_file=True) + assert pf.clean_target_path == "new_file.go" diff --git a/src/vuln_analysis/utils/token_utils.py b/src/vuln_analysis/utils/token_utils.py new file mode 100644 index 000000000..fb29bbf6b --- /dev/null +++ b/src/vuln_analysis/utils/token_utils.py @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Token counting and truncation utilities for agent context management.""" + +import tiktoken + +from vuln_analysis.tools.tool_names import ToolNames + +_tiktoken_enc = tiktoken.get_encoding("cl100k_base") + + +def count_tokens(text: str) -> int: + """Count the number of tokens in a text string using the cl100k_base encoding. + + Falls back to a rough character-based estimate if encoding fails. + """ + try: + return len(_tiktoken_enc.encode(text)) + except Exception: + return len(text) // 4 + + +def estimate_tokens(runtime_prompt: str, messages: list, observation) -> int: + """Estimate total token count of the current agent context (prompt + messages + observation).""" + parts = [runtime_prompt] + for msg in messages: + if hasattr(msg, "content") and isinstance(msg.content, str): + parts.append(msg.content) + if observation is not None: + for item in (observation.memory or []): + parts.append(item) + for item in (observation.results or []): + parts.append(item) + return count_tokens("\n".join(parts)) + + +def truncate_tool_output(tool_output: str, tool_name: str, max_tokens: int = 400) -> str: + """Truncate tool output to fit within a token budget. + + Uses tool-specific strategies: Call Chain Analyzer keeps the header, + Code Keyword Search preserves section headers, and the default strategy + keeps head (70%) and tail (30%) of the output. + """ + token_count = count_tokens(tool_output) + if token_count <= max_tokens: + return tool_output + + lines = tool_output.split('\n') + + if tool_name == ToolNames.CALL_CHAIN_ANALYZER: + head = '\n'.join(lines[:3]) + remaining = token_count - count_tokens(head) + return f"{head}\n[... truncated {remaining} tokens ...]" + + if tool_name == ToolNames.CODE_KEYWORD_SEARCH: + kept_lines = [] + kept_tokens = 0 + for line in lines: + is_header = line.startswith("---") or "Main application" in line or "library dependencies" in line or line.strip() == "" + line_tokens = count_tokens(line) + if is_header or kept_tokens + line_tokens <= max_tokens: + kept_lines.append(line) + kept_tokens += line_tokens + if kept_tokens >= max_tokens: + break + kept_lines.append(f"[... truncated {token_count - kept_tokens} tokens ...]") + return '\n'.join(kept_lines) + + if tool_name == ToolNames.SOURCE_GREP: + blocks = tool_output.split('\n--\n') + kept_blocks, kept_tokens = [], 0 + for block in blocks: + block_tokens = count_tokens(block) + if kept_tokens + block_tokens > max_tokens: + break + kept_blocks.append(block) + kept_tokens += block_tokens + truncated_count = len(blocks) - len(kept_blocks) + result = '\n--\n'.join(kept_blocks) + if truncated_count > 0: + result += f"\n[... truncated {truncated_count} more matches ...]" + return result + + head_budget = int(max_tokens * 0.7) + tail_budget = max_tokens - head_budget + head_lines, head_tokens = [], 0 + for line in lines: + lt = count_tokens(line) + if head_tokens + lt > head_budget: + break + head_lines.append(line) + head_tokens += lt + tail_lines, tail_tokens = [], 0 + for line in reversed(lines): + lt = count_tokens(line) + if tail_tokens + lt > tail_budget: + break + tail_lines.insert(0, line) + tail_tokens += lt + truncated = token_count - head_tokens - tail_tokens + return '\n'.join(head_lines) + f"\n[... truncated {truncated} tokens ...]\n" + '\n'.join(tail_lines) + + +def truncate_tool_output_list( + tool_output: str, + tool_name: str, + max_tokens: int = 1000, + max_chunks: int = 2 +) -> list[str]: + """Split tool output into chunks, each up to max_tokens. Returns up to max_chunks. + + Use this when you want to process large outputs in multiple passes rather than + truncating to a single smaller output. Each chunk can be processed separately + and findings merged. + + Calls truncate_tool_output internally to apply tool-specific truncation logic. + """ + total_tokens = count_tokens(tool_output) + if total_tokens <= max_tokens: + return [truncate_tool_output(tool_output, tool_name, max_tokens)] + + # Split by lines and create raw chunks + lines = tool_output.split('\n') + chunks = [] + current_lines = [] + current_tokens = 0 + + for line in lines: + line_tokens = count_tokens(line) + if current_tokens + line_tokens > max_tokens and current_lines: + # Apply tool-specific truncation to each chunk + raw_chunk = '\n'.join(current_lines) + chunks.append(truncate_tool_output(raw_chunk, tool_name, max_tokens)) + if len(chunks) >= max_chunks: + break + current_lines = [] + current_tokens = 0 + current_lines.append(line) + current_tokens += line_tokens + + # Handle remaining lines as final chunk + if current_lines and len(chunks) < max_chunks: + raw_chunk = '\n'.join(current_lines) + chunks.append(truncate_tool_output(raw_chunk, tool_name, max_tokens)) + + return chunks if chunks else [truncate_tool_output(tool_output, tool_name, max_tokens)] diff --git a/src/vuln_analysis/utils/version_check.py b/src/vuln_analysis/utils/version_check.py new file mode 100644 index 000000000..818c768fe --- /dev/null +++ b/src/vuln_analysis/utils/version_check.py @@ -0,0 +1,688 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Version vulnerability gate: route to univers (easy) or LLM (hard).""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from enum import Enum +from typing import Any, Awaitable, Callable, Literal + +from pydantic import BaseModel, Field + +from univers import versions + +from exploit_iq_commons.data_models.checker_status import EnumIdentifyResult +from exploit_iq_commons.data_models.cve_intel import CveIntel +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.utils.package_identifier import ( + _extract_dist_tag, + _interpret_fix_state, + _match_package_state_for_distro, +) + +logger = LoggingFactory.get_agent_logger(__name__) + +ECOSYSTEM_RPM = "rpm" +INTEL_VALUE_NOT_APPLICABLE = "N/A" + +_LETTER_SUFFIX_VERSION_RE = re.compile(r"^\d+\.\d+\.\d+[a-z]", re.IGNORECASE) +_PLAIN_SEMVER_RE = re.compile(r"^\d+\.\d+") +_MODULE_NEVRA_MARKER = ".module+" +_MULTI_BRANCH_DESCRIPTION_RE = re.compile( + r"before\s+.+,\s+.+\s+and\s+", + re.IGNORECASE, +) + +KEY_FIRST_PATCHED = "first_patched" +KEY_VULNERABLE_RANGE = "vulnerable_range" +KEY_VERSION_START_INCL = "version_start_incl" +KEY_VERSION_START_EXCL = "version_start_excl" +KEY_VERSION_END_INCL = "version_end_incl" +KEY_VERSION_END_EXCL = "version_end_excl" +KEY_FIX_STATE = "fix_state" + +NVD_RANGE_KEYS = ( + KEY_VERSION_START_EXCL, + KEY_VERSION_END_EXCL, + KEY_VERSION_START_INCL, + KEY_VERSION_END_INCL, +) + +# ============================================================ +# Stdlib/Toolchain Vulnerability Prompt (LLM Hard Path) +# ============================================================ + +STDLIB_VULNERABILITY_PROMPT = """You are a security expert analyzing whether a CVE affects a project's language toolchain/stdlib. + +CVE ID: {cve_id} +Ecosystem: {ecosystem} + +CVE Description: +{description} + +Project's toolchain version: {toolchain_version} +(This is the language version from go.mod, pyproject.toml, package.json, or pom.xml) + +Context: +- This CVE had no matching packages in structured vulnerability databases (GHSA/NVD configurations were empty) +- This suggests it may be a stdlib/toolchain vulnerability affecting the language runtime itself +- Common examples: Go crypto/x509, net/http; Python ssl, urllib; Java security providers + +Analysis Tasks: +1. Identify if this CVE affects a stdlib/runtime component (not a third-party library) +2. Determine which versions of the toolchain are affected based on the description +3. Compare the project's toolchain version against the affected range + +Rules: +1. Pay close attention to statements like "affects only Go 1.24 release candidates" or "fixed in Python 3.11.4" +2. If the CVE only affects pre-release/RC versions and the project uses a stable release, it's NOT vulnerable +3. If the project's toolchain version is at or past the fix version, it's NOT vulnerable +4. If you cannot determine the affected versions from the description, assume vulnerable (fail safe) + +Respond with: +- is_vulnerable: whether the project's toolchain version is affected +- affected_component: the stdlib component (e.g., "crypto/x509", "net/http", "ssl") +- reason: clear explanation of your analysis""" + +ComparisonMode = Literal["first_patched", "nvd_range", "none", "ambiguous"] + + +class VersionCheckPath(str, Enum): + VENDOR_NOT_AFFECTED = "vendor_not_affected" + EASY = "easy" + HARD = "hard" + + +class HardPathReason(str, Enum): + LETTER_SUFFIX_VERSION = "letter_suffix_version" + MODULE_NEVRA = "module_nevra" + MULTI_BRANCH_DESCRIPTION = "multi_branch_description" + CROSS_DIST_INTEL = "cross_dist_intel" + AXIS_MISMATCH = "axis_mismatch" + PARSE_ERROR = "parse_error" + COMPARATOR_DISAGREEMENT = "comparator_disagreement" + NO_STRUCTURED_BOUNDS = "no_structured_bounds" + NON_SEMVER_INSTALLED = "non_semver_installed" + AMBIGUOUS_INTEL = "ambiguous_intel" + + +@dataclass(frozen=True) +class VersionCheckClassification: + path: VersionCheckPath + hard_reason: HardPathReason | None = None + + +class StdlibVulnerabilityResult(BaseModel): + """Structured output for LLM stdlib/toolchain vulnerability analysis. + + Used when no specific packages match from intel (GHSA/NVD empty) but the CVE + may affect the language runtime/stdlib (e.g., Go crypto/x509, Python ssl). + """ + + is_vulnerable: bool = Field( + description="True if the project's toolchain version is affected by this CVE" + ) + affected_component: str = Field( + description="The stdlib/runtime component affected (e.g., 'crypto/x509', 'net/http')" + ) + reason: str = Field( + description="Explanation of why the toolchain is or isn't vulnerable" + ) + + +# Type alias for the LLM checker callable (dependency injection) +StdlibLLMChecker = Callable[ + [str], Awaitable[StdlibVulnerabilityResult] +] + + +def get_cve_description(intel: CveIntel) -> str: + """Collect the best available CVE description text for hard-path LLM prompts. + + Combines the primary description (NVD > GHSA > RHSA details) with the RHSA + statement field when available. The RHSA statement often contains critical + context about affected versions/configurations (e.g., "This vulnerability + affects only the Go 1.24 release candidates"). + """ + parts: list[str] = [] + + if intel.nvd and intel.nvd.cve_description: + parts.append(intel.nvd.cve_description) + elif intel.ghsa: + if intel.ghsa.description: + parts.append(intel.ghsa.description) + elif intel.ghsa.summary: + parts.append(intel.ghsa.summary) + elif intel.rhsa and intel.rhsa.details: + parts.append(" ".join(intel.rhsa.details)) + + if intel.rhsa and intel.rhsa.statement: + parts.append(f"RHSA Statement: {intel.rhsa.statement}") + + return "\n\n".join(parts) + + +async def check_stdlib_vulnerability( + cve_intel: CveIntel, + ecosystem: str, + toolchain_version: str, + llm_checker: StdlibLLMChecker, +) -> tuple[bool, str, str]: + """Check if a stdlib/toolchain CVE affects the project using LLM analysis. + + This is the LLM hard-path for stdlib vulnerabilities. Called when no packages + matched from structured intel (GHSA/NVD empty), suggesting this may be a + language runtime vulnerability (e.g., Go crypto/x509, Python ssl). + + Args: + cve_intel: The CVE intel object containing description and metadata + ecosystem: The ecosystem identifier (e.g., 'go', 'pypi', 'npm', 'maven') + toolchain_version: The project's language version (e.g., '1.21' for Go) + llm_checker: Async callable that invokes the LLM with structured output. + Signature: (prompt: str) -> StdlibVulnerabilityResult + + Returns: + Tuple of (is_vulnerable, affected_component, reason) + """ + description = get_cve_description(cve_intel) + prompt = STDLIB_VULNERABILITY_PROMPT.format( + cve_id=cve_intel.vuln_id, + ecosystem=ecosystem, + description=description or "No description available", + toolchain_version=toolchain_version or "unknown", + ) + + try: + result = await llm_checker(prompt) + logger.info( + "Stdlib vulnerability check for %s: vulnerable=%s, component=%s", + cve_intel.vuln_id, + result.is_vulnerable, + result.affected_component, + ) + return result.is_vulnerable, result.affected_component, result.reason + + except Exception as e: + logger.error( + "LLM stdlib vulnerability check failed for %s", + cve_intel.vuln_id, + exc_info=True, + ) + return True, "unknown", f"LLM check failed, assuming vulnerable: {e}" + + +def extract_dist_from_installed(installed_version: str) -> str | None: + """Extract elN dist tag from an installed version or NVR string.""" + return _extract_dist_tag(installed_version) + + +def classify_version_check( + *, + installed_version: str, + version_info: dict[str, Any], + description: str | None, + ecosystem: str, + package_name: str, + intel: CveIntel | None = None, +) -> VersionCheckClassification: + """Decide which version-check path to use. Does not compare versions.""" + if not installed_version or installed_version == "unknown": + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.NO_STRUCTURED_BOUNDS, + ) + + vendor_classification = _classify_vendor_fix_state( + version_info=version_info, + package_name=package_name, + installed_version=installed_version, + intel=intel, + ) + if vendor_classification is not None: + return vendor_classification + + if _description_has_multi_branch(description): + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.MULTI_BRANCH_DESCRIPTION, + ) + + if ecosystem == ECOSYSTEM_RPM: + if _installed_version_is_non_perfect(installed_version, ecosystem): + return _hard_for_installed(installed_version, ecosystem) + + comparison_mode = _comparison_mode(version_info) + if comparison_mode == "none": + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.NO_STRUCTURED_BOUNDS, + ) + if comparison_mode == "ambiguous": + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.AMBIGUOUS_INTEL, + ) + + if comparison_mode == "first_patched": + return _classify_first_patched_match( + installed_version=installed_version, + version_info=version_info, + ecosystem=ecosystem, + ) + + return _classify_nvd_range_match( + installed_version=installed_version, + version_info=version_info, + ecosystem=ecosystem, + ) + + +def deterministic_version_check( + *, + installed_version: str, + version_info: dict[str, Any], + ecosystem: str, +) -> tuple[bool, str]: + """Compare versions with univers. Caller must have classified as EASY first.""" + comparison_mode = _comparison_mode(version_info) + if comparison_mode == "first_patched": + return _deterministic_first_patched_check( + installed_version=installed_version, + version_info=version_info, + ecosystem=ecosystem, + ) + if comparison_mode == "nvd_range": + return _deterministic_nvd_range_check( + installed_version=installed_version, + version_info=version_info, + ecosystem=ecosystem, + ) + + msg = "deterministic_version_check called without a supported comparison mode" + logger.error(msg) + raise ValueError(msg) + + +def _deterministic_first_patched_check( + *, + installed_version: str, + version_info: dict[str, Any], + ecosystem: str, +) -> tuple[bool, str]: + first_patched = _intel_value(version_info.get(KEY_FIRST_PATCHED)) + if not first_patched: + raise ValueError("first_patched comparison requested but value is missing") + + version_func = _version_func_for_ecosystem(ecosystem) + installed = version_func(installed_version) + reference = version_func(first_patched) + is_vulnerable = installed < reference + return is_vulnerable, ( + f"Installed version {installed_version} " + f"{'<' if is_vulnerable else '>='} " + f"first_patched {first_patched}" + ) + + +def _deterministic_nvd_range_check( + *, + installed_version: str, + version_info: dict[str, Any], + ecosystem: str, +) -> tuple[bool, str]: + version_range = [_intel_value(version_info.get(key)) for key in NVD_RANGE_KEYS] + in_range = _check_version_in_range(installed_version, version_range, ecosystem) + return in_range, ( + f"Installed version {installed_version} " + f"{'in' if in_range else 'outside'} " + f"NVD range" + ) + + +def _classify_vendor_fix_state( + *, + version_info: dict[str, Any], + package_name: str, + installed_version: str, + intel: CveIntel | None, +) -> VersionCheckClassification | None: + fix_state = version_info.get(KEY_FIX_STATE) + if fix_state and _interpret_fix_state(fix_state) == EnumIdentifyResult.NO: + return VersionCheckClassification(VersionCheckPath.VENDOR_NOT_AFFECTED) + + if intel is None or intel.rhsa is None or not intel.rhsa.package_state: + return None + + target_dist = extract_dist_from_installed(installed_version) + matched = _match_package_state_for_distro( + intel.rhsa.package_state, + package_name, + target_dist, + ) + if matched is None: + return None + + if _interpret_fix_state(matched.fix_state) == EnumIdentifyResult.NO: + return VersionCheckClassification(VersionCheckPath.VENDOR_NOT_AFFECTED) + + return None + + +def _classify_first_patched_match( + *, + installed_version: str, + version_info: dict[str, Any], + ecosystem: str, +) -> VersionCheckClassification: + first_patched = _intel_value(version_info.get(KEY_FIRST_PATCHED)) + if not first_patched: + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.NO_STRUCTURED_BOUNDS, + ) + + if ecosystem == ECOSYSTEM_RPM: + if _bound_is_non_perfect(first_patched, ecosystem): + return _hard_for_bound(first_patched, ecosystem) + rpm_hard = _classify_rpm_perfect_match(installed_version, first_patched) + if rpm_hard is not None: + return rpm_hard + + if not _versions_parse_cleanly(installed_version, first_patched, ecosystem): + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.PARSE_ERROR, + ) + + return VersionCheckClassification(VersionCheckPath.EASY) + + +def _classify_nvd_range_match( + *, + installed_version: str, + version_info: dict[str, Any], + ecosystem: str, +) -> VersionCheckClassification: + bound_values = [ + _intel_value(version_info.get(key)) + for key in NVD_RANGE_KEYS + if _intel_value(version_info.get(key)) + ] + if not bound_values: + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.NO_STRUCTURED_BOUNDS, + ) + + if ecosystem == ECOSYSTEM_RPM: + for bound in bound_values: + if _bound_is_non_perfect(bound, ecosystem): + return _hard_for_bound(bound, ecosystem) + for bound in bound_values: + rpm_hard = _classify_rpm_perfect_match(installed_version, bound) + if rpm_hard is not None: + return rpm_hard + + for bound in bound_values: + if not _versions_parse_cleanly(installed_version, bound, ecosystem): + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.PARSE_ERROR, + ) + + return VersionCheckClassification(VersionCheckPath.EASY) + + +def _comparison_mode(version_info: dict[str, Any]) -> ComparisonMode: + first_patched = _intel_value(version_info.get(KEY_FIRST_PATCHED)) + vulnerable_range = _intel_value(version_info.get(KEY_VULNERABLE_RANGE)) + has_nvd_range = any( + _intel_value(version_info.get(key)) is not None + for key in NVD_RANGE_KEYS + ) + + if vulnerable_range: + return "ambiguous" + + if first_patched and has_nvd_range: + return "ambiguous" + + if first_patched: + return "first_patched" + + if has_nvd_range: + return "nvd_range" + + return "none" + + +def _description_has_multi_branch(description: str | None) -> bool: + if not description: + return False + return bool(_MULTI_BRANCH_DESCRIPTION_RE.search(description)) + + +def _installed_version_is_non_perfect(installed_version: str, ecosystem: str) -> bool: + if _MODULE_NEVRA_MARKER in installed_version: + return True + + upstream = _extract_upstream_component(installed_version, ecosystem) + if upstream is None: + return True + if _has_letter_suffix(upstream): + return True + if ecosystem != ECOSYSTEM_RPM and not _looks_like_plain_semver(upstream): + return True + return False + + +def _hard_for_installed(installed_version: str, ecosystem: str) -> VersionCheckClassification: + if _MODULE_NEVRA_MARKER in installed_version: + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.MODULE_NEVRA, + ) + + upstream = _extract_upstream_component(installed_version, ecosystem) + if upstream and _has_letter_suffix(upstream): + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.LETTER_SUFFIX_VERSION, + ) + + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.NON_SEMVER_INSTALLED, + ) + + +def _bound_is_non_perfect(bound: str, ecosystem: str) -> bool: + if _has_letter_suffix(bound): + return True + + if ecosystem != ECOSYSTEM_RPM: + return not _looks_like_plain_semver(bound.lstrip("vV")) + + upstream = _extract_upstream_component(bound, ECOSYSTEM_RPM) + if upstream and _has_letter_suffix(upstream): + return True + + if _extract_dist_tag(bound) is not None: + return False + + return "-" in bound + + +def _hard_for_bound(bound: str, ecosystem: str) -> VersionCheckClassification: + if _has_letter_suffix(bound) or ( + ecosystem == ECOSYSTEM_RPM + and _extract_upstream_component(bound, ECOSYSTEM_RPM) + and _has_letter_suffix(_extract_upstream_component(bound, ECOSYSTEM_RPM) or "") + ): + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.LETTER_SUFFIX_VERSION, + ) + + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.AXIS_MISMATCH, + ) + + +def _classify_rpm_perfect_match( + installed_version: str, + bound: str, +) -> VersionCheckClassification | None: + installed_dist = extract_dist_from_installed(installed_version) + bound_dist = _extract_dist_tag(bound) + + if bound_dist is not None: + if installed_dist is None or installed_dist != bound_dist: + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.CROSS_DIST_INTEL, + ) + if not _versions_parse_cleanly(installed_version, bound, ECOSYSTEM_RPM): + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.PARSE_ERROR, + ) + if not _rpm_comparators_agree(installed_version, bound): + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.COMPARATOR_DISAGREEMENT, + ) + return None + + if installed_dist is not None: + return VersionCheckClassification( + VersionCheckPath.HARD, + HardPathReason.AXIS_MISMATCH, + ) + + return None + + +def _intel_value(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip() + if not text or text == INTEL_VALUE_NOT_APPLICABLE: + return None + return text + + +def _has_letter_suffix(version: str) -> bool: + return bool(_LETTER_SUFFIX_VERSION_RE.match(version.strip())) + + +def _looks_like_plain_semver(version: str) -> bool: + return bool(_PLAIN_SEMVER_RE.match(version.strip())) + + +def _extract_upstream_component(version: str, ecosystem: str) -> str | None: + if ecosystem != ECOSYSTEM_RPM: + return version.lstrip("vV").split("+")[0].split("-")[0] or None + + if _MODULE_NEVRA_MARKER in version: + return version.split("-", 1)[0] or None + + if "-" not in version: + return version + + return version.split("-", 1)[0] or None + + +_ECOSYSTEM_VERSION_MAPPING = { + ECOSYSTEM_RPM: versions.RpmVersion, + "pypi": versions.PypiVersion, + "npm": versions.SemverVersion, + "go": versions.GolangVersion, + "maven": versions.MavenVersion, +} + + +def _version_func_for_ecosystem(ecosystem: str): + return _ECOSYSTEM_VERSION_MAPPING.get(ecosystem, versions.GenericVersion) + + +def _versions_parse_cleanly(installed: str, bound: str, ecosystem: str) -> bool: + version_func = _version_func_for_ecosystem(ecosystem) + try: + version_func(installed) + version_func(bound) + except versions.InvalidVersion: + logger.debug( + "Version parse failed for installed=%r bound=%r ecosystem=%s", + installed, + bound, + ecosystem, + exc_info=True, + ) + return False + return True + + +def _rpm_comparators_agree(installed: str, bound: str) -> bool: + try: + generic_less = versions.GenericVersion(installed) < versions.GenericVersion(bound) + rpm_less = versions.RpmVersion(installed) < versions.RpmVersion(bound) + except versions.InvalidVersion: + logger.debug( + "RPM comparator probe failed for installed=%r bound=%r", + installed, + bound, + exc_info=True, + ) + return False + return generic_less == rpm_less + + +def _check_version_in_range( + version_to_check: str, + version_range: list[str | None], + ecosystem: str, +) -> bool: + """Range check mirroring package_identifier._check_version_in_range.""" + ver_start_excl, ver_end_excl, ver_start_incl, ver_end_incl = version_range + version_func = _version_func_for_ecosystem(ecosystem) + + try: + vtc = version_func(version_to_check) + vsi = version_func(ver_start_incl) if ver_start_incl else None + vse = version_func(ver_start_excl) if ver_start_excl else None + vei = version_func(ver_end_incl) if ver_end_incl else None + vee = version_func(ver_end_excl) if ver_end_excl else None + except versions.InvalidVersion: + logger.debug( + "Range parse failed for version_to_check=%r", + version_to_check, + exc_info=True, + ) + raise + + if vsi and not (vsi <= vtc): + return False + if vse and not (vse < vtc): + return False + if vei and not (vtc <= vei): + return False + if vee and not (vtc < vee): + return False + return True diff --git a/src/vuln_analysis/utils/vex/__init__.py b/src/vuln_analysis/utils/vex/__init__.py new file mode 100644 index 000000000..a1744724e --- /dev/null +++ b/src/vuln_analysis/utils/vex/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/vuln_analysis/utils/vex/implementations/__init__.py b/src/vuln_analysis/utils/vex/implementations/__init__.py new file mode 100644 index 000000000..a1744724e --- /dev/null +++ b/src/vuln_analysis/utils/vex/implementations/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/vuln_analysis/utils/vex/implementations/csaf_generator.py b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py new file mode 100644 index 000000000..c5fe5f300 --- /dev/null +++ b/src/vuln_analysis/utils/vex/implementations/csaf_generator.py @@ -0,0 +1,306 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +import os +from pathlib import Path +import re +import tempfile +from typing import Any, Dict + +from exploit_iq_commons.data_models.cve_intel import CveIntel +from vuln_analysis.data_models.state import AgentMorpheusEngineState + +from ..vex_generator_base import VexGenerator +from ..vex_utils import build_oci_image_purl, get_vex_validator, build_patch_recommendation +from csaf.generator import CSAFGenerator + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) + +# Regex pattern for OCI digest format +OCI_DIGEST_PATTERN = r"^[a-z0-9]+:[a-f0-9]{32,}$" +OCI_DIGEST_RE = re.compile(OCI_DIGEST_PATTERN) + +# Note categories +NOTE_CATEGORY_DESCRIPTION = "description" +NOTE_CATEGORY_SUMMARY = "summary" +NOTE_CATEGORY_GENERAL = "general" +NOTE_CATEGORY_OTHER = "other" +NOTE_CATEGORY_LEGAL_DISCLAIMER = "legal_disclaimer" + +# Note titles +NOTE_TITLE_VULNERABILITY_DESCRIPTION = "Vulnerability description" +NOTE_TITLE_VULNERABILITY_SUMMARY = "Vulnerability summary" +NOTE_TITLE_RHSA_STATEMENT = "Red Hat Security Advisory Statement" +NOTE_TITLE_EXPLOITIQ_SUMMARY = "ExploitIQ Analysis Summary" +NOTE_TITLE_EXPLOITIQ_JUSTIFICATION_REASONING = "ExploitIQ Analysis Justification Reasoning" +NOTE_TITLE_EXPLOITIQ_JUSTIFICATION_LABEL = "ExploitIQ Analysis Justification Label" +NOTE_TITLE_UNOFFICIAL_CONTENT = "Unofficial Content Notice" + +# Disclaimer text +DISCLAIMER_TEXT = ( + "This CSAF document is generated as custom content for internal or experimental use. " + "It is not an official Red Hat security document, has not undergone formal validation or review, " + "and is not digitally signed. The information provided here should not be regarded as authoritative " + "or replace official Red Hat advisories or VEX statements." +) + +# Author information +AUTHOR_NAME = "Red Hat Product Security" +AUTHOR_URL = "https://access.redhat.com/security/" + +# Default values +DEFAULT_VENDOR = "unknown" +DEFAULT_IMPACT = "unknown" + +# Justification labels +JUSTIFICATION_LABEL_VULNERABLE = "vulnerable" + +# ExploitIQ justification labels mapped to CSAF 2.0 VEX flag labels +EXPLOITIQ_TO_CSAF_JUSTIFICATION_MAP: dict[str, str] = { + "false_positive": "component_not_present", + "code_not_present": "vulnerable_code_not_present", + "code_not_reachable": "vulnerable_code_not_in_execute_path", + "requires_configuration": "vulnerable_code_cannot_be_controlled_by_adversary", + "requires_dependency": "component_not_present", + "requires_environment": "vulnerable_code_cannot_be_controlled_by_adversary", + "protected_by_compiler": "inline_mitigations_already_exist", + "protected_at_runtime": "inline_mitigations_already_exist", + "protected_at_perimeter": "vulnerable_code_cannot_be_controlled_by_adversary", + "protected_by_mitigating_control": "inline_mitigations_already_exist", + "uncertain": "component_not_present", +} +DEFAULT_CSAF_JUSTIFICATION = "component_not_present" + +# Vulnerability statuses +STATUS_KNOWN_AFFECTED = "known_affected" +STATUS_KNOWN_NOT_AFFECTED = "known_not_affected" + +# Remediation +REMEDIATION_TYPE_VENDOR_FIX = "vendor_fix" +REMEDIATION_MESSAGE_TEMPLATE = "Upgrade to the first patched version(s): {patch_recommendation}." +REMEDIATION_MESSAGE_NO_RECOMMENDATION = "No remediation recommendation available for this vulnerability." + +# File names +CSAF_OUTPUT_FILENAME = "csaf.json" + +# Error messages +ERROR_CSAF_VALIDATION_FAILED = "CSAF document does not conform to CSAF 2.0 schema." + +# CSAF VEX schema path +CSAF_SCHEMA_PATH = Path(__file__).resolve().parents[3] / "configs" / "vex" / "csaf" / "v2.0" / "csaf_json_schema.json" + + +def _map_justification_to_csaf_label(exploitiq_label: str | None) -> str: + if not exploitiq_label: + return DEFAULT_CSAF_JUSTIFICATION + return EXPLOITIQ_TO_CSAF_JUSTIFICATION_MAP.get(exploitiq_label, DEFAULT_CSAF_JUSTIFICATION) + + +def _enrich_vulnerabilities_with_notes( + csaf_json: Dict[str, Any], + intel_map: Dict[str, CveIntel], + final_summaries: Dict[str, str], + justifications: Dict[str, Dict[str, str]] +) -> None: + """ + Enrich each vulnerability in the CSAF document with informational notes + from intel sources and analysis results. + + Returns the enriched csaf_json document. + """ + for v in csaf_json.get("vulnerabilities", []): + vuln_id = v.get("cve") + notes = v.get("notes", []) + + ci = intel_map.get(vuln_id) + + # Update or remove GHSA description note + ghsa_description = ci.ghsa.description if (ci and ci.ghsa and ci.ghsa.description) else None + if ghsa_description: + for note in notes: + if note.get("category") == NOTE_CATEGORY_DESCRIPTION: + note["title"] = NOTE_TITLE_VULNERABILITY_DESCRIPTION + note["text"] = ghsa_description + break + else: + # Remove description note if no GHSA description available + notes[:] = [note for note in notes if note.get("category") != NOTE_CATEGORY_DESCRIPTION] + + # Add GHSA summary if available + ghsa_summary = ci.ghsa.summary if (ci and ci.ghsa and ci.ghsa.summary) else None + if ghsa_summary: + notes.append({ + "category": NOTE_CATEGORY_SUMMARY, + "text": ghsa_summary, + "title": NOTE_TITLE_VULNERABILITY_SUMMARY + }) + + # Add RHSA statement if available + rhsa_statement = ci.rhsa.statement if (ci and ci.rhsa and ci.rhsa.statement) else None + if rhsa_statement: + notes.append({ + "category": NOTE_CATEGORY_GENERAL, + "text": rhsa_statement, + "title": NOTE_TITLE_RHSA_STATEMENT + }) + + # Add ExploitIQ analysis summary + summary = final_summaries.get(vuln_id) + notes.append({ + "category": NOTE_CATEGORY_OTHER, + "title": NOTE_TITLE_EXPLOITIQ_SUMMARY, + "text": summary + }) + + # Add ExploitIQ justification details + justification = justifications.get(vuln_id) + notes.append({ + "category": NOTE_CATEGORY_OTHER, + "title": NOTE_TITLE_EXPLOITIQ_JUSTIFICATION_REASONING, + "text": justification.get("justification") + }) + notes.append({ + "category": NOTE_CATEGORY_OTHER, + "title": NOTE_TITLE_EXPLOITIQ_JUSTIFICATION_LABEL, + "text": justification.get("justification_label") + }) + + v["notes"] = notes + + +def _enrich_product_tree_with_purl(csaf_json: Dict[str, Any], purl: str | None) -> None: + """Add product_identification_helper.purl to each product in the product tree.""" + if not purl: + return + + def visit(obj: Any) -> None: + if isinstance(obj, dict): + if "product_id" in obj and "name" in obj: + helper = obj.setdefault("product_identification_helper", {}) + helper["purl"] = purl + for value in obj.values(): + visit(value) + elif isinstance(obj, list): + for item in obj: + visit(item) + + visit(csaf_json.get("product_tree", {})) + + +class CsafVexGenerator(VexGenerator): + """ + CSAF VEX generator. Builds a CSAF JSON document and validates it with the csaf-tool. + """ + + def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: + + message: AgentMorpheusEngineInput = state.original_input + + csaf_gen = CSAFGenerator() + + product_name = message.input.image.name + product_tag = message.input.image.tag + + csaf_gen.set_header_title(f"ExploitIQ VEX Document - {product_name}{"@" if OCI_DIGEST_RE.fullmatch(product_tag) else ":"}{product_tag}") + + csaf_gen.set_value("notes",[ + { + "category": NOTE_CATEGORY_LEGAL_DISCLAIMER, + "text": DISCLAIMER_TEXT, + "title": NOTE_TITLE_UNOFFICIAL_CONTENT + } + ]) + + csaf_gen.set_value("author", AUTHOR_NAME) + csaf_gen.set_value("author_url", AUTHOR_URL) + + vendor = (product_name.split("/")[-2] if "/" in product_name else "") or DEFAULT_VENDOR + csaf_gen.add_product(product_name=product_name, vendor=vendor, release=product_tag) + + intel_map: Dict[str, CveIntel] = {ci.vuln_id: ci for ci in message.info.intel} + sbom_names: set[str] | None = None + if message.input.image.sbom_info and message.input.image.sbom_info.packages: + sbom_names = {pkg.name for pkg in message.input.image.sbom_info.packages} + + for vuln_id, justification in state.justifications.items(): + + ci = intel_map.get(vuln_id) + impact = ci.rhsa.threat_severity if ci and ci.rhsa and ci.rhsa.threat_severity else DEFAULT_IMPACT + + justification_label = justification.get("justification_label") + is_vulnerable = justification_label == JUSTIFICATION_LABEL_VULNERABLE + csaf_justification = _map_justification_to_csaf_label(justification_label) + + if is_vulnerable: + patch_recommendation = build_patch_recommendation(ci, sbom_names) + comment = ( + REMEDIATION_MESSAGE_TEMPLATE.format(patch_recommendation=patch_recommendation) + if patch_recommendation else REMEDIATION_MESSAGE_NO_RECOMMENDATION + ) + + csaf_gen.add_vulnerability( + product_name=product_name, + release=product_tag, + id=vuln_id, + status=STATUS_KNOWN_AFFECTED, + description="", + comment=impact, + remediation=REMEDIATION_TYPE_VENDOR_FIX, + action=comment + ) + + else: + csaf_gen.add_vulnerability( + product_name=product_name, + release=product_tag, + id=vuln_id, + status=STATUS_KNOWN_NOT_AFFECTED, + description="", + comment=impact, + justification=csaf_justification, + ) + + csaf_gen.generate_csaf() + + with tempfile.TemporaryDirectory() as tmp_dir: + path = os.path.join(tmp_dir, CSAF_OUTPUT_FILENAME) + csaf_gen.publish_csaf(path) + + # Load CSAF JSON + with open(path, "r") as f: + csaf_json = json.load(f) + + # Enrich the CSAF in memory + image = message.input.image + if image.analysis_type == "image": + product_purl = build_oci_image_purl(image.name, image.tag, image.digest) + _enrich_product_tree_with_purl(csaf_json, product_purl) + _enrich_vulnerabilities_with_notes( + csaf_json, intel_map, state.final_summaries, state.justifications + ) + + # Validate the CSAF document against the JSON schema + errors = list(get_vex_validator(CSAF_SCHEMA_PATH).iter_errors(csaf_json)) + if errors: + logger.error("%s Found %d error(s):", ERROR_CSAF_VALIDATION_FAILED, len(errors)) + for e in errors: + logger.error(" %s: %s", e.json_path, e.message) + return {} + + return csaf_json \ No newline at end of file diff --git a/src/vuln_analysis/utils/vex/tests/__init__.py b/src/vuln_analysis/utils/vex/tests/__init__.py new file mode 100644 index 000000000..a1744724e --- /dev/null +++ b/src/vuln_analysis/utils/vex/tests/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py b/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py new file mode 100644 index 000000000..4418a7663 --- /dev/null +++ b/src/vuln_analysis/utils/vex/tests/test_csaf_generator_integration.py @@ -0,0 +1,442 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Integration tests for CSAF VEX generator. +""" + +from unittest.mock import patch + +import pytest + +from exploit_iq_commons.data_models.common import AnalysisType +from exploit_iq_commons.data_models.cve_intel import CveIntel, CveIntelGhsa, CveIntelRhsa +from exploit_iq_commons.data_models.info import AgentMorpheusInfo, SBOMPackage +from exploit_iq_commons.data_models.input import ( + AgentMorpheusEngineInput, + AgentMorpheusInput, + ImageInfoInput, + ManualSBOMInfoInput, + ScanInfoInput, + SourceDocumentsInfo, + VulnInfo, +) +from vuln_analysis.data_models.state import AgentMorpheusEngineState +from vuln_analysis.utils.vex.implementations.csaf_generator import CsafVexGenerator +from vuln_analysis.utils.vex.vex_generator_loader import load_vex_generator +from vuln_analysis.utils.vex.vex_utils import build_oci_image_purl + + +_DEFAULT_SOURCE_INFO = [ + SourceDocumentsInfo( + type="code", + git_repo="https://github.com/example/repo", + ref="main", + include=["*.py"], + exclude=[], + ) +] +_DEFAULT_PRODUCT_NAME = "registry.example.com/myapp" +_DEFAULT_PRODUCT_TAG = "v1.0.0" +_DEFAULT_JUSTIFICATION = {"justification": "reason", "justification_label": "vulnerable"} +_DEFAULT_VULNS = ["CVE-2024-1234"] +_DEFAULT_SUMMARY = "Analysis summary for {v}" +_DEFAULT_SBOM_PACKAGES = [SBOMPackage(name="test-package", version="1.0.0", system="npm")] + + +@pytest.fixture(scope="module") +def mock_state() -> AgentMorpheusEngineState: + """Fixture providing a default mock AgentMorpheusEngineState for testing. + This state has one vulnerable CVE with a known affected status, with no GHSA or RHSA intel data. + """ + return create_mock_state() + + +def create_mock_state( + vulns: list[str] = _DEFAULT_VULNS, + justification: dict[str, dict[str, str]] | None = None, + intel: list[CveIntel] | None = None, + product_name: str = _DEFAULT_PRODUCT_NAME, + product_tag: str = _DEFAULT_PRODUCT_TAG, + sbom_packages: list[SBOMPackage] | None = _DEFAULT_SBOM_PACKAGES, +) -> AgentMorpheusEngineState: + """Create a mock AgentMorpheusEngineState for testing.""" + + intel = intel or [CveIntel(vuln_id=v) for v in vulns] + + sbom_info = None if sbom_packages is None else ManualSBOMInfoInput(packages=sbom_packages) + + image_info = ImageInfoInput( + name=product_name, + tag=product_tag, + analysis_type=AnalysisType.IMAGE, + source_info=_DEFAULT_SOURCE_INFO, + sbom_info=sbom_info, + ) + + engine_input = AgentMorpheusEngineInput( + input=AgentMorpheusInput( + scan=ScanInfoInput(vulns=[VulnInfo(vuln_id=v) for v in vulns]), + image=image_info, + ), + info=AgentMorpheusInfo(intel=intel), + ) + + return AgentMorpheusEngineState( + cve_intel=intel, + original_input=engine_input, + final_summaries={v: _DEFAULT_SUMMARY.format(v=v) for v in vulns}, + justifications={v: justification or _DEFAULT_JUSTIFICATION for v in vulns}, + ) + + +class TestCsafVexGeneratorIntegration: + """Integration tests for CsafVexGenerator.generate() method.""" + + def test_loader_returns_working_generator(self, mock_state): + """Test that load_vex_generator returns a functional generator.""" + generator = load_vex_generator("csaf") + result = generator.generate(mock_state) + + assert "document" in result + assert "product_tree" in result + assert "vulnerabilities" in result + + def test_document_has_correct_title(self, mock_state): + """Test that document title contains product name and tag.""" + generator = CsafVexGenerator() + result = generator.generate(mock_state) + + title = result["document"].get("title") + assert "ExploitIQ VEX Document - " + _DEFAULT_PRODUCT_NAME + ":" + _DEFAULT_PRODUCT_TAG in title + + def test_oci_digest_tag_uses_at_separator(self): + """Test that OCI digest tags use @ separator instead of : in title.""" + oci_digest = "sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" + state = create_mock_state( + product_tag=oci_digest, + ) + + generator = CsafVexGenerator() + result = generator.generate(state) + + title = result["document"].get("title") + assert "ExploitIQ VEX Document - " + _DEFAULT_PRODUCT_NAME + "@" + oci_digest in title + + def test_document_has_disclaimer_note(self, mock_state): + """Test that document includes the disclaimer note.""" + generator = CsafVexGenerator() + result = generator.generate(mock_state) + + notes = result["document"].get("notes", []) + disclaimer_notes = [n for n in notes if n.get("category") == "legal_disclaimer"] + assert len(disclaimer_notes) == 1 + assert "Unofficial Content Notice" in disclaimer_notes[0]["title"] + + def test_document_has_author_note(self, mock_state): + """Test that document includes the author note.""" + generator = CsafVexGenerator() + result = generator.generate(mock_state) + + publisher = result["document"].get("publisher") + assert "Red Hat Product Security" in publisher.get("name") + assert "https://access.redhat.com/security/" in publisher.get("namespace") + + def test_product_tree_contains_product(self, mock_state): + """Test that product tree contains the analyzed product.""" + generator = CsafVexGenerator() + result = generator.generate(mock_state) + + product_tree = result["product_tree"] + assert _DEFAULT_PRODUCT_NAME in product_tree.get("branches")[0].get("branches")[0].get("name") + assert _DEFAULT_PRODUCT_TAG in product_tree.get("branches")[0].get("branches")[0].get("branches")[0].get("name") + + def test_product_tree_includes_oci_purl(self, mock_state): + """Test that product tree includes product_identification_helper with OCI purl.""" + generator = CsafVexGenerator() + result = generator.generate(mock_state) + + product = ( + result["product_tree"] + .get("branches")[0] + .get("branches")[0] + .get("branches")[0] + .get("product") + ) + helper = product.get("product_identification_helper", {}) + assert helper.get("purl") == build_oci_image_purl(_DEFAULT_PRODUCT_NAME, _DEFAULT_PRODUCT_TAG) + + def test_product_tree_purl_prefers_digest_over_tag(self): + """Test that explicit digest is used in purl instead of tag.""" + oci_digest = "sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" + state = create_mock_state(product_tag="v1.0.0") + state.original_input.input.image.digest = oci_digest + + generator = CsafVexGenerator() + result = generator.generate(state) + + product = ( + result["product_tree"] + .get("branches")[0] + .get("branches")[0] + .get("branches")[0] + .get("product") + ) + helper = product.get("product_identification_helper", {}) + assert helper.get("purl") == build_oci_image_purl( + _DEFAULT_PRODUCT_NAME, "v1.0.0", oci_digest + ) + + def test_vulnerable_cve_has_known_affected_status(self, mock_state): + """Test that vulnerable CVEs get 'known_affected' status.""" + generator = CsafVexGenerator() + result = generator.generate(mock_state) + + vuln = result["vulnerabilities"][0] + product_status = vuln.get("product_status", {}) + assert "known_affected" in product_status + + def test_not_vulnerable_cve_has_known_not_affected_status(self): + """Test that non-vulnerable CVEs get 'known_not_affected' status.""" + state = create_mock_state( + justification={"justification": "Code path not reachable", "justification_label": "not_vulnerable"}, + ) + + generator = CsafVexGenerator() + result = generator.generate(state) + + vuln = result["vulnerabilities"][0] + product_status = vuln.get("product_status", {}) + assert "known_not_affected" in product_status + + def test_code_not_reachable_maps_to_csaf_execute_path_flag(self): + """Test that code_not_reachable maps to the CSAF execute-path flag.""" + state = create_mock_state( + justification={ + "justification": "Vulnerable function exists but is not called.", + "justification_label": "code_not_reachable", + }, + ) + + generator = CsafVexGenerator() + result = generator.generate(state) + + vuln = result["vulnerabilities"][0] + assert vuln["flags"][0]["label"] == "vulnerable_code_not_in_execute_path" + label_notes = [ + n for n in vuln.get("notes", []) + if n.get("title") == "ExploitIQ Analysis Justification Label" + ] + assert label_notes[0]["text"] == "code_not_reachable" + + def test_vulnerable_cve_includes_remediation(self): + """Test that vulnerable CVEs include remediation information when patch is available.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, + ] + ) + intel = [CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa)] + + state = create_mock_state( + intel=intel, + sbom_packages=[SBOMPackage(name="lodash", version="4.17.21", system="npm")], + ) + + generator = CsafVexGenerator() + result = generator.generate(state) + + vuln = result["vulnerabilities"][0] + remediations = vuln.get("remediations", []) + assert len(remediations) > 0 + assert "lodash:4.17.21" in remediations[0].get("details") + + def test_no_patch_available_shows_fallback_message(self, mock_state): + """Test 'No remediation recommendation available' message when no patch is available.""" + generator = CsafVexGenerator() + result = generator.generate(mock_state) + + vuln = result["vulnerabilities"][0] + remediations = vuln.get("remediations", []) + assert "No remediation recommendation available" in remediations[0].get("details") + + def test_multiple_vulnerabilities_all_included(self): + """Test that multiple vulnerabilities are all included in output.""" + state = create_mock_state( + vulns=["CVE-2024-1234", "CVE-2024-5678", "CVE-2024-9999"] + ) + + generator = CsafVexGenerator() + result = generator.generate(state) + + cve_ids = [v.get("cve") for v in result["vulnerabilities"]] + assert "CVE-2024-1234" in cve_ids + assert "CVE-2024-5678" in cve_ids + assert "CVE-2024-9999" in cve_ids + + def test_vulnerabilities_have_notes_enriched(self): + """Test that vulnerabilities have notes added from enrichment.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + summary="Test vulnerability summary", + description="Detailed description of the vulnerability", + ) + rhsa = CveIntelRhsa(threat_severity="Important", statement="RHSA statement.") + intel = [CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa, rhsa=rhsa)] + + state = create_mock_state( + intel=intel, + ) + + generator = CsafVexGenerator() + result = generator.generate(state) + + vuln = result["vulnerabilities"][0] + notes = vuln.get("notes", []) + + other_notes = [n for n in notes if n.get("category") == "other"] + assert len(other_notes) == 3 # analysis summary + justification reasoning + justification label + + description_note = [n for n in notes if n.get("category") == "description"] + assert len(description_note) == 1 # ghsa description + + summary_note = [n for n in notes if n.get("category") == "summary"] + assert len(summary_note) == 1 # ghsa summary + + general_notes = [n for n in notes if n.get("category") == "general"] + assert len(general_notes) == 1 # rhsa statement + + def test_description_note_removed_when_no_ghsa(self, mock_state): + """Test that description note is removed when no GHSA data is available.""" + generator = CsafVexGenerator() + result = generator.generate(mock_state) + + vuln = result["vulnerabilities"][0] + notes = vuln.get("notes", []) + description_notes = [n for n in notes if n.get("category") == "description"] + assert len(description_notes) == 0 # Description note should be removed + + def test_rhsa_threat_severity_used_as_impact(self): + """Test that RHSA threat severity is included as impact/comment.""" + rhsa = CveIntelRhsa(threat_severity="Important") + intel = [CveIntel(vuln_id="CVE-2024-1234", rhsa=rhsa)] + + state = create_mock_state( + intel=intel, + ) + + generator = CsafVexGenerator() + result = generator.generate(state) + + vuln = result["vulnerabilities"][0] + impact = vuln.get("threats")[0].get("details") + assert "Important" in impact + + +class TestCsafVexGeneratorEdgeCases: + """Integration tests for edge cases and error handling.""" + + @pytest.mark.parametrize("sbom_packages", [None, []], ids=["sbom_info_none", "empty_packages"]) + def test_includes_all_packages_when_no_sbom_filtering(self, sbom_packages): + """Test that all packages are included when SBOM is None or has no packages.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, + {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, + ] + ) + intel = [CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa)] + state = create_mock_state( + intel=intel, + sbom_packages=sbom_packages + ) + + generator = CsafVexGenerator() + result = generator.generate(state) + + assert "vulnerabilities" in result + vuln = result["vulnerabilities"][0] + remediations = vuln.get("remediations", []) + assert "lodash:4.17.21" in remediations[0].get("details") + assert "express:4.18.0" in remediations[0].get("details") + + def test_handles_sbom_packages_with_matching_and_non_matching_packages(self): + """Test handling when SBOM has a matching package and a non-matching package.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, + {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, + ] + ) + intel = [CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa)] + state = create_mock_state( + intel=intel, + sbom_packages=[SBOMPackage(name="lodash", version="4.17.21", system="npm")], + ) + + generator = CsafVexGenerator() + result = generator.generate(state) + + assert "vulnerabilities" in result + vuln = result["vulnerabilities"][0] + remediations = vuln.get("remediations", []) + assert "lodash:4.17.21" in remediations[0].get("details") + assert "express:4.18.0" not in remediations[0].get("details") + + def test_handles_missing_rhsa_severity(self, mock_state): + """Test handling when RHSA threat severity is missing.""" + generator = CsafVexGenerator() + result = generator.generate(mock_state) + + assert "vulnerabilities" in result + + vuln = result["vulnerabilities"][0] + impact = vuln.get("threats")[0].get("details") + assert "unknown" in impact + + def test_handles_product_name_without_slash(self): + """Test vendor extraction when product name has no slash.""" + state = create_mock_state( + product_name="simpleapp", + ) + + generator = CsafVexGenerator() + result = generator.generate(state) + + product_tree = result["product_tree"] + assert "unknown" == product_tree.get("branches")[0].get("name") + assert "simpleapp" == product_tree.get("branches")[0].get("branches")[0].get("name") + + def test_validation_failure_returns_empty_dict(self, mock_state): + """Test that validation failure returns empty dict.""" + generator = CsafVexGenerator() + + # Mock the enrichment function to inject invalid data (modifies csaf_json in-place) + def mock_enrich(csaf_json, intel_map, final_summaries, justifications): + # Inject invalid note category (violates schema) + for v in csaf_json.get("vulnerabilities", []): + v["notes"] = [{"category": "not_allowed_category", "title": "Test", "text": "test text"}] + + with patch( + "vuln_analysis.utils.vex.implementations.csaf_generator._enrich_vulnerabilities_with_notes", + side_effect=mock_enrich + ): + result = generator.generate(mock_state) + + assert result == {} + diff --git a/src/vuln_analysis/utils/vex/vex_generator_base.py b/src/vuln_analysis/utils/vex/vex_generator_base.py new file mode 100644 index 000000000..fa8d3c452 --- /dev/null +++ b/src/vuln_analysis/utils/vex/vex_generator_base.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any, Dict + +from vuln_analysis.data_models.state import AgentMorpheusEngineState + + +class VexGenerator(ABC): + """ + Abstract base for VEX document generators. + """ + + @abstractmethod + def generate(self, state: AgentMorpheusEngineState) -> Dict[str, Any]: + """ + Generate a VEX document as a JSON-serializable dict from the engine state. + """ + pass + + diff --git a/src/vuln_analysis/utils/vex/vex_generator_loader.py b/src/vuln_analysis/utils/vex/vex_generator_loader.py new file mode 100644 index 000000000..c8b46ff1c --- /dev/null +++ b/src/vuln_analysis/utils/vex/vex_generator_loader.py @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from .vex_generator_base import VexGenerator +from .implementations.csaf_generator import CsafVexGenerator + +# Supported VEX formats +CSAF = "csaf" + +# Error messages +ERROR_UNSUPPORTED_VEX_FORMAT = "Unsupported VEX format: {fmt}" + + +def load_vex_generator(fmt: str) -> VexGenerator: + """ + Return a VEX generator implementation for the requested format. + """ + + fmt_lower = fmt.lower() + + if fmt_lower == CSAF: + return CsafVexGenerator() + raise ValueError(ERROR_UNSUPPORTED_VEX_FORMAT.format(fmt=fmt_lower)) + diff --git a/src/vuln_analysis/utils/vex/vex_utils.py b/src/vuln_analysis/utils/vex/vex_utils.py new file mode 100644 index 000000000..f41c92048 --- /dev/null +++ b/src/vuln_analysis/utils/vex/vex_utils.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from functools import lru_cache +import json +from pathlib import Path + +from jsonschema import Draft202012Validator +from packageurl import PackageURL + +from exploit_iq_commons.data_models.cve_intel import CveIntel +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from urllib.parse import urlparse + +logger = LoggingFactory.get_agent_logger(__name__) + + +@lru_cache(maxsize=None) +def get_vex_validator(schema_path: Path) -> Draft202012Validator: + """ + Load schema and create a cached VEX document validator instance. + + Args: + schema_path: Path to the JSON schema file. + + Returns: + A Draft202012Validator instance for the schema. + """ + with open(schema_path) as f: + schema = json.load(f) + logger.debug("VEX document schema loaded from %s", schema_path) + return Draft202012Validator(schema) + + +def get_patched_package(vuln: dict) -> tuple[str | None, str | None]: + """ + Extract package name and first patched version from a GHSA vulnerability dict. + + Args: + vuln: A GHSA vulnerability dictionary. + + Returns: + Tuple of (package_name, first_patched_version), either may be None. + """ + pkg = vuln.get("package") or {} + return pkg.get("name"), vuln.get("first_patched_version") + + +def build_oci_image_purl( + image_name: str, + tag: str | None = None, + digest: str | None = None, +) -> str | None: + """ + Build an OCI package URL (purl) for a container image. + + Prefers an explicit digest, then falls back to the image tag. + """ + image_path = image_name + parsed = urlparse(f"//{image_path}") + registry = parsed.netloc + # qualifiers include registry and full name which all already exist in image_path + qualifiers = {"repository_url": image_path} if image_path else {} + path_parts = [part for part in parsed.path.strip("/").split("/") if part] + if path_parts: + if len(path_parts) > 1: + name = path_parts[-1] + namespace = "/".join(path_parts[:-1]) + else: + name = path_parts[0] + namespace = None + elif parsed.netloc: + name = parsed.netloc + namespace = None + else: + name = image_path + namespace = None + + version = digest or tag + # oci purl specification required to emit namespace, therefor it is set to None + purl = PackageURL( + type="oci", + namespace=None, + name=name, + version=version, + qualifiers=qualifiers if qualifiers else None, + ) + logger.debug( + "Building OCI image purl components: registry=%s, qualifiers=%s, name=%s, version=%s", + registry, + qualifiers, + name, + version, + ) + logger.debug("Resulting OCI image purl: %s", purl.to_string()) + return purl.to_string() + + +def build_patch_recommendation(ci: CveIntel, sbom_package_names: set[str] | None) -> str: + """ + Build a patch recommendation string from GHSA data. + + Args: + ci: CveIntel object containing GHSA vulnerability data. + sbom_package_names: Optional set of package names from SBOM to filter by. + + Returns: + Comma-separated string of "name:version" pairs, or empty string if none found. + + Notes: + - If SBOM provided: returns only patches for packages in the SBOM. + - If no SBOM: returns all unique patches. + """ + if not ci or not ci.ghsa or not ci.ghsa.vulnerabilities: + return "" + + vulns = ci.ghsa.vulnerabilities + + name_to_version: dict[str, str] = {} + for v in vulns: + name, patch = get_patched_package(v) + if not name or not patch: + continue + if name in name_to_version: + continue + # If SBOM provided, only include packages that are in the SBOM + if sbom_package_names is not None and name not in sbom_package_names: + continue + name_to_version[name] = patch + + if not name_to_version: + return "" + return ", ".join(f"{name}:{patch}" for name, patch in name_to_version.items()) diff --git a/src/vuln_analysis/utils/vulnerability_intel_sanitizer.py b/src/vuln_analysis/utils/vulnerability_intel_sanitizer.py new file mode 100644 index 000000000..99632b919 --- /dev/null +++ b/src/vuln_analysis/utils/vulnerability_intel_sanitizer.py @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Sanitize LLM-extracted VulnerabilityIntel after L1 extraction.""" + +from __future__ import annotations + +import re +from pathlib import Path + +from exploit_iq_commons.data_models.checker_status import VulnerabilityIntel +from vuln_analysis.functions.code_agent_graph_defs import ParsedPatch + +_BOOLEAN_OP_RE = re.compile(r"\s+(?:OR|AND)\s+", re.IGNORECASE) + + +def _patch_basenames(parsed_patch: ParsedPatch) -> set[str]: + names: set[str] = set() + for patch_file in parsed_patch.files: + for path in (patch_file.source_path, patch_file.target_path): + if path: + names.add(Path(path).name.lower()) + return names + + +def _has_boolean_operator(keyword: str) -> bool: + return bool(_BOOLEAN_OP_RE.search(keyword)) + + +def _patch_line_presence(parsed_patch: ParsedPatch) -> tuple[bool, bool]: + """Return (has_removed_lines, has_added_lines) across all hunks.""" + has_removed_lines = False + has_added_lines = False + + for patch_file in parsed_patch.files: + for hunk in patch_file.hunks: + if hunk.removed_lines: + has_removed_lines = True + if hunk.added_lines: + has_added_lines = True + if has_removed_lines and has_added_lines: + return has_removed_lines, has_added_lines + + return has_removed_lines, has_added_lines + + +class VulnerabilityIntelSanitizer: + """Apply shape rules to L1 VulnerabilityIntel; extensible one method per rule.""" + + def __init__(self, parsed_patch: ParsedPatch | None = None) -> None: + self._parsed_patch = parsed_patch + + @property + def _has_trusted_patch(self) -> bool: + return self._parsed_patch is not None and bool(self._parsed_patch.files) + + @property + def _is_additive_only_patch(self) -> bool: + if not self._has_trusted_patch: + return False + + has_removed_lines, has_added_lines = _patch_line_presence(self._parsed_patch) + return has_added_lines and not has_removed_lines + + def apply(self, intel: VulnerabilityIntel) -> VulnerabilityIntel: + intel = self.sanitize_affected_files(intel) + intel = self.sanitize_additive_only_patch_intel(intel) + intel = self.filter_vulnerable_functions(intel) + return self.filter_search_keywords(intel) + + def sanitize_affected_files(self, intel: VulnerabilityIntel) -> VulnerabilityIntel: + parsed_patch = self._parsed_patch + if parsed_patch is None or not parsed_patch.files: + return intel.model_copy(update={"affected_files": []}) + + allowed = _patch_basenames(parsed_patch) + kept = [ + path + for path in intel.affected_files + if Path(path).name.lower() in allowed + ] + return intel.model_copy(update={"affected_files": kept}) + + def filter_vulnerable_functions(self, intel: VulnerabilityIntel) -> VulnerabilityIntel: + kept = [name for name in intel.vulnerable_functions if " " not in name] + return intel.model_copy(update={"vulnerable_functions": kept}) + + def sanitize_additive_only_patch_intel(self, intel: VulnerabilityIntel) -> VulnerabilityIntel: + """Drop vulnerable-side intel when patch has only added lines.""" + if not self._is_additive_only_patch: + return intel + + return intel.model_copy( + update={ + "vulnerable_functions": [], + "vulnerable_variables": [], + "vulnerable_patterns": [], + } + ) + + def filter_search_keywords(self, intel: VulnerabilityIntel) -> VulnerabilityIntel: + kept = [ + kw + for kw in intel.search_keywords + if " " not in kw or _has_boolean_operator(kw) + ] + return intel.model_copy(update={"search_keywords": kept}) diff --git a/src/vuln_analysis/utils/vulnerable_dependency_checker.py b/src/vuln_analysis/utils/vulnerable_dependency_checker.py index 9e3ac3c4a..3b459dd13 100644 --- a/src/vuln_analysis/utils/vulnerable_dependency_checker.py +++ b/src/vuln_analysis/utils/vulnerable_dependency_checker.py @@ -14,7 +14,6 @@ # limitations under the License. import asyncio -import logging import os import re import urllib.parse @@ -26,13 +25,14 @@ from tqdm import tqdm from univers import versions -from vuln_analysis.data_models.cve_intel import CveIntelNvd -from vuln_analysis.data_models.dependencies import DependencyPackage +from exploit_iq_commons.data_models.cve_intel import CveIntelNvd +from exploit_iq_commons.data_models.dependencies import DependencyPackage from vuln_analysis.utils.clients.intel_client import IntelClient -from vuln_analysis.utils.string_utils import package_names_match +from exploit_iq_commons.utils.string_utils import package_names_match from vuln_analysis.utils.url_utils import url_join -logger = logging.getLogger(__name__) +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +logger = LoggingFactory.get_agent_logger(__name__) REQUESTS_TIMEOUT = 2 diff --git a/src/vuln_analysis/utils/web_patch_fetcher.py b/src/vuln_analysis/utils/web_patch_fetcher.py new file mode 100644 index 000000000..320aeb71b --- /dev/null +++ b/src/vuln_analysis/utils/web_patch_fetcher.py @@ -0,0 +1,1120 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Web Patch Fetcher - unified patch fetching from intel references and OSV API. + +This module provides a unified interface for fetching upstream fix patches from: +- Intel references (GHSA, NVD, RHSA, Ubuntu) +- Chromium issue tracker (via Gerrit/Gitiles) +- OSV API (fallback) + +Supported platforms: +- GitHub (/commit/, /pull/) +- kernel.org (cgit format, short /stable/c/ format) +- gitweb (e.g. git.samba.org — ``?p=repo.git;a=patch;h=``) +- gitiles (Chromium/Google source, base64-encoded patches) +""" + +from __future__ import annotations + +import base64 +import os +import re +from typing import Literal, TYPE_CHECKING + +from urllib.parse import urlparse, unquote + +import aiohttp +from pydantic import BaseModel +from unidiff import PatchSet + +from exploit_iq_commons.logging.loggers_factory import LoggingFactory +from vuln_analysis.utils.async_http_utils import request_with_retry +from vuln_analysis.utils.intel_utils import CHROMIUM_ISSUE_PATTERN +from vuln_analysis.utils.gerrit_client import ( + search_changes_by_bug, + list_merged_changes, + select_gerrit_change, + get_current_commit_sha, + project_to_gitiles_repo_url, + build_gitiles_patch_url, +) +from vuln_analysis.functions.code_agent_graph_defs import ( + OSVPatchResult, + ParsedPatch, + PatchFile, + PatchHunk, +) + +if TYPE_CHECKING: + from langchain_core.language_models import BaseChatModel + +logger = LoggingFactory.get_agent_logger(__name__) + +# Re-export for backwards compatibility +WebPatchResult = OSVPatchResult + +# Environment configuration +_GITHUB_TOKEN = os.environ.get("GHSA_API_KEY") +_OSV_API_URL = os.environ.get("OSV_API_URL", "https://api.osv.dev/v1/vulns/") +_OSV_TIMEOUT_SECONDS = int(os.environ.get("OSV_TIMEOUT_SECONDS", "10")) +_PATCH_FETCH_TIMEOUT_SECONDS = int(os.environ.get("PATCH_FETCH_TIMEOUT_SECONDS", "30")) + +# Binary file extensions to skip when parsing patches +_BINARY_FILE_EXTENSIONS = frozenset({ + '.uu', '.uue', '.iso', '.bin', '.gz', '.bz2', '.xz', '.zip', '.tar', '.tgz', '.tbz2', + '.png', '.jpg', '.jpeg', '.gif', '.bmp', '.ico', '.webp', + '.pdf', '.doc', '.docx', '.xls', '.xlsx', + '.exe', '.dll', '.so', '.dylib', '.a', '.o', '.obj', + '.pyc', '.pyo', '.class', '.jar', '.war', +}) + +# URL patterns for platform detection and resolution +_GITHUB_COMMIT_PATTERN = re.compile( + r"https?://github\.com/([^/]+/[^/]+)/commit/([a-f0-9]+)" +) +_GITHUB_PR_PATTERN = re.compile( + r"https?://github\.com/([^/]+/[^/]+)/pull/(\d+)" +) +_GITHUB_REPO_PATTERN = re.compile( + r"https?://github\.com/([^/]+/[^/]+?)(?:\.git)?/?$" +) + +# kernel.org cgit patterns +_KERNEL_CGIT_COMMIT_PATTERN = re.compile( + r"(https://git\.kernel\.org/.+\.git)/commit/\?id=([a-f0-9]+)" +) +_KERNEL_CGIT_PATCH_PATTERN = re.compile( + r"https://git\.kernel\.org/.+\.git/patch/\?id=[a-f0-9]+" +) +_KERNEL_SHORT_PATTERN = re.compile( + r"https://git\.kernel\.org/(\w+)/c/([a-f0-9]+)" +) + +# Mapping from kernel.org short paths to full repo paths +_KERNEL_SHORT_PATH_MAP = { + "stable": "linux/kernel/git/stable/linux.git", + "torvalds": "linux/kernel/git/torvalds/linux.git", + "next": "linux/kernel/git/next/linux-next.git", +} + +# Ubuntu patch reference pattern (e.g., "upstream: https://github.com/.../commit/...") +_UBUNTU_PATCH_URL_PATTERN = re.compile( + r"(?:upstream:\s*)?(https://github\.com/[^/]+/[^/]+/commit/([a-f0-9]+))" +) + +# gitweb (Samba, etc.): ?p=repo.git;a=commit|patch;h= +_GITWEB_COMMIT_PATTERN = re.compile( + r"(https?://[^/]+)/\?p=([^;&]+\.git);a=commit;h=([a-f0-9]+)" +) +_GITWEB_PATCH_PATTERN = re.compile( + r"(https?://[^/]+)/\?p=([^;&]+\.git);a=patch;h=([a-f0-9]+)" +) + +# Gitiles (Chromium/Google source): /+/ with optional ^! and ?format=TEXT +# Matches: https://chromium.googlesource.com/angle/angle/+/abc123 +# https://chromium.googlesource.com/chromium/src/+/abc123%5E%21?format=TEXT +_GITILES_COMMIT_PATTERN = re.compile( + r"(https://[^/]+\.googlesource\.com/[^+]+)/\+/([a-f0-9]+)(?:%5E%21|\^!)?" +) + + +# --------------------------------------------------------------------------- +# Data Models +# --------------------------------------------------------------------------- + +class OSVAffectedRange(BaseModel): + """Represents a Git range from an OSV affected block.""" + repo_url: str | None = None + fixed_commit: str | None = None + introduced_commit: str | None = None + + +class ResolvedUrl(BaseModel): + """Result of resolving a URL to a patch download URL.""" + patch_url: str + platform: Literal["github", "kernel.org", "gitweb", "cgit", "gitiles"] + url_type: Literal["commit", "pull", "chromium_issue"] + repo_url: str | None = None + commit_sha: str | None = None + + +# --------------------------------------------------------------------------- +# Helper Functions +# --------------------------------------------------------------------------- + +def _is_binary_file_path(path: str) -> bool: + """Check if file path has a binary file extension.""" + path_lower = path.lower() + return any(path_lower.endswith(ext) for ext in _BINARY_FILE_EXTENSIONS) + + +def _parse_patch_content(patch_content: str, patch_filename: str) -> ParsedPatch | None: + """Parse patch content string into structured ParsedPatch model.""" + try: + patch_set = PatchSet.from_string(patch_content) + except Exception: + logger.warning("_parse_patch_content: failed to parse patch content") + return None + + files: list[PatchFile] = [] + for patched_file in patch_set: + if patched_file.is_binary_file: + continue + if _is_binary_file_path(patched_file.target_file): + continue + + hunks: list[PatchHunk] = [] + for hunk in patched_file: + context, removed, added = [], [], [] + for line in hunk: + if line.is_context: + context.append(str(line.value).rstrip("\n")) + elif line.is_removed: + removed.append(str(line.value).rstrip("\n")) + elif line.is_added: + added.append(str(line.value).rstrip("\n")) + + hunks.append(PatchHunk( + source_start=hunk.source_start, + source_length=hunk.source_length, + target_start=hunk.target_start, + target_length=hunk.target_length, + section_header=hunk.section_header or "", + context_lines=context, + removed_lines=removed, + added_lines=added, + )) + + files.append(PatchFile( + source_path=patched_file.source_file, + target_path=patched_file.target_file, + hunks=hunks, + is_new_file=patched_file.is_added_file, + is_deleted_file=patched_file.is_removed_file, + )) + + return ParsedPatch(patch_filename=patch_filename, files=files) + + +def _extract_commit_metadata(patch_content: str) -> tuple[str | None, str | None, str | None]: + """Extract commit message, author, and date from .patch format. + + Works with both GitHub and kernel.org patch formats. + """ + lines = patch_content.split('\n') + author = None + date = None + subject_lines = [] + in_subject = False + + for line in lines: + if line.startswith('From:'): + author = line[5:].strip() + elif line.startswith('Date:'): + date = line[5:].strip() + elif line.startswith('Subject:'): + in_subject = True + subject_part = line[8:].strip() + if subject_part.startswith('[PATCH'): + idx = subject_part.find(']') + if idx != -1: + subject_part = subject_part[idx + 1:].strip() + subject_lines.append(subject_part) + elif in_subject: + if line.startswith('---') or line.startswith('diff --git'): + break + if line.strip() == '': + in_subject = False + else: + subject_lines.append(line.strip()) + + commit_message = ' '.join(subject_lines).strip() if subject_lines else None + return commit_message, author, date + + +def build_patch_url_from_repo(repo_url: str, commit_sha: str) -> str | None: + """Build a patch download URL from a git repo URL and commit SHA. + + Supports GitHub, gitweb (shallow ``host/repo.git`` paths), and cgit path-info. + """ + match = _GITHUB_REPO_PATTERN.match(repo_url) + if match: + repo_path = match.group(1) + return f"https://github.com/{repo_path}/commit/{commit_sha}.patch" + if "github.com" in repo_url: + parts = repo_url.rstrip('/').split('/') + if len(parts) >= 2: + repo_path = '/'.join(parts[-2:]).replace('.git', '') + return f"https://github.com/{repo_path}/commit/{commit_sha}.patch" + + parsed = urlparse(repo_url.rstrip('/')) + if not parsed.path.endswith('.git'): + return None + + path_parts = [p for p in parsed.path.split('/') if p] + if not path_parts: + return None + + project = path_parts[-1] + base = f"{parsed.scheme}://{parsed.netloc}" + + # Shallow repo path (e.g. git.samba.org/rsync.git) → gitweb + if len(path_parts) == 1: + return f"{base}/?p={project};a=patch;h={commit_sha}" + + # Deep path (e.g. git.kernel.org/pub/scm/.../linux.git) → cgit path-info + return f"{repo_url.rstrip('/')}/patch/?id={commit_sha}" + + +# --------------------------------------------------------------------------- +# WebPatchFetcher - Unified Patch Fetcher +# --------------------------------------------------------------------------- + +class WebPatchFetcher: + """Unified patch fetcher for intel references and direct URLs. + + Supports: + - GitHub commit URLs (/commit/) + - GitHub PR URLs (/pull/) + - kernel.org cgit URLs (/commit/?id=, /patch/?id=) + - kernel.org short URLs (/stable/c/, /torvalds/c/) + - gitweb URLs (?p=repo.git;a=commit|patch;h=) + + Usage: + async with aiohttp.ClientSession() as session: + fetcher = WebPatchFetcher(session) + result = await fetcher.fetch_from_intel_refs(candidates, "CVE-2024-1234") + if result: + # Use result.parsed_patch + pass + """ + + def __init__( + self, + session: aiohttp.ClientSession, + timeout: int = _PATCH_FETCH_TIMEOUT_SECONDS, + ): + self._session = session + self._timeout = aiohttp.ClientTimeout(total=timeout) + self._headers = {"User-Agent": "vuln-analysis/1.0"} + if _GITHUB_TOKEN: + self._headers["Authorization"] = f"token {_GITHUB_TOKEN}" + self._cache: dict[str, WebPatchResult] = {} + + async def fetch_from_intel_refs( + self, + candidates: dict[str, list[str]], + cve_id: str, + cve_description: str | None = None, + llm: "BaseChatModel | None" = None, + ) -> WebPatchResult | None: + """Fetch patch from intel reference URLs in priority order. + + Priority: + 1. /commit/ URLs (ubuntu_patches first, then ghsa, nvd, rhsa, ubuntu) + 2. /pull/ URLs (ghsa, nvd, rhsa, ubuntu) + 3. Chromium issue URLs (require Gerrit/Gitiles resolution) + + Args: + candidates: Dict mapping source to list of URLs + Example: {"ghsa": ["https://github.com/.../commit/..."], "nvd": [...]} + cve_id: CVE identifier for logging and result + cve_description: Optional CVE description for Chromium CL disambiguation + llm: Optional LangChain LLM for Chromium CL selection when multiple MERGED CLs exist + + Returns: + WebPatchResult on success, None if no valid patch found + """ + prioritized = self._prioritize_and_dedupe(candidates) + + if not prioritized: + logger.debug("No candidate URLs found for %s", cve_id) + return None + + urls_tried = 0 + for url, source, url_type in prioritized: + urls_tried += 1 + + if url_type == "chromium_issue": + # Handle Chromium issue URLs via Gerrit/Gitiles + result = await self._fetch_chromium_issue( + url, cve_id, source, cve_description, llm + ) + else: + # Standard commit/PR URL handling + result = await self.fetch_from_url(url, cve_id, source=source, url_type_hint=url_type) + + if result and result.parsed_patch: + logger.info( + "Intel refs: Found patch for %s from %s (%s, %s)", + cve_id, source, url_type, result.platform + ) + return result + + logger.info("Intel refs: No valid patch found for %s after trying %d URLs", cve_id, urls_tried) + return None + + async def fetch_from_url( + self, + url: str, + cve_id: str, + source: str | None = None, + url_type_hint: str | None = None, + ) -> WebPatchResult | None: + """Fetch and parse a patch from a single URL. + + Args: + url: URL to fetch (commit, PR, or patch URL) + cve_id: CVE identifier + source: Source provider (for result metadata) + url_type_hint: URL type hint (commit/pull) + + Returns: + WebPatchResult on success, None on failure + """ + cache_key = url.lower().rstrip("/") + if cache_key in self._cache: + return self._cache[cache_key] + + # Resolve URL to patch download URL + resolved = self._resolve_to_patch_url(url) + if not resolved: + logger.debug("Could not resolve URL to patch format: %s", url) + return None + + # Fetch patch content + patch_content = await self._fetch_patch_content(resolved.patch_url) + if not patch_content: + return None + + # Extract metadata and parse + commit_message, commit_author, commit_date = _extract_commit_metadata(patch_content) + commit_sha = resolved.commit_sha or "unknown" + parsed_patch = _parse_patch_content( + patch_content, + f"{cve_id}_{commit_sha[:8]}.patch" + ) + + if not parsed_patch: + logger.warning("Failed to parse patch content from %s", url) + return None + + result = WebPatchResult( + cve_id=cve_id, + fixed_commit=commit_sha[:8] if len(commit_sha) > 8 else commit_sha, + repo_url=resolved.repo_url or "", + patch_url=resolved.patch_url, + patch_content=patch_content, + parsed_patch=parsed_patch, + commit_message=commit_message, + commit_author=commit_author, + commit_date=commit_date, + source=source, + url_type=url_type_hint or resolved.url_type, + platform=resolved.platform, + ) + + # Cache successful result + self._cache[cache_key] = result + return result + + async def _fetch_chromium_issue( + self, + issue_url: str, + cve_id: str, + source: str, + cve_description: str | None, + llm: "BaseChatModel | None", + ) -> WebPatchResult | None: + """Fetch patch for a Chromium issue via Gerrit/Gitiles. + + Flow: + 1. Extract bug ID from issue URL + 2. Search Gerrit for CLs with that bug ID + 3. Filter to MERGED CLs only + 4. Select the correct CL (single = use it, multiple = LLM selection) + 5. Get commit SHA from selected CL + 6. Fetch patch from Gitiles + + Args: + issue_url: Chromium issue URL (e.g., https://issues.chromium.org/issues/466192044) + cve_id: CVE identifier + source: Source provider (ghsa, nvd, etc.) + cve_description: CVE description for LLM disambiguation + llm: Optional LangChain LLM for CL selection + + Returns: + WebPatchResult on success, None on failure + """ + # Extract bug ID from URL + match = CHROMIUM_ISSUE_PATTERN.match(issue_url) + if not match: + logger.debug("Invalid Chromium issue URL: %s", issue_url) + return None + + bug_id = match.group(1) + logger.info("Chromium issue: Searching Gerrit for bug %s", bug_id) + + # Search Gerrit for CLs with this bug ID + raw_changes = await search_changes_by_bug(self._session, bug_id) + if not raw_changes: + logger.info("Chromium issue: No CLs found for bug %s", bug_id) + return None + + # Filter to MERGED only + merged = list_merged_changes(raw_changes) + if not merged: + logger.info("Chromium issue: No MERGED CLs found for bug %s", bug_id) + return None + + logger.info( + "Chromium issue: Found %d MERGED CLs for bug %s: %s", + len(merged), bug_id, [c.submission_id for c in merged] + ) + + # Select the correct CL + selected_id = await select_gerrit_change( + merged, cve_id, cve_description or "", llm + ) + if selected_id is None: + logger.info("Chromium issue: Could not select CL for bug %s", bug_id) + return None + + # Find the selected candidate to get project info + selected = next((c for c in merged if c.submission_id == selected_id), None) + if not selected: + logger.warning("Chromium issue: Selected ID %d not found in candidates", selected_id) + return None + + logger.info( + "Chromium issue: Selected CL %d (%s) for bug %s", + selected_id, selected.project, bug_id + ) + + # Get commit SHA from the selected CL + commit_sha = await get_current_commit_sha(self._session, selected_id) + if not commit_sha: + logger.warning("Chromium issue: Could not get commit SHA for CL %d", selected_id) + return None + + # Build Gitiles URL and fetch patch + repo_url = project_to_gitiles_repo_url(selected.project) + patch_url = build_gitiles_patch_url(repo_url, commit_sha) + + logger.info("Chromium issue: Fetching patch from %s", patch_url) + + # Fetch patch content (Gitiles base64 decoding handled automatically) + patch_content = await self._fetch_patch_content(patch_url) + if not patch_content: + logger.warning("Chromium issue: Failed to fetch patch from Gitiles") + return None + + # Parse patch + commit_message, commit_author, commit_date = _extract_commit_metadata(patch_content) + parsed_patch = _parse_patch_content( + patch_content, + f"{cve_id}_{commit_sha[:8]}.patch" + ) + + if not parsed_patch: + logger.warning("Chromium issue: Failed to parse patch from Gitiles") + return None + + return WebPatchResult( + cve_id=cve_id, + fixed_commit=commit_sha[:8] if len(commit_sha) > 8 else commit_sha, + repo_url=repo_url, + patch_url=patch_url, + patch_content=patch_content, + parsed_patch=parsed_patch, + commit_message=commit_message or selected.subject, + commit_author=commit_author, + commit_date=commit_date, + source=source, + url_type="chromium_issue", + platform="gitiles", + ) + + def _prioritize_and_dedupe( + self, + candidates: dict[str, list[str]], + ) -> list[tuple[str, str, str]]: + """Prioritize and deduplicate candidate URLs. + + Returns list of (url, source, url_type) tuples in priority order. + + Priority order: + 1. /commit/ URLs (direct patches, highest confidence) + 2. /pull/ URLs (may contain multiple commits) + 3. Chromium issue URLs (require Gerrit/Gitiles resolution, lowest priority) + """ + seen: set[str] = set() + result: list[tuple[str, str, str]] = [] + + # Priority 1: /commit/ URLs (and kernel.org commit URLs) + # Order: ubuntu_patches first (curated), then other sources + commit_sources = ["ubuntu_patches", "ghsa", "nvd", "rhsa", "ubuntu"] + for source in commit_sources: + for url in candidates.get(source, []): + normalized = self._normalize_url_for_dedupe(url) + if self._is_commit_url(url) and normalized not in seen: + seen.add(normalized) + result.append((url, source, "commit")) + + # Priority 2: /pull/ URLs + pr_sources = ["ghsa", "nvd", "rhsa", "ubuntu"] + for source in pr_sources: + for url in candidates.get(source, []): + normalized = self._normalize_url_for_dedupe(url) + if self._is_pr_url(url) and normalized not in seen: + seen.add(normalized) + result.append((url, source, "pull")) + + # Priority 3: Chromium issue URLs (lowest priority - require Gerrit/Gitiles resolution) + chromium_sources = ["ghsa", "nvd", "rhsa", "ubuntu"] + for source in chromium_sources: + for url in candidates.get(source, []): + normalized = self._normalize_url_for_dedupe(url) + if self._is_chromium_issue_url(url) and normalized not in seen: + seen.add(normalized) + result.append((url, source, "chromium_issue")) + + return result + + def _normalize_url_for_dedupe(self, url: str) -> str: + """Normalize URL for deduplication.""" + # Remove trailing slash, convert to lowercase + normalized = url.rstrip("/").lower() + # Remove .patch suffix for comparison + if normalized.endswith(".patch"): + normalized = normalized[:-6] + # Remove query string for commit URLs (but keep for kernel.org) + if "github.com" in normalized and "?" in normalized: + normalized = normalized.split("?")[0] + return normalized + + def _is_commit_url(self, url: str) -> bool: + """Check if URL is a commit URL (GitHub, kernel.org, or gitweb).""" + url_lower = url.lower() + return ( + "/commit/" in url_lower or + "/c/" in url_lower or # kernel.org short form + "?id=" in url_lower or # cgit path-info form + ";a=commit;" in url_lower # gitweb form + ) + + def _is_pr_url(self, url: str) -> bool: + """Check if URL is a PR URL.""" + return "/pull/" in url.lower() + + def _is_chromium_issue_url(self, url: str) -> bool: + """Check if URL is a Chromium issue tracker URL. + + These URLs point to issues.chromium.org/issues/ and require + resolution via Gerrit/Gitiles to fetch the actual patch. + """ + return bool(CHROMIUM_ISSUE_PATTERN.match(url)) + + def _resolve_to_patch_url(self, url: str) -> ResolvedUrl | None: + """Resolve a URL to its patch download URL. + + Dispatches to platform-specific handlers. + """ + # Try GitHub first + resolved = self._resolve_github_url(url) + if resolved: + return resolved + + # Try gitweb (e.g. git.samba.org) + resolved = self._resolve_gitweb_url(url) + if resolved: + return resolved + + # Try kernel.org + resolved = self._resolve_kernel_org_url(url) + if resolved: + return resolved + + # Try Gitiles (Chromium/Google) + resolved = self._resolve_gitiles_url(url) + if resolved: + return resolved + + return None + + def _resolve_github_url(self, url: str) -> ResolvedUrl | None: + """Resolve GitHub commit or PR URL to patch URL.""" + # GitHub commit URL + match = _GITHUB_COMMIT_PATTERN.match(url) + if match: + repo_path, commit_sha = match.groups() + patch_url = f"https://github.com/{repo_path}/commit/{commit_sha}.patch" + return ResolvedUrl( + patch_url=patch_url, + platform="github", + url_type="commit", + repo_url=f"https://github.com/{repo_path}", + commit_sha=commit_sha, + ) + + # GitHub PR URL + match = _GITHUB_PR_PATTERN.match(url) + if match: + repo_path, pr_number = match.groups() + patch_url = f"https://github.com/{repo_path}/pull/{pr_number}.patch" + return ResolvedUrl( + patch_url=patch_url, + platform="github", + url_type="pull", + repo_url=f"https://github.com/{repo_path}", + commit_sha=f"PR-{pr_number}", + ) + + # Ubuntu-style "upstream: https://github.com/.../commit/..." + match = _UBUNTU_PATCH_URL_PATTERN.search(url) + if match: + commit_url, commit_sha = match.groups() + patch_url = f"{commit_url}.patch" + repo_url = commit_url.rsplit("/commit/", 1)[0] + return ResolvedUrl( + patch_url=patch_url, + platform="github", + url_type="commit", + repo_url=repo_url, + commit_sha=commit_sha, + ) + + return None + + def _resolve_gitweb_url(self, url: str) -> ResolvedUrl | None: + """Resolve gitweb commit or patch URL (e.g. git.samba.org).""" + match = _GITWEB_PATCH_PATTERN.match(url) + if match: + base, project, commit_sha = match.groups() + repo_url = f"{base}/{project}" + return ResolvedUrl( + patch_url=url, + platform="gitweb", + url_type="commit", + repo_url=repo_url, + commit_sha=commit_sha, + ) + + match = _GITWEB_COMMIT_PATTERN.match(url) + if match: + base, project, commit_sha = match.groups() + patch_url = f"{base}/?p={project};a=patch;h={commit_sha}" + repo_url = f"{base}/{project}" + return ResolvedUrl( + patch_url=patch_url, + platform="gitweb", + url_type="commit", + repo_url=repo_url, + commit_sha=commit_sha, + ) + + return None + + def _resolve_kernel_org_url(self, url: str) -> ResolvedUrl | None: + """Resolve kernel.org URL to patch URL.""" + # Already a patch URL + if _KERNEL_CGIT_PATCH_PATTERN.match(url): + # Extract commit SHA from ?id= parameter + sha_match = re.search(r"\?id=([a-f0-9]+)", url) + commit_sha = sha_match.group(1) if sha_match else "unknown" + repo_url = url.split("/patch/")[0] if "/patch/" in url else None + return ResolvedUrl( + patch_url=url, + platform="kernel.org", + url_type="commit", + repo_url=repo_url, + commit_sha=commit_sha, + ) + + # cgit commit URL: .../commit/?id= + match = _KERNEL_CGIT_COMMIT_PATTERN.match(url) + if match: + repo_base, commit_sha = match.groups() + patch_url = f"{repo_base}/patch/?id={commit_sha}" + return ResolvedUrl( + patch_url=patch_url, + platform="kernel.org", + url_type="commit", + repo_url=repo_base, + commit_sha=commit_sha, + ) + + # Short form: /stable/c/, /torvalds/c/, etc. + match = _KERNEL_SHORT_PATTERN.match(url) + if match: + tree, commit_sha = match.groups() + repo_path = _KERNEL_SHORT_PATH_MAP.get(tree) + if repo_path: + patch_url = f"https://git.kernel.org/pub/scm/{repo_path}/patch/?id={commit_sha}" + repo_url = f"https://git.kernel.org/pub/scm/{repo_path}" + return ResolvedUrl( + patch_url=patch_url, + platform="kernel.org", + url_type="commit", + repo_url=repo_url, + commit_sha=commit_sha, + ) + else: + logger.warning("Unknown kernel.org tree: %s", tree) + + return None + + def _resolve_gitiles_url(self, url: str) -> ResolvedUrl | None: + """Resolve Gitiles commit URL to patch URL. + + Gitiles is used by Chromium and other Google projects. + The patch is fetched via /+/^! with ?format=TEXT which returns base64-encoded content. + """ + # Decode URL-encoded characters for pattern matching + decoded_url = unquote(url) + + match = _GITILES_COMMIT_PATTERN.match(decoded_url) + if not match: + # Try with the original URL (might have literal ^!) + match = _GITILES_COMMIT_PATTERN.match(url) + if not match: + return None + + repo_url, commit_sha = match.groups() + # Build patch URL with ^! suffix (pre-encoded) and format=TEXT + # Must use %5E%21 with yarl.URL(encoded=True) to avoid mixed encoding + patch_url = f"{repo_url}/+/{commit_sha}%5E%21?format=TEXT" + + return ResolvedUrl( + patch_url=patch_url, + platform="gitiles", + url_type="commit", + repo_url=repo_url, + commit_sha=commit_sha, + ) + + async def _fetch_patch_content(self, patch_url: str) -> str | None: + """Fetch patch content from URL with rate limit handling. + + Automatically handles Gitiles base64-encoded responses. + """ + is_gitiles = ".googlesource.com" in patch_url and "format=TEXT" in patch_url + + # For Gitiles URLs, use a fresh session with yarl.URL(encoded=True) + # to preserve the exact %5E%21 encoding and avoid re-encoding issues + if is_gitiles: + content = await self._fetch_gitiles_patch(patch_url) + if content: + return self._decode_gitiles_response(content) + else: + return None + + try: + async with request_with_retry( + session=self._session, + request_kwargs={ + 'method': 'GET', + 'url': patch_url, + 'timeout': self._timeout, + 'headers': self._headers, + }, + max_retries=3, + sleep_time=0.5, + log_on_error=False, + ) as response: + content = await response.text() + return content + + except aiohttp.ClientResponseError as e: + if e.status == 404: + logger.debug("Patch not found: %s", patch_url) + elif e.status == 429: + logger.warning("Rate limited on %s, skipping", patch_url) + else: + logger.warning("Patch fetch failed: %s - %s", patch_url, e) + return None + except Exception as e: + logger.warning("Patch fetch failed: %s - %s", patch_url, e) + return None + + async def _fetch_gitiles_patch(self, patch_url: str) -> str | None: + """Fetch patch from Gitiles using a fresh session. + + Gitiles requires exact URL encoding for ^! suffix (%5E%21). + Using a fresh session avoids any URL re-encoding issues from shared sessions. + """ + import requests + # 2. Fetch the encoded data + response = requests.get(patch_url,timeout=(3.05, 15)) + + if response.status_code == 200: + return response.text + else: + logger.warning("Gitiles patch fetch failed: %s - %s", patch_url, response.status_code) + return None + + + def _decode_gitiles_response(self, content: str) -> str | None: + """Decode Gitiles base64-encoded patch response. + + Args: + content: Base64-encoded content from Gitiles ?format=TEXT endpoint + + Returns: + Decoded patch content, or None on decode failure + """ + try: + decoded_bytes = base64.b64decode(content) + return decoded_bytes.decode("utf-8") + except Exception as e: + logger.warning("Failed to decode Gitiles base64 response: %s", e) + return None + + +# --------------------------------------------------------------------------- +# OSVClient - OSV API Client +# --------------------------------------------------------------------------- + +class OSVClient: + """Client for querying OSV API and fetching fix patches. + + Usage: + async with aiohttp.ClientSession() as session: + fetcher = WebPatchFetcher(session) + client = OSVClient(session, fetcher) + result = await client.get_fix_patch("CVE-2024-1234", "3.0.7", "openssl") + """ + + def __init__( + self, + session: aiohttp.ClientSession, + patch_fetcher: WebPatchFetcher | None = None, + osv_timeout: int = _OSV_TIMEOUT_SECONDS, + ): + self._session = session + self._patch_fetcher = patch_fetcher or WebPatchFetcher(session) + self._osv_timeout = aiohttp.ClientTimeout(total=osv_timeout) + + async def get_fix_patch( + self, + cve_id: str, + upstream_version: str | None = None, # pylint: disable=unused-argument + package_name: str | None = None, # pylint: disable=unused-argument + ) -> WebPatchResult | None: + """Query OSV API and fetch the fix patch. + + Args: + cve_id: CVE identifier (e.g., "CVE-2024-1234") + upstream_version: Upstream version (e.g., "3.0.7") - reserved for future filtering + package_name: Optional package name - reserved for future filtering + + Returns: + WebPatchResult with patch data, or None if no fix found + """ + try: + osv_data = await self._query_osv(cve_id) + if not osv_data: + return None + + # Try to get fix commit URL from references first (most specific) + patch_url = self._extract_commit_from_references(osv_data) + repo_url = None + + if patch_url: + # Extract repo_url from the patch URL + if '/commit/' in patch_url: + repo_url = patch_url.split('/commit/')[0] + elif '/pull/' in patch_url: + repo_url = patch_url.split('/pull/')[0] + logger.info("OSV: Found fix commit in references for %s", cve_id) + else: + return None + #OSV affected GIT fixed events mark version boundaries (e.g. release tags), + #not security patches, so using them as + # fix diffs sends the checker the wrong upstream patch. + # Find fix commit from affected block + #affected = self._find_matching_affected(osv_data) + #if not affected: + # logger.info("OSV: No affected block with fix found for %s", cve_id) + # return None + + #range_info = self._extract_fix_commit(affected) + #if not range_info.fixed_commit or not range_info.repo_url: + # logger.info("OSV: No fixed commit found for %s", cve_id) + # return None + + #patch_url = self._build_patch_url(range_info.repo_url, range_info.fixed_commit) + #repo_url = range_info.repo_url + + if not patch_url: + logger.info("OSV: Could not build patch URL for %s (unsupported repo host?)", cve_id) + return None + + # Use WebPatchFetcher to fetch and parse + url_for_resolution = patch_url + if patch_url.endswith('.patch') and 'github.com' in patch_url: + url_for_resolution = patch_url[:-6] + result = await self._patch_fetcher.fetch_from_url( + url_for_resolution, + cve_id, + source="osv", + ) + + if result: + # Ensure we have the correct repo_url + if repo_url and not result.repo_url: + result.repo_url = repo_url + return result + + return None + + except Exception: + logger.warning("OSV patch retrieval failed for %s", cve_id, exc_info=True) + return None + + async def _query_osv(self, cve_id: str) -> dict | None: + """Query OSV API for CVE data.""" + url = f"{_OSV_API_URL}{cve_id}" + try: + async with request_with_retry( + session=self._session, + request_kwargs={ + 'method': 'GET', + 'url': url, + 'timeout': self._osv_timeout, + }, + max_retries=3, + sleep_time=0.5, + log_on_error=False, + ) as response: + return await response.json() + except aiohttp.ClientResponseError as e: + if e.status == 404: + logger.info("OSV: CVE %s not found", cve_id) + else: + logger.warning("OSV query failed for %s: %s", cve_id, e) + return None + except Exception as e: + logger.warning("OSV query failed for %s: %s", cve_id, e) + return None + + def _extract_commit_from_references(self, osv_data: dict) -> str | None: + """Extract fix commit URL from OSV references.""" + references = osv_data.get("references", []) + + for ref in references: + if ref.get("type") == "FIX": + url = ref.get("url", "") + if "github.com" in url and ("/commit/" in url or "/pull/" in url): + if not url.endswith(".patch"): + return f"{url}.patch" + return url + if ";a=commit;" in url: + return url.replace(";a=commit;", ";a=patch;") + if ";a=patch;" in url: + return url + + return None + + def _find_matching_affected(self, osv_data: dict) -> dict | None: + """Find an affected block with a GIT range containing a fixed commit.""" + for affected in osv_data.get("affected", []): + for range_block in affected.get("ranges", []): + if range_block.get("type") == "GIT": + for event in range_block.get("events", []): + if "fixed" in event: + return affected + return None + + def _extract_fix_commit(self, affected: dict) -> OSVAffectedRange: + """Extract fixed commit hash and repo URL from affected block.""" + result = OSVAffectedRange() + + ranges = affected.get("ranges", []) + for range_block in ranges: + if range_block.get("type") != "GIT": + continue + + repo = range_block.get("repo") + if repo: + result.repo_url = repo + + events = range_block.get("events", []) + for event in events: + if "introduced" in event and event["introduced"] != "0": + result.introduced_commit = event["introduced"] + if "fixed" in event: + result.fixed_commit = event["fixed"] + + if result.fixed_commit: + break + + return result + + def _build_patch_url(self, repo_url: str, commit_sha: str) -> str | None: + """Build patch download URL from repo URL and commit SHA.""" + patch_url = build_patch_url_from_repo(repo_url, commit_sha) + if not patch_url: + logger.debug("Unsupported repo URL: %s", repo_url) + return patch_url + + +# --------------------------------------------------------------------------- +# Shared helper — used by both full pipeline and package checker paths +# --------------------------------------------------------------------------- + +async def fetch_patch_for_cve( + session: aiohttp.ClientSession, + candidates: dict[str, list[str]], + vuln_id: str, + cve_description: str | None = None, + llm: "BaseChatModel | None" = None, + upstream_version: str | None = None, + package_name: str | None = None, +) -> WebPatchResult | None: + """Try intel-ref URLs first, then fall back to OSV API. + + This is the shared fetch logic used by both the full pipeline + (cve_fetch_patches) and the package checker (upstream_search_preprocess). + + Args: + session: aiohttp session to reuse for all HTTP calls. + candidates: URL candidates from extract_commit_url_candidates(). + vuln_id: CVE identifier. + cve_description: Optional CVE description for Chromium CL disambiguation. + llm: Optional LLM for Chromium CL selection. + upstream_version: Optional version for OSV filtering. + package_name: Optional package name for OSV filtering. + + Returns: + WebPatchResult on success, None if no valid patch found. + """ + fetcher = WebPatchFetcher(session=session) + has_candidates = any(urls for urls in candidates.values()) + + if has_candidates: + result = await fetcher.fetch_from_intel_refs( + candidates, vuln_id, cve_description=cve_description, llm=llm, + ) + if result and result.parsed_patch: + return result + + client = OSVClient(session=session, patch_fetcher=fetcher) + result = await client.get_fix_patch(vuln_id, upstream_version, package_name) + if result and result.parsed_patch: + return result + + return None diff --git a/tests/agent_test_helpers.py b/tests/agent_test_helpers.py new file mode 100644 index 000000000..2d60aed2b --- /dev/null +++ b/tests/agent_test_helpers.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import MagicMock + +from vuln_analysis.tools.tool_names import ToolNames + + +class MockTool: + def __init__(self, name: str): + self.name = name + + +ALL_TOOLS = [ + MockTool(ToolNames.CODE_SEMANTIC_SEARCH), + MockTool(ToolNames.DOCS_SEMANTIC_SEARCH), + MockTool(ToolNames.CODE_KEYWORD_SEARCH), + MockTool(ToolNames.FUNCTION_LOCATOR), + MockTool(ToolNames.CALL_CHAIN_ANALYZER), + MockTool(ToolNames.FUNCTION_CALLER_FINDER), + MockTool(ToolNames.CVE_WEB_SEARCH), + MockTool(ToolNames.CONTAINER_ANALYSIS_DATA), + MockTool(ToolNames.FUNCTION_LIBRARY_VERSION_FINDER), + MockTool(ToolNames.CONFIGURATION_SCANNER), + MockTool(ToolNames.IMPORT_USAGE_ANALYZER), +] + + +def make_builder(tools=None): + builder = MagicMock() + builder.get_tools = MagicMock(return_value=list(tools if tools is not None else ALL_TOOLS)) + return builder + + +def make_config(**overrides): + config = MagicMock() + config.tool_names = overrides.get("tool_names", []) + config.transitive_search_tool_enabled = overrides.get("transitive_search_tool_enabled", True) + config.cve_web_search_enabled = overrides.get("cve_web_search_enabled", True) + config.max_iterations = 10 + return config + + +def make_state(code_vdb_path="/path", doc_vdb_path="/path", code_index_path="/path"): + state = MagicMock() + state.code_vdb_path = code_vdb_path + state.doc_vdb_path = doc_vdb_path + state.code_index_path = code_index_path + return state diff --git a/tests/test_agent_registry.py b/tests/test_agent_registry.py new file mode 100644 index 000000000..6260495df --- /dev/null +++ b/tests/test_agent_registry.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Unit tests for agent_registry: register_agent decorator, get_agent_class, get_all_agent_types. +""" + +import pytest + +from vuln_analysis.functions.agent_registry import ( + _AGENT_REGISTRY, + register_agent, + get_agent_class, + get_all_agent_types, +) + + +class TestRealAgentRegistration: + """Verify that importing the agent modules registers both agents.""" + + def test_reachability_registered(self): + import vuln_analysis.functions.reachability_agent # noqa: F401 + assert "reachability" in _AGENT_REGISTRY + + def test_code_understanding_registered(self): + import vuln_analysis.functions.code_understanding_agent # noqa: F401 + assert "code_understanding" in _AGENT_REGISTRY + + def test_get_all_agent_types_contains_both(self): + import vuln_analysis.functions.reachability_agent # noqa: F401 + import vuln_analysis.functions.code_understanding_agent # noqa: F401 + types = get_all_agent_types() + assert "reachability" in types + assert "code_understanding" in types + + def test_get_agent_class_reachability(self): + from vuln_analysis.functions.reachability_agent import ReachabilityAgent + assert get_agent_class("reachability") is ReachabilityAgent + + def test_get_agent_class_code_understanding(self): + from vuln_analysis.functions.code_understanding_agent import CodeUnderstandingAgent + assert get_agent_class("code_understanding") is CodeUnderstandingAgent + + +class TestGetAgentClassErrors: + """Test error cases for get_agent_class.""" + + def test_unknown_type_raises_key_error(self): + with pytest.raises(KeyError, match="Unknown agent type 'nonexistent'"): + get_agent_class("nonexistent") + + def test_error_message_lists_registered_types(self): + import vuln_analysis.functions.reachability_agent # noqa: F401 + import vuln_analysis.functions.code_understanding_agent # noqa: F401 + with pytest.raises(KeyError) as exc_info: + get_agent_class("bogus") + msg = str(exc_info.value) + assert "reachability" in msg + assert "code_understanding" in msg + + +class TestRegisterAgentDecorator: + """Test the register_agent decorator mechanics using a dummy class.""" + + def setup_method(self): + self._saved = dict(_AGENT_REGISTRY) + + def teardown_method(self): + _AGENT_REGISTRY.clear() + _AGENT_REGISTRY.update(self._saved) + + def test_decorator_registers_class(self): + @register_agent("test_dummy") + class DummyAgent: + pass + + assert get_agent_class("test_dummy") is DummyAgent + + def test_decorator_returns_original_class(self): + @register_agent("test_dummy2") + class DummyAgent: + pass + + assert DummyAgent.__name__ == "DummyAgent" + + def test_re_registration_replaces_class(self): + @register_agent("test_replace") + class First: + pass + + @register_agent("test_replace") + class Second: + pass + + assert get_agent_class("test_replace") is Second + + def test_registered_type_appears_in_all_types(self): + @register_agent("test_listed") + class Listed: + pass + + assert "test_listed" in get_all_agent_types() diff --git a/tests/test_async_http_utils.py b/tests/test_async_http_utils.py new file mode 100644 index 000000000..828010517 --- /dev/null +++ b/tests/test_async_http_utils.py @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Unit tests for retry_async decorator in async_http_utils.py. +""" + +import aiohttp +import pytest +from unittest.mock import Mock + +from vuln_analysis.utils.async_http_utils import retry_async + +# Constants for test data +SUCCESS_RESULT = "Success" +RETRY_THRESHOLD = 3 # Number of attempts before success + + +@pytest.mark.asyncio +async def test_basic_retry_on_timeout(): + """Test that retry_async retries on TimeoutError.""" + call_count = 0 + + @retry_async() + async def failing_function(): + nonlocal call_count + call_count += 1 + # Fail on first 2 attempts, succeed on 3rd + if call_count < RETRY_THRESHOLD: + raise TimeoutError("Network timeout") + return SUCCESS_RESULT + + result = await failing_function() + + assert result == SUCCESS_RESULT + # Verify retry occurred: 2 failures + 1 success = 3 total attempts + assert call_count == RETRY_THRESHOLD + + +@pytest.mark.asyncio +async def test_retry_on_500(): + """Test that retry_async retries on server errors.""" + call_count = 0 + + @retry_async() + async def failing_function(): + nonlocal call_count + call_count += 1 + # Simulate transient server errors that eventually resolve + if call_count < RETRY_THRESHOLD: + raise Exception("Server error 500") + return SUCCESS_RESULT + + result = await failing_function() + + assert result == SUCCESS_RESULT + # Verify retry occurred: function called 3 times before success + assert call_count == RETRY_THRESHOLD + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status_code", [402, 429]) +async def test_no_retry_when_excluded(status_code): + """Test that retry_async does NOT retry when status is excluded.""" + call_count = 0 + + @retry_async(exclude_http_statuses=[status_code]) + async def failing_function(): + nonlocal call_count + call_count += 1 + # Always raise excluded status (should NOT be retried) + raise aiohttp.ClientResponseError( + request_info=Mock(), + history=(), + status=status_code + ) + + # Expect immediate exception without retry + with pytest.raises(aiohttp.ClientResponseError): + await failing_function() + + # Verify NO retry: should be called exactly once then exception raised + assert call_count == 1 diff --git a/tests/test_base_graph_agent.py b/tests/test_base_graph_agent.py new file mode 100644 index 000000000..834c84cf7 --- /dev/null +++ b/tests/test_base_graph_agent.py @@ -0,0 +1,848 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Unit tests for BaseGraphAgent: should_continue routing, default hooks, agent_type property, +thought_node context pruning. +""" + +import pytest +from unittest.mock import MagicMock, AsyncMock, patch +from langchain_core.messages import SystemMessage, AIMessage, HumanMessage, ToolMessage + +from vuln_analysis.functions.base_graph_agent import BaseGraphAgent +from vuln_analysis.functions.react_internals import Thought, ToolCall, Observation + + +class _ConcreteAgent(BaseGraphAgent): + """Minimal concrete subclass for testing base class behavior.""" + + async def pre_process_node(self, state): + return state + + @staticmethod + def get_tools(builder, config, state): + return [] + + @staticmethod + def create_rules_tracker(): + return MagicMock() + + +def _make_agent(): + mock_llm = MagicMock() + mock_llm.with_structured_output = MagicMock(return_value=MagicMock()) + config = MagicMock() + config.max_iterations = 10 + return _ConcreteAgent(tools=[], llm=mock_llm, config=config) + + +class TestShouldContinue: + """Test should_continue routing logic.""" + + @pytest.mark.asyncio + async def test_returns_end_on_finish_mode(self): + agent = _make_agent() + thought = Thought(thought="done", mode="finish", actions=None, final_answer="answer") + state = {"thought": thought, "step": 3, "max_steps": 10} + result = await agent.should_continue(state) + assert result == "__end__" + + @pytest.mark.asyncio + async def test_returns_forced_finish_at_max_steps(self): + agent = _make_agent() + thought = Thought( + thought="still working", + mode="act", + actions=ToolCall(tool="some_tool", query="q", reason="testing"), + final_answer=None, + ) + state = {"thought": thought, "step": 10, "max_steps": 10} + result = await agent.should_continue(state) + assert result == "forced_finish_node" + + @pytest.mark.asyncio + async def test_returns_forced_finish_beyond_max_steps(self): + agent = _make_agent() + thought = Thought( + thought="still working", + mode="act", + actions=ToolCall(tool="some_tool", query="q", reason="testing"), + final_answer=None, + ) + state = {"thought": thought, "step": 15, "max_steps": 10} + result = await agent.should_continue(state) + assert result == "forced_finish_node" + + @pytest.mark.asyncio + async def test_returns_tool_node_when_continuing(self): + agent = _make_agent() + thought = Thought( + thought="need more info", + mode="act", + actions=ToolCall(tool="some_tool", query="q", reason="testing"), + final_answer=None, + ) + state = {"thought": thought, "step": 3, "max_steps": 10} + result = await agent.should_continue(state) + assert result == "tool_node" + + @pytest.mark.asyncio + async def test_returns_thought_node_when_thought_is_none(self): + agent = _make_agent() + state = {"thought": None, "step": 0, "max_steps": 10} + result = await agent.should_continue(state) + assert result == "thought_node" + + @pytest.mark.asyncio + async def test_forced_finish_when_thought_none_at_max_steps(self): + """Step limit must be enforced even when thought is None (e.g. after + check_finish_allowed repeatedly blocks). Without this, the agent + self-loops thought_node→thought_node until GraphRecursionError.""" + agent = _make_agent() + state = {"thought": None, "step": 10, "max_steps": 10} + result = await agent.should_continue(state) + assert result == "forced_finish_node" + + @pytest.mark.asyncio + async def test_forced_finish_when_thought_none_beyond_max_steps(self): + agent = _make_agent() + state = {"thought": None, "step": 15, "max_steps": 10} + result = await agent.should_continue(state) + assert result == "forced_finish_node" + + @pytest.mark.asyncio + async def test_uses_config_max_iterations_as_fallback(self): + agent = _make_agent() + thought = Thought( + thought="working", + mode="act", + actions=ToolCall(tool="some_tool", query="q", reason="testing"), + final_answer=None, + ) + state = {"thought": thought, "step": 10} + result = await agent.should_continue(state) + assert result == "forced_finish_node" + + +class TestDefaultHooks: + """Test default hook implementations on BaseGraphAgent.""" + + def test_post_observation_returns_empty_dict(self): + agent = _make_agent() + result = agent.post_observation(state={}, tool_used="X", tool_output="Y", tool_input_detail="Z") + assert result == {} + + def test_should_truncate_returns_false(self): + agent = _make_agent() + result = agent.should_truncate_tool_output(state={}, tool_used="X") + assert result is False + + def test_agent_type_property(self): + agent = _make_agent() + assert agent.agent_type == "base" + + def test_build_comprehension_context_returns_full_context(self): + agent = _make_agent() + state = {"critical_context": ["CVE Description: test vuln", "Vulnerable module: xstream"]} + result = agent.build_comprehension_context(state) + assert "CVE Description: test vuln" in result + assert "Vulnerable module: xstream" in result + + def test_build_comprehension_context_empty_state(self): + agent = _make_agent() + assert agent.build_comprehension_context({}) == "N/A" + + def test_build_comprehension_context_empty_list(self): + agent = _make_agent() + assert agent.build_comprehension_context({"critical_context": []}) == "N/A" + + @patch("vuln_analysis.functions.base_graph_agent.ctx_state") + def test_sanitize_findings_replaces_wrong_cve(self, mock_ctx): + intel = MagicMock() + intel.vuln_id = "CVE-2021-43859" + ws = MagicMock() + ws.cve_intel = [intel] + mock_ctx.get.return_value = ws + + agent = _make_agent() + findings = ["Found CVE-2020-26217 in code", "Package present"] + result = agent.sanitize_findings(findings, {}) + assert "CVE-2020-26217" not in result[0] + assert "the investigated vulnerability" in result[0] + assert result[1] == "Package present" + + @patch("vuln_analysis.functions.base_graph_agent.ctx_state") + def test_sanitize_findings_keeps_correct_cve(self, mock_ctx): + intel = MagicMock() + intel.vuln_id = "CVE-2021-43859" + ws = MagicMock() + ws.cve_intel = [intel] + mock_ctx.get.return_value = ws + + agent = _make_agent() + findings = ["Affects CVE-2021-43859"] + result = agent.sanitize_findings(findings, {}) + assert result == ["Affects CVE-2021-43859"] + + @patch("vuln_analysis.functions.base_graph_agent.ctx_state") + def test_sanitize_findings_empty_list(self, mock_ctx): + ws = MagicMock() + ws.cve_intel = [] + mock_ctx.get.return_value = ws + + agent = _make_agent() + assert agent.sanitize_findings([], {}) == [] + + +class TestInit: + """Test BaseGraphAgent constructor wires up LLM wrappers.""" + + def test_creates_four_structured_output_llms(self): + mock_llm = MagicMock() + config = MagicMock() + config.max_iterations = 10 + agent = _ConcreteAgent(tools=["t1", "t2"], llm=mock_llm, config=config) + + assert mock_llm.with_structured_output.call_count == 4 + assert agent.tools == ["t1", "t2"] + assert agent.config is config + + +def _make_thought_response(mode="finish", final_answer="done"): + return Thought(thought="thinking", mode=mode, actions=None, final_answer=final_answer) + + +def _long_content(n_words=500): + return " ".join(["word"] * n_words) + + +class TestThoughtNodePruning: + """Test that thought_node prunes messages when tokens exceed the limit.""" + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_prunes_middle_messages_when_over_limit(self, mock_tracer): + agent = _make_agent() + agent.config.context_window_token_limit = 100 + agent.thought_llm.ainvoke = AsyncMock(return_value=_make_thought_response()) + + long = _long_content(200) + state = { + "runtime_prompt": "system prompt", + "messages": [ + HumanMessage(content=long), + AIMessage(content=long), + ToolMessage(content=long, tool_call_id="tc1"), + AIMessage(content=long), + ToolMessage(content="recent tool output", tool_call_id="tc2"), + HumanMessage(content="recent question"), + ], + "observation": None, + "step": 2, + } + + await agent.thought_node(state) + + invoked_messages = agent.thought_llm.ainvoke.call_args[0][0] + num_original = 1 + 6 # system prompt + 6 state messages + assert len(invoked_messages) < num_original + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_no_pruning_when_under_limit(self, mock_tracer): + agent = _make_agent() + agent.config.context_window_token_limit = 50000 + agent.thought_llm.ainvoke = AsyncMock(return_value=_make_thought_response()) + + state = { + "runtime_prompt": "short prompt", + "messages": [ + HumanMessage(content="hello"), + AIMessage(content="response"), + ], + "observation": None, + "step": 1, + } + + await agent.thought_node(state) + + invoked_messages = agent.thought_llm.ainvoke.call_args[0][0] + contents = [m.content for m in invoked_messages if hasattr(m, "content")] + assert "hello" in contents + assert "response" in contents + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_preserves_system_prompt_and_last_message(self, mock_tracer): + agent = _make_agent() + agent.config.context_window_token_limit = 50 + agent.thought_llm.ainvoke = AsyncMock(return_value=_make_thought_response()) + + long = _long_content(200) + state = { + "runtime_prompt": "system prompt", + "messages": [ + HumanMessage(content=long), + AIMessage(content=long), + ToolMessage(content=long, tool_call_id="tc1"), + HumanMessage(content="latest question"), + ], + "observation": None, + "step": 3, + } + + await agent.thought_node(state) + + invoked_messages = agent.thought_llm.ainvoke.call_args[0][0] + contents = [m.content for m in invoked_messages if hasattr(m, "content")] + assert "system prompt" in contents + assert "latest question" in contents + assert long not in contents + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_pruning_includes_observation_context_in_count(self, mock_tracer): + agent = _make_agent() + agent.config.context_window_token_limit = 200 + agent.thought_llm.ainvoke = AsyncMock(return_value=_make_thought_response()) + + long = _long_content(100) + obs = Observation( + memory=[_long_content(50)], + results=[_long_content(50)], + ) + state = { + "runtime_prompt": "system prompt", + "messages": [ + HumanMessage(content=long), + AIMessage(content=long), + ToolMessage(content="tool out", tool_call_id="tc1"), + HumanMessage(content="question"), + ], + "observation": obs, + "step": 2, + } + + await agent.thought_node(state) + + invoked_messages = agent.thought_llm.ainvoke.call_args[0][0] + assert any("KNOWLEDGE" in m.content for m in invoked_messages if hasattr(m, "content") and isinstance(m.content, str)) + contents = [m.content for m in invoked_messages if hasattr(m, "content")] + assert long not in contents + + +class TestThoughtNodeBadToolArguments: + """Test that thought_node recovers from bad tool arguments instead of crashing. + + Mirrors the old AgentExecutor's handle_parsing_errors behavior: when the LLM + produces a ToolCall with missing required fields, the agent should get an error + message and retry rather than killing the entire graph. + """ + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_recovers_from_missing_arguments(self, mock_tracer): + """When all ToolCall fields are None, thought_node returns an error + HumanMessage with thought=None so should_continue loops back.""" + agent = _make_agent() + agent.config.context_window_token_limit = 50000 + + bad_actions = ToolCall( + tool="Function Library Version Finder", + package_name=None, + function_name=None, + query=None, + tool_input=None, + reason="check version", + ) + bad_response = Thought( + thought="check the version", + mode="act", + actions=bad_actions, + final_answer=None, + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=bad_response) + + state = { + "runtime_prompt": "system prompt", + "messages": [HumanMessage(content="Is SslHandler used?")], + "observation": None, + "step": 2, + } + + result = await agent.thought_node(state) + + assert result["thought"] is None + assert result["step"] == 3 + assert result["output"] == "waiting for the agent to respond" + assert len(result["messages"]) == 2 + ai_msg = result["messages"][0] + assert isinstance(ai_msg, AIMessage) + assert "check the version" in ai_msg.content + error_msg = result["messages"][1] + assert isinstance(error_msg, HumanMessage) + assert "ERROR" in error_msg.content + assert "Function Library Version Finder" in error_msg.content + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_recovery_routes_back_to_thought_node_bad_args(self, mock_tracer): + """After a bad-arguments recovery, should_continue returns 'thought_node' + because thought is None — the agent gets another chance.""" + agent = _make_agent() + + state = {"thought": None, "step": 3, "max_steps": 10} + route = await agent.should_continue(state) + assert route == "thought_node" + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_recovery_still_counts_toward_max_steps(self, mock_tracer): + """A bad-arguments iteration increments step, so the agent hits + forced_finish_node when step reaches max_steps.""" + agent = _make_agent() + agent.config.context_window_token_limit = 50000 + + bad_actions = ToolCall( + tool="Some Tool", + reason="testing", + ) + bad_response = Thought( + thought="trying something", + mode="act", + actions=bad_actions, + final_answer=None, + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=bad_response) + + state = { + "runtime_prompt": "prompt", + "messages": [HumanMessage(content="question")], + "observation": None, + "step": 9, + "max_steps": 10, + } + + result = await agent.thought_node(state) + + assert result["step"] == 10 + assert result["thought"] is None + + route = await agent.should_continue(result) + assert route == "forced_finish_node" + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_valid_tool_call_still_works(self, mock_tracer): + """Verify that valid tool calls are not affected by the ValueError handling.""" + agent = _make_agent() + agent.config.context_window_token_limit = 50000 + + good_actions = ToolCall( + tool="Configuration Scanner", + query="netty SSL settings", + reason="check config", + ) + good_response = Thought( + thought="scan for config", + mode="act", + actions=good_actions, + final_answer=None, + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=good_response) + + state = { + "runtime_prompt": "system prompt", + "messages": [HumanMessage(content="question")], + "observation": None, + "step": 1, + } + + result = await agent.thought_node(state) + + assert result["thought"] is good_response + assert result["step"] == 2 + ai_msg = result["messages"][0] + assert isinstance(ai_msg, AIMessage) + assert ai_msg.tool_calls[0]["name"] == "Configuration Scanner" + assert ai_msg.tool_calls[0]["args"] == {"query": "netty SSL settings"} + + +class TestCheckFinishAllowedBlocking: + """Test that blocked finishes include AIMessage and respect step limits.""" + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_blocked_finish_includes_ai_message(self, mock_tracer): + """When check_finish_allowed blocks, the LLM's finish attempt must be + recorded as an AIMessage so the chat model sees its own response and + the rejection in proper Human/AI alternation.""" + agent = _make_agent() + agent.config.context_window_token_limit = 50000 + agent.check_finish_allowed = MagicMock( + return_value=(False, "You MUST use Function Locator first.") + ) + + finish_response = Thought( + thought="I have enough info", + mode="finish", + actions=None, + final_answer="The function is not reachable.", + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=finish_response) + + state = { + "runtime_prompt": "system prompt", + "messages": [HumanMessage(content="Is the function reachable?")], + "observation": None, + "step": 2, + } + + result = await agent.thought_node(state) + + assert result["thought"] is None + assert result["step"] == 3 + assert len(result["messages"]) == 2 + ai_msg = result["messages"][0] + assert isinstance(ai_msg, AIMessage) + assert "The function is not reachable." in ai_msg.content + human_msg = result["messages"][1] + assert isinstance(human_msg, HumanMessage) + assert "Function Locator" in human_msg.content + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_blocked_finish_ai_message_falls_back_to_thought(self, mock_tracer): + """When final_answer is None, the AIMessage should use the thought text.""" + agent = _make_agent() + agent.config.context_window_token_limit = 50000 + agent.check_finish_allowed = MagicMock( + return_value=(False, "Call CCA first.") + ) + + finish_response = Thought( + thought="seems done", + mode="finish", + actions=None, + final_answer=None, + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=finish_response) + + state = { + "runtime_prompt": "prompt", + "messages": [HumanMessage(content="question")], + "observation": None, + "step": 0, + } + + result = await agent.thought_node(state) + + ai_msg = result["messages"][0] + assert isinstance(ai_msg, AIMessage) + assert "seems done" in ai_msg.content + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_blocked_finish_at_max_steps_routes_to_forced_finish(self, mock_tracer): + """If check_finish_allowed blocks at step 9 (incrementing to 10), + should_continue must route to forced_finish_node, not self-loop.""" + agent = _make_agent() + agent.config.context_window_token_limit = 50000 + agent.check_finish_allowed = MagicMock( + return_value=(False, "Call FL and CCA first.") + ) + + finish_response = Thought( + thought="done", + mode="finish", + actions=None, + final_answer="answer", + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=finish_response) + + state = { + "runtime_prompt": "prompt", + "messages": [HumanMessage(content="question")], + "observation": None, + "step": 9, + "max_steps": 10, + } + + result = await agent.thought_node(state) + assert result["step"] == 10 + assert result["thought"] is None + + route = await agent.should_continue(result) + assert route == "forced_finish_node" + + @pytest.mark.asyncio + @patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER") + async def test_bad_args_includes_ai_message(self, mock_tracer): + """Bad tool arguments error must include an AIMessage with the LLM's + original thought for proper chat alternation.""" + agent = _make_agent() + agent.config.context_window_token_limit = 50000 + + bad_actions = ToolCall( + tool="Function Locator", + reason="locate function", + ) + bad_response = Thought( + thought="Let me find the function", + mode="act", + actions=bad_actions, + final_answer=None, + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=bad_response) + + state = { + "runtime_prompt": "prompt", + "messages": [HumanMessage(content="question")], + "observation": None, + "step": 3, + } + + result = await agent.thought_node(state) + + assert len(result["messages"]) == 2 + ai_msg = result["messages"][0] + assert isinstance(ai_msg, AIMessage) + assert "Let me find the function" in ai_msg.content + error_msg = result["messages"][1] + assert isinstance(error_msg, HumanMessage) + assert "ERROR" in error_msg.content + + +class TestSelectPackage: + """Tests for _select_package image-match fast path and LLM fallback.""" + + def _make_workflow_state(self, image_name="registry.redhat.io/openshift4/ose-docker-builder", + git_repo="https://github.com/openshift/builder"): + si = MagicMock() + si.git_repo = git_repo + image = MagicMock() + image.name = image_name + image.source_info = [si] + ws = MagicMock() + ws.original_input.input.image = image + return ws + + @pytest.mark.asyncio + async def test_image_match_skips_llm(self): + """When a candidate name matches the image/repo, LLM is not called.""" + agent = _make_agent() + candidates = [ + {"name": "builder", "source": "rhsa"}, + {"name": "kernel", "source": "rhsa"}, + {"name": "glibc", "source": "rhsa"}, + ] + ws = self._make_workflow_state() + + with patch("vuln_analysis.utils.intel_utils.filter_context_to_package", + side_effect=lambda ctx, pkg, cands: ctx): + ctx, selected = await agent._select_package( + "go", candidates, ["CVE desc"], ws, + ) + + assert selected == "builder" + agent.package_filter_llm.ainvoke.assert_not_called() + + @pytest.mark.asyncio + async def test_no_match_calls_llm(self): + """When no candidate matches the image, LLM is called.""" + agent = _make_agent() + mock_selection = MagicMock() + mock_selection.selected_package = "xstream" + mock_selection.reason = "ecosystem match" + agent.package_filter_llm.ainvoke = AsyncMock(return_value=mock_selection) + + candidates = [ + {"name": "xstream", "source": "ghsa", "ecosystem": "Maven"}, + {"name": "kernel", "source": "rhsa"}, + ] + ws = self._make_workflow_state(image_name="registry.redhat.io/infinispan/server", + git_repo="https://github.com/infinispan/infinispan") + + with patch("vuln_analysis.utils.intel_utils.filter_context_to_package", + side_effect=lambda ctx, pkg, cands: ctx): + ctx, selected = await agent._select_package( + "java", candidates, ["CVE desc"], ws, + ) + + assert selected == "xstream" + agent.package_filter_llm.ainvoke.assert_called_once() + + @pytest.mark.asyncio + async def test_image_match_with_many_candidates(self): + """1000+ candidates with image match -> LLM skipped, no overflow.""" + agent = _make_agent() + candidates = [{"name": f"rhsa-product-{i}", "source": "rhsa"} for i in range(1200)] + candidates.append({"name": "builder", "source": "rhsa"}) + ws = self._make_workflow_state() + + with patch("vuln_analysis.utils.intel_utils.filter_context_to_package", + side_effect=lambda ctx, pkg, cands: ctx): + ctx, selected = await agent._select_package( + "go", candidates, ["CVE desc"], ws, + ) + + assert selected == "builder" + agent.package_filter_llm.ainvoke.assert_not_called() + + @pytest.mark.asyncio + async def test_single_candidate_no_llm(self): + """Single candidate is used directly without LLM.""" + agent = _make_agent() + candidates = [{"name": "jinja2", "source": "ghsa"}] + ws = self._make_workflow_state() + + with patch("vuln_analysis.utils.intel_utils.filter_context_to_package", + side_effect=lambda ctx, pkg, cands: ctx): + ctx, selected = await agent._select_package( + "python", candidates, ["CVE desc"], ws, + ) + + assert selected == "jinja2" + agent.package_filter_llm.ainvoke.assert_not_called() + + +class TestForcedFinishNode: + """Tests for forced_finish_node: includes conversation history with selective pruning.""" + + @pytest.mark.asyncio + async def test_includes_history_and_observation(self): + """forced_finish_node should include conversation history AND observation memory.""" + agent = _make_agent() + agent.config.context_window_token_limit = 999999 + mock_response = Thought( + thought="summarizing", mode="finish", actions=None, + final_answer="Based on evidence, the function is not reachable.", + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=mock_response) + + obs = Observation( + results=["CCA returned (False, [])"], + memory=["Package validated: commons-beanutils:1.9.4", + "FL found PropertyUtilsBean.getProperty", + "CCA: function not reachable from app code"], + ) + state = { + "step": 10, "max_steps": 10, + "runtime_prompt": "You are a security analyst.", + "messages": [ + AIMessage(content="I will check the function"), + HumanMessage(content="CCA result: (False, [])"), + ], + "observation": obs, + "thought": None, + } + + with patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER"): + result = await agent.forced_finish_node(state) + + call_args = agent.thought_llm.ainvoke.call_args[0][0] + contents = [m.content for m in call_args if hasattr(m, "content")] + assert "I will check the function" in contents, "Conversation history should be in the prompt" + assert "CCA result: (False, [])" in contents, "Tool output should be in the prompt" + knowledge_msgs = [m for m in call_args if hasattr(m, "content") and "KNOWLEDGE" in m.content] + assert len(knowledge_msgs) == 1, "Observation memory should be in the prompt" + assert "LATEST FINDINGS" in knowledge_msgs[0].content + assert "CCA returned (False, [])" in knowledge_msgs[0].content + assert result["output"] == "Based on evidence, the function is not reachable." + + @pytest.mark.asyncio + async def test_prunes_history_when_over_token_limit(self): + """forced_finish_node should prune oldest messages when over the token limit, + while preserving the system prompt, observation, and finish prompt.""" + agent = _make_agent() + agent.config.context_window_token_limit = 100 + + mock_response = Thought( + thought="done", mode="finish", actions=None, + final_answer="Not exploitable.", + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=mock_response) + + obs = Observation( + results=["CCA: not reachable"], + memory=["FL found the function"], + ) + long = _long_content(200) + state = { + "step": 10, "max_steps": 10, + "runtime_prompt": "system prompt", + "messages": [ + HumanMessage(content=long), + AIMessage(content=long), + HumanMessage(content=long), + AIMessage(content="recent reasoning"), + ], + "observation": obs, + "input": "Is the function reachable?", + "thought": None, + } + + with patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER"): + result = await agent.forced_finish_node(state) + + call_args = agent.thought_llm.ainvoke.call_args[0][0] + contents = [m.content for m in call_args if hasattr(m, "content")] + assert "system prompt" in contents, "System prompt must survive pruning" + assert any("KNOWLEDGE" in c for c in contents), "Observation must survive pruning" + assert any("FORCED" in c or "Is the function reachable?" in c for c in contents), \ + "Finish prompt must survive pruning" + assert long not in contents, "Old long messages should be pruned" + assert result["output"] == "Not exploitable." + + @pytest.mark.asyncio + async def test_works_without_observation(self): + """forced_finish_node should work even when no observations exist.""" + agent = _make_agent() + agent.config.context_window_token_limit = 999999 + mock_response = Thought( + thought="no evidence", mode="finish", actions=None, + final_answer="No evidence found.", + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=mock_response) + + state = { + "step": 10, "max_steps": 10, + "runtime_prompt": "You are a security analyst.", + "messages": [HumanMessage(content="some message")], + "observation": None, + "thought": None, + } + + with patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER"): + result = await agent.forced_finish_node(state) + + call_args = agent.thought_llm.ainvoke.call_args[0][0] + contents = [m.content for m in call_args if hasattr(m, "content")] + assert "some message" in contents, "History should be included when no pruning needed" + assert result["output"] == "No evidence found." + + @pytest.mark.asyncio + async def test_fallback_on_non_finish_response(self): + """forced_finish_node returns default message when LLM doesn't finish.""" + agent = _make_agent() + agent.config.context_window_token_limit = 999999 + mock_response = Thought( + thought="I want to call another tool", mode="act", + actions=ToolCall(tool="Function Locator", package_name="pkg", function_name="fn", reason="test"), + final_answer=None, + ) + agent.thought_llm.ainvoke = AsyncMock(return_value=mock_response) + + state = { + "step": 10, "max_steps": 10, + "runtime_prompt": "You are a security analyst.", + "messages": [], + "observation": None, + "thought": None, + } + + with patch("vuln_analysis.functions.base_graph_agent.AGENT_TRACER"): + result = await agent.forced_finish_node(state) + + assert "Failed to generate a final answer" in result["output"] diff --git a/tests/test_base_tool_descriptions.py b/tests/test_base_tool_descriptions.py new file mode 100644 index 000000000..f457f59ff --- /dev/null +++ b/tests/test_base_tool_descriptions.py @@ -0,0 +1,167 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the base build_tool_descriptions() function. + +Verifies that the consolidated base function provides simple tool descriptions +that can be formatted by specialized functions for different contexts. +""" + +import sys +from pathlib import Path + +# Add src to path +src_path = Path(__file__).parent.parent / "src" +sys.path.insert(0, str(src_path)) + +from vuln_analysis.utils.prompting import build_tool_descriptions +from vuln_analysis.tools.tool_names import ToolNames + + +def test_base_returns_list(): + """Test that base function returns a list, not a string.""" + tool_names = [ToolNames.CODE_SEMANTIC_SEARCH] + + result = build_tool_descriptions(tool_names) + + assert isinstance(result, list) + print("✓ Base function returns list") + + +def test_base_descriptions_format(): + """Test that base descriptions have consistent format.""" + tool_names = [ + ToolNames.CODE_SEMANTIC_SEARCH, + ToolNames.CODE_KEYWORD_SEARCH + ] + + result = build_tool_descriptions(tool_names) + + # Each description should have format: "Tool Name: Description" + for desc in result: + assert ":" in desc + parts = desc.split(":", 1) + assert len(parts) == 2 + assert len(parts[0].strip()) > 0 # Tool name + assert len(parts[1].strip()) > 0 # Description + + print("✓ Base descriptions have consistent format") + + +def test_base_all_tools(): + """Test that base function includes all available tools.""" + tool_names = [ + ToolNames.CODE_SEMANTIC_SEARCH, + ToolNames.DOCS_SEMANTIC_SEARCH, + ToolNames.CODE_KEYWORD_SEARCH, + ToolNames.CALL_CHAIN_ANALYZER, + ToolNames.FUNCTION_CALLER_FINDER, + ToolNames.CVE_WEB_SEARCH, + ToolNames.CONTAINER_ANALYSIS_DATA, + ToolNames.FUNCTION_LIBRARY_VERSION_FINDER + ] + + result = build_tool_descriptions(tool_names) + + # Should have 7 descriptions (CONTAINER_ANALYSIS_DATA has no description in the base function) + assert len(result) == 7 + + # Verify all tools are present + all_text = " ".join(result) + assert "Code Semantic Search" in all_text + assert "Docs Semantic Search" in all_text + assert "Code Keyword Search" in all_text + assert "Call Chain Analyzer" in all_text + assert "Function Caller Finder" in all_text + assert "CVE Web Search" in all_text + assert "Container Analysis Data" in all_text + assert "Function Library Version Finder" in all_text + + print("✓ Base function includes all tools") + + +def test_base_empty_list(): + """Test that base function returns empty list when no tools.""" + tool_names = [] + + result = build_tool_descriptions(tool_names) + + assert result == [] + assert isinstance(result, list) + + print("✓ Base function returns empty list for no tools") + + +def test_checklist_formats_descriptions(): + """Test that checklist calling code formats descriptions correctly.""" + tool_names = [ + ToolNames.CODE_SEMANTIC_SEARCH, + ToolNames.DOCS_SEMANTIC_SEARCH + ] + + # Simulate what checklist_prompt_generator.py does + tool_descs = build_tool_descriptions(tool_names) + + if tool_descs: + formatted_descs = ["- " + desc for desc in tool_descs] + tool_descriptions = "The following tools can be used to answer checklist questions:\n " + "\n ".join(formatted_descs) + else: + tool_descriptions = "Analysis tools will be used to investigate these questions." + + # Verify checklist formatting + assert "The following tools can be used to answer checklist questions:" in tool_descriptions + assert "Code Semantic Search" in tool_descriptions + assert "Docs Semantic Search" in tool_descriptions + assert " - " in tool_descriptions # Indented bullet points + + print("✓ Checklist formats descriptions correctly") + + +def test_mod_few_shot_structure(): + """Test that MOD_FEW_SHOT has required structure and placeholders.""" + from vuln_analysis.utils.prompting import MOD_FEW_SHOT + + # Check for required section markers + assert "" in MOD_FEW_SHOT + assert "" in MOD_FEW_SHOT + assert "" in MOD_FEW_SHOT + assert "" in MOD_FEW_SHOT + assert "" in MOD_FEW_SHOT + assert "" in MOD_FEW_SHOT + assert "" in MOD_FEW_SHOT + + # Check for placeholders + assert "{tool_descriptions}" in MOD_FEW_SHOT + assert "{examples}" in MOD_FEW_SHOT + + # Check for key instructions + assert "3-5 checklist items" in MOD_FEW_SHOT + assert "FIRST" in MOD_FEW_SHOT or "first" in MOD_FEW_SHOT + assert "vulnerable function" in MOD_FEW_SHOT + + print("✓ MOD_FEW_SHOT structure validated") + + +if __name__ == "__main__": + print("Running Base Tool Descriptions tests...\n") + + test_base_returns_list() + test_base_descriptions_format() + test_base_all_tools() + test_base_empty_list() + test_checklist_formats_descriptions() + test_mod_few_shot_structure() + + print("\n✅ All base tool descriptions tests passed!") diff --git a/tests/test_brew_downloader.py b/tests/test_brew_downloader.py new file mode 100644 index 000000000..819e810c0 --- /dev/null +++ b/tests/test_brew_downloader.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for BrewDownloader profile resolution and artifact orchestration.""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from vuln_analysis.tools.brew_downloader import ( + BrewDownloader, + BrewProfileNotImplementedError, + BrewProfileType, + resolve_brew_profile, +) + + +class TestResolveBrewProfile: + def test_resolves_internal_from_config(self, monkeypatch): + monkeypatch.delenv("RPM_USER_TYPE", raising=False) + assert resolve_brew_profile("internal") == BrewProfileType.INTERNAL + + def test_resolves_external_from_config(self, monkeypatch): + monkeypatch.delenv("RPM_USER_TYPE", raising=False) + assert resolve_brew_profile("external") == BrewProfileType.EXTERNAL + + def test_env_overrides_config(self, monkeypatch): + monkeypatch.setenv("RPM_USER_TYPE", "external") + assert resolve_brew_profile("internal") == BrewProfileType.EXTERNAL + + def test_yaml_env_default_syntax_without_expansion(self, monkeypatch): + monkeypatch.delenv("RPM_USER_TYPE", raising=False) + assert resolve_brew_profile("${RPM_USER_TYPE:-internal}") == BrewProfileType.INTERNAL + + def test_yaml_env_default_syntax_uses_env(self, monkeypatch): + monkeypatch.setenv("RPM_USER_TYPE", "external") + assert resolve_brew_profile("${RPM_USER_TYPE:-internal}") == BrewProfileType.EXTERNAL + + def test_unknown_profile_raises(self, monkeypatch): + monkeypatch.delenv("RPM_USER_TYPE", raising=False) + with pytest.raises(BrewProfileNotImplementedError): + resolve_brew_profile("unknown") + + +class TestBrewProfileFlags: + def test_external_profile_disables_build_log_fetch(self, tmp_path): + downloader = BrewDownloader( + BrewProfileType.EXTERNAL, + str(tmp_path / "rpms"), + str(tmp_path / "checker"), + ) + assert downloader.auto_fetch_build_log is False + assert downloader.download_binary_rpm_enabled is False + + def test_internal_profile_enables_build_log_fetch(self, tmp_path): + downloader = BrewDownloader( + BrewProfileType.INTERNAL, + str(tmp_path / "rpms"), + str(tmp_path / "checker"), + ) + assert downloader.auto_fetch_build_log is True + + +class TestDownloadTargetArtifacts: + def test_skips_build_log_when_auto_fetch_disabled(self, tmp_path): + rpm_cache = tmp_path / "rpms" + checker_dir = tmp_path / "checker" + downloader = BrewDownloader( + BrewProfileType.EXTERNAL, + str(rpm_cache), + str(checker_dir), + ) + build = {"id": 1, "nvr": "curl-8.11.1-8.fc42"} + srpm_file = rpm_cache / "curl-8.11.1-8.fc42.src.rpm" + srpm_file.write_bytes(b"fake-srpm") + + downloader._session = MagicMock() + downloader._pathinfo = MagicMock() + + with ( + patch.object(downloader, "search_build", return_value=build), + patch.object(downloader, "_get_srpm_url", return_value="https://example/srpm"), + patch.object(downloader, "download_srpm", return_value=srpm_file), + patch.object(downloader, "download_build_log") as mock_build_log, + patch( + "vuln_analysis.tools.brew_downloader.SourceRPMDownloader.extract_src_rpm", + ), + ): + artifacts = downloader.download_target_artifacts( + "curl", "8.11.1", "8.fc42", "x86_64", + ) + + mock_build_log.assert_not_called() + assert artifacts.build_log_path is None + assert artifacts.srpm_path == checker_dir / "source" + + +class TestSourceAcquisitionCacheCondition: + """Mirror the cache-hit predicate used in cve_source_acquisition.""" + + @staticmethod + def _is_full_cache_hit(source_exists: bool, log_exists: bool, auto_fetch_build_log: bool) -> bool: + return source_exists and (log_exists or not auto_fetch_build_log) + + def test_source_only_hit_when_auto_fetch_false(self): + assert self._is_full_cache_hit( + source_exists=True, + log_exists=False, + auto_fetch_build_log=False, + ) + + def test_requires_log_when_auto_fetch_true(self): + assert not self._is_full_cache_hit( + source_exists=True, + log_exists=False, + auto_fetch_build_log=True, + ) + assert self._is_full_cache_hit( + source_exists=True, + log_exists=True, + auto_fetch_build_log=True, + ) diff --git a/tests/test_build_code_understanding_tools.py b/tests/test_build_code_understanding_tools.py new file mode 100644 index 000000000..1eb880385 --- /dev/null +++ b/tests/test_build_code_understanding_tools.py @@ -0,0 +1,276 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for CodeUnderstandingAgent: tool selection, availability, and comprehension hooks.""" + +from unittest.mock import MagicMock, patch + +from agent_test_helpers import MockTool, make_builder, make_config, make_state +from vuln_analysis.functions.code_understanding_agent import CodeUnderstandingAgent +from vuln_analysis.functions.code_understanding_internals import CodeUnderstandingRulesTracker +from vuln_analysis.tools.tool_names import ToolNames + + +def _get_tools(builder=None, config=None, state=None): + return CodeUnderstandingAgent.get_tools( + builder or make_builder(), + config or make_config(), + state or make_state(), + ) + + +class TestGetTools: + """Test CodeUnderstandingAgent.get_tools selection and availability logic.""" + + def test_filters_to_exactly_4_cu_tools(self): + result = _get_tools() + assert len(result) == 4 + + def test_output_tool_names(self): + result = _get_tools() + result_names = {t.name for t in result} + expected_names = { + ToolNames.DOCS_SEMANTIC_SEARCH, + ToolNames.CODE_KEYWORD_SEARCH, + ToolNames.CONFIGURATION_SCANNER, + ToolNames.IMPORT_USAGE_ANALYZER, + } + assert result_names == expected_names + + def test_excludes_reachability_tools(self): + tools = [ + MockTool(ToolNames.FUNCTION_LOCATOR), + MockTool(ToolNames.CALL_CHAIN_ANALYZER), + MockTool(ToolNames.FUNCTION_CALLER_FINDER), + MockTool(ToolNames.DOCS_SEMANTIC_SEARCH), + MockTool(ToolNames.CODE_KEYWORD_SEARCH), + ] + result = _get_tools(builder=make_builder(tools)) + result_names = {t.name for t in result} + assert ToolNames.FUNCTION_LOCATOR not in result_names + assert ToolNames.CALL_CHAIN_ANALYZER not in result_names + assert ToolNames.FUNCTION_CALLER_FINDER not in result_names + assert len(result) == 2 + + def test_excludes_web_and_container_tools(self): + tools = [ + MockTool(ToolNames.CVE_WEB_SEARCH), + MockTool(ToolNames.CONTAINER_ANALYSIS_DATA), + MockTool(ToolNames.DOCS_SEMANTIC_SEARCH), + MockTool(ToolNames.CODE_KEYWORD_SEARCH), + ] + result = _get_tools(builder=make_builder(tools)) + result_names = {t.name for t in result} + assert ToolNames.CVE_WEB_SEARCH not in result_names + assert ToolNames.CONTAINER_ANALYSIS_DATA not in result_names + assert len(result) == 2 + + def test_excludes_version_finder(self): + tools = [ + MockTool(ToolNames.FUNCTION_LIBRARY_VERSION_FINDER), + MockTool(ToolNames.DOCS_SEMANTIC_SEARCH), + MockTool(ToolNames.CODE_KEYWORD_SEARCH), + ] + result = _get_tools(builder=make_builder(tools)) + result_names = {t.name for t in result} + assert ToolNames.FUNCTION_LIBRARY_VERSION_FINDER not in result_names + assert len(result) == 2 + + def test_empty_builder_returns_empty(self): + result = _get_tools(builder=make_builder(tools=[])) + assert result == [] + + def test_no_matching_tools_returns_empty(self): + tools = [ + MockTool(ToolNames.FUNCTION_LOCATOR), + MockTool(ToolNames.CALL_CHAIN_ANALYZER), + MockTool(ToolNames.FUNCTION_CALLER_FINDER), + MockTool(ToolNames.CVE_WEB_SEARCH), + ] + result = _get_tools(builder=make_builder(tools)) + assert result == [] + + def test_preserves_tool_object_identity(self): + docs_tool = MockTool(ToolNames.DOCS_SEMANTIC_SEARCH) + keyword_tool = MockTool(ToolNames.CODE_KEYWORD_SEARCH) + locator_tool = MockTool(ToolNames.FUNCTION_LOCATOR) + builder = make_builder(tools=[docs_tool, keyword_tool, locator_tool]) + result = _get_tools(builder=builder) + assert len(result) == 2 + assert docs_tool in result + assert keyword_tool in result + assert locator_tool not in result + + def test_partial_overlap(self): + tools = [ + MockTool(ToolNames.DOCS_SEMANTIC_SEARCH), + MockTool(ToolNames.CODE_KEYWORD_SEARCH), + MockTool(ToolNames.FUNCTION_LOCATOR), + MockTool(ToolNames.CVE_WEB_SEARCH), + ] + result = _get_tools(builder=make_builder(tools)) + result_names = {t.name for t in result} + expected_names = { + ToolNames.DOCS_SEMANTIC_SEARCH, + ToolNames.CODE_KEYWORD_SEARCH, + } + assert len(result) == 2 + assert result_names == expected_names + + +class TestGetToolsAvailability: + """get_tools filters out tools whose infrastructure prerequisites are not met.""" + + def test_filters_docs_semantic_search_when_no_vdb(self): + state = make_state(doc_vdb_path=None) + result = _get_tools(state=state) + assert ToolNames.DOCS_SEMANTIC_SEARCH not in {t.name for t in result} + + def test_filters_code_keyword_search_when_no_index(self): + state = make_state(code_index_path=None) + result = _get_tools(state=state) + assert ToolNames.CODE_KEYWORD_SEARCH not in {t.name for t in result} + + def test_cu_only_tools_always_kept(self): + state = make_state(code_vdb_path=None, doc_vdb_path=None, code_index_path=None) + result = _get_tools(state=state) + result_names = {t.name for t in result} + assert ToolNames.CONFIGURATION_SCANNER in result_names + + def test_filters_import_usage_analyzer_when_no_index(self): + state = make_state(code_index_path=None) + result = _get_tools(state=state) + assert ToolNames.IMPORT_USAGE_ANALYZER not in {t.name for t in result} + + def test_import_usage_analyzer_available_with_index(self): + state = make_state(code_index_path="/some/path") + result = _get_tools(state=state) + assert ToolNames.IMPORT_USAGE_ANALYZER in {t.name for t in result} + + +class TestCodeUnderstandingAgentMeta: + """Test create_rules_tracker and agent_type for CodeUnderstandingAgent.""" + + def test_create_rules_tracker_returns_cu_tracker(self): + tracker = CodeUnderstandingAgent.create_rules_tracker() + assert isinstance(tracker, CodeUnderstandingRulesTracker) + + def test_create_rules_tracker_returns_fresh_instance(self): + t1 = CodeUnderstandingAgent.create_rules_tracker() + t2 = CodeUnderstandingAgent.create_rules_tracker() + assert t1 is not t2 + + def test_agent_type_is_cu(self): + mock_llm = MagicMock() + mock_llm.with_structured_output = MagicMock(return_value=MagicMock()) + config = MagicMock() + config.max_iterations = 10 + agent = CodeUnderstandingAgent(tools=[], llm=mock_llm, config=config) + assert agent.agent_type == "cu" + + +def _make_cu_agent(): + mock_llm = MagicMock() + mock_llm.with_structured_output = MagicMock(return_value=MagicMock()) + config = MagicMock() + config.max_iterations = 10 + return CodeUnderstandingAgent(tools=[], llm=mock_llm, config=config) + + +def _mock_ctx_state(*vuln_ids): + """Return a mock workflow state with cve_intel entries for the given vuln IDs.""" + intel_list = [] + for vid in vuln_ids: + intel = MagicMock() + intel.vuln_id = vid + intel_list.append(intel) + ws = MagicMock() + ws.cve_intel = intel_list + return ws + + +class TestCUComprehensionHooks: + """Test CU agent comprehension context reduction and CVE sanitization.""" + + @patch("vuln_analysis.functions.code_understanding_agent.ctx_state") + def test_build_comprehension_context_contains_vuln_id_and_package(self, mock_ctx): + mock_ctx.get.return_value = _mock_ctx_state("CVE-2021-43859") + agent = _make_cu_agent() + state = {"app_package": "com.thoughtworks.xstream:xstream"} + result = agent.build_comprehension_context(state) + assert "CVE-2021-43859" in result + assert "com.thoughtworks.xstream:xstream" in result + + @patch("vuln_analysis.functions.code_understanding_agent.ctx_state") + def test_build_comprehension_context_includes_grounding_instruction(self, mock_ctx): + mock_ctx.get.return_value = _mock_ctx_state("CVE-2021-43859") + agent = _make_cu_agent() + result = agent.build_comprehension_context({"app_package": "pkg"}) + assert "Only extract facts explicitly stated in the tool output" in result + assert "Do not add CVE IDs" in result + + @patch("vuln_analysis.functions.code_understanding_agent.ctx_state") + def test_build_comprehension_context_excludes_critical_context(self, mock_ctx): + mock_ctx.get.return_value = _mock_ctx_state("CVE-2021-43859") + agent = _make_cu_agent() + state = { + "app_package": "xstream", + "critical_context": ["GHSA description: XStream can cause DoS", "NVD: high severity"], + } + result = agent.build_comprehension_context(state) + assert "GHSA description" not in result + assert "NVD" not in result + + @patch("vuln_analysis.functions.code_understanding_agent.ctx_state") + def test_build_comprehension_context_unknown_fallbacks(self, mock_ctx): + ws = MagicMock() + ws.cve_intel = [] + mock_ctx.get.return_value = ws + agent = _make_cu_agent() + result = agent.build_comprehension_context({}) + assert "unknown" in result + + @patch("vuln_analysis.functions.base_graph_agent.ctx_state") + def test_sanitize_findings_replaces_wrong_cve(self, mock_ctx): + mock_ctx.get.return_value = _mock_ctx_state("CVE-2021-43859") + agent = _make_cu_agent() + findings = ["XStream 1.4.18 is vulnerable to CVE-2020-26217"] + result = agent.sanitize_findings(findings, {}) + assert result == ["XStream 1.4.18 is vulnerable to the investigated vulnerability"] + + @patch("vuln_analysis.functions.base_graph_agent.ctx_state") + def test_sanitize_findings_keeps_correct_cve(self, mock_ctx): + mock_ctx.get.return_value = _mock_ctx_state("CVE-2021-43859") + agent = _make_cu_agent() + findings = ["Affects CVE-2021-43859"] + result = agent.sanitize_findings(findings, {}) + assert result == ["Affects CVE-2021-43859"] + + @patch("vuln_analysis.functions.base_graph_agent.ctx_state") + def test_sanitize_findings_replaces_multiple_wrong_cves(self, mock_ctx): + mock_ctx.get.return_value = _mock_ctx_state("CVE-2021-43859") + agent = _make_cu_agent() + findings = ["CVE-2020-26217 and CVE-2019-10086 affect this, also CVE-2021-43859"] + result = agent.sanitize_findings(findings, {}) + assert "CVE-2020-26217" not in result[0] + assert "CVE-2019-10086" not in result[0] + assert "CVE-2021-43859" in result[0] + assert result[0].count("the investigated vulnerability") == 2 + + @patch("vuln_analysis.functions.base_graph_agent.ctx_state") + def test_sanitize_findings_no_cve_ids_unchanged(self, mock_ctx): + mock_ctx.get.return_value = _mock_ctx_state("CVE-2021-43859") + agent = _make_cu_agent() + findings = ["XStream 1.4.18 found in dependencies", "Package is present"] + result = agent.sanitize_findings(findings, {}) + assert result == findings + + @patch("vuln_analysis.functions.base_graph_agent.ctx_state") + def test_sanitize_findings_multi_cve_intel(self, mock_ctx): + mock_ctx.get.return_value = _mock_ctx_state("CVE-2021-43859", "CVE-2021-39144") + agent = _make_cu_agent() + findings = ["CVE-2021-43859 and CVE-2021-39144 and CVE-2020-26217"] + result = agent.sanitize_findings(findings, {}) + assert "CVE-2021-43859" in result[0] + assert "CVE-2021-39144" in result[0] + assert "CVE-2020-26217" not in result[0] diff --git a/tests/test_code_understanding_internals.py b/tests/test_code_understanding_internals.py new file mode 100644 index 000000000..ad52a8c5f --- /dev/null +++ b/tests/test_code_understanding_internals.py @@ -0,0 +1,176 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from vuln_analysis.functions.code_understanding_internals import ( + CodeUnderstandingRulesTracker, + CU_AGENT_SYS_PROMPT, + CU_AGENT_THOUGHT_INSTRUCTIONS, +) + + +class TestCodeUnderstandingRulesTracker: + def test_iua_allowed_without_survey(self): + """IUA can be called at any time — no gating.""" + tracker = CodeUnderstandingRulesTracker() + tracker.set_allowed_tools(["Import Usage Analyzer"]) + + violated, msg = tracker.check_thought_behavior( + "Import Usage Analyzer", + "com.example.package", + ["imports found"] + ) + + assert violated is False + assert msg == "" + + def test_allowed_tool_passes(self): + tracker = CodeUnderstandingRulesTracker() + tracker.set_allowed_tools(["Code Keyword Search"]) + + violated, msg = tracker.check_thought_behavior( + "Code Keyword Search", + "some query", + ["results"] + ) + + assert violated is False + + def test_docs_semantic_search_passes(self): + tracker = CodeUnderstandingRulesTracker() + tracker.set_allowed_tools(["Docs Semantic Search"]) + + violated, msg = tracker.check_thought_behavior( + "Docs Semantic Search", + "query", + ["docs"] + ) + + assert violated is False + + def test_check_rule7_fires(self): + tracker = CodeUnderstandingRulesTracker() + tracker.set_allowed_tools(["Code Keyword Search"]) + + tracker.check_thought_behavior( + "Code Keyword Search", + "com.example.Class", + [] + ) + + violated, msg = tracker.check_thought_behavior( + "Code Keyword Search", + "com.example.Another", + [] + ) + + assert violated is True + assert "Rule 7" in msg + assert "dots" in msg + + def test_check_allowed_tools_rejects_unknown(self): + tracker = CodeUnderstandingRulesTracker() + tracker.set_allowed_tools(["Code Keyword Search"]) + + violated, msg = tracker.check_thought_behavior( + "Unknown Tool", + "query", + ["results"] + ) + + assert violated is True + assert "AVAILABLE_TOOLS" in msg + assert "Code Keyword Search" in msg + + def test_check_passes_and_adds_action(self): + tracker = CodeUnderstandingRulesTracker() + tracker.set_allowed_tools(["Code Keyword Search"]) + + assert "Code Keyword Search" not in tracker.action_history + + violated, msg = tracker.check_thought_behavior( + "Code Keyword Search", + "import xstream", + ["result1", "result2"] + ) + + assert violated is False + assert msg == "" + assert "Code Keyword Search" in tracker.action_history + assert len(tracker.action_history["Code Keyword Search"]) == 1 + assert tracker.action_history["Code Keyword Search"][0]["input"] == "import xstream" + + def test_duplicate_config_scanner_blocked(self): + tracker = CodeUnderstandingRulesTracker() + tracker.set_allowed_tools(["Configuration Scanner"]) + tracker.check_thought_behavior( + "Configuration Scanner", "deserialization beanutils", ["config found"] + ) + violated, msg = tracker.check_thought_behavior( + "Configuration Scanner", "deserialization beanutils", ["config found"] + ) + assert violated is True + assert "already called" in msg + + def test_config_scanner_different_query_allowed(self): + tracker = CodeUnderstandingRulesTracker() + tracker.set_allowed_tools(["Configuration Scanner"]) + tracker.check_thought_behavior( + "Configuration Scanner", "deserialization beanutils", ["config found"] + ) + violated, msg = tracker.check_thought_behavior( + "Configuration Scanner", "security allowlist", ["other config"] + ) + assert violated is False + assert msg == "" + + def test_check_failing_does_not_add_action(self): + tracker = CodeUnderstandingRulesTracker() + tracker.set_allowed_tools(["Configuration Scanner"]) + + violated, msg = tracker.check_thought_behavior( + "Unknown Tool", + "query", + ["results"] + ) + + assert violated is True + assert "Unknown Tool" not in tracker.action_history + + +class TestCUConstants: + def test_sys_prompt_is_nonempty_string(self): + assert isinstance(CU_AGENT_SYS_PROMPT, str) + assert len(CU_AGENT_SYS_PROMPT) > 0 + assert len(CU_AGENT_SYS_PROMPT.strip()) > 0 + + def test_sys_prompt_mentions_code_understanding(self): + assert "code understanding" in CU_AGENT_SYS_PROMPT.lower() + + def test_sys_prompt_does_not_mention_call_chain_analyzer(self): + assert "Call Chain Analyzer" not in CU_AGENT_SYS_PROMPT + assert "call chain analyzer" not in CU_AGENT_SYS_PROMPT.lower() + + def test_thought_instructions_have_rules_tags(self): + assert "" in CU_AGENT_THOUGHT_INSTRUCTIONS + assert "" in CU_AGENT_THOUGHT_INSTRUCTIONS + + def test_thought_instructions_have_three_examples(self): + assert "" in CU_AGENT_THOUGHT_INSTRUCTIONS + assert "" in CU_AGENT_THOUGHT_INSTRUCTIONS + assert "" in CU_AGENT_THOUGHT_INSTRUCTIONS + + def test_thought_instructions_rules_numbered_1_to_9(self): + for i in range(1, 10): + assert f"{i}." in CU_AGENT_THOUGHT_INSTRUCTIONS diff --git a/tests/test_code_understanding_prompt_factory.py b/tests/test_code_understanding_prompt_factory.py new file mode 100644 index 000000000..e05484b3b --- /dev/null +++ b/tests/test_code_understanding_prompt_factory.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from vuln_analysis.tools.tool_names import ToolNames +from vuln_analysis.utils.code_understanding_prompt_factory import ( + CU_TOOL_GENERAL_DESCRIPTIONS, + CU_TOOL_SELECTION_STRATEGY, +) + + +class TestCUToolGeneralDescriptions: + def test_has_4_entries(self): + assert len(CU_TOOL_GENERAL_DESCRIPTIONS) == 4 + + def test_keys_match_cu_tool_names(self): + expected_keys = { + ToolNames.DOCS_SEMANTIC_SEARCH, + ToolNames.CODE_KEYWORD_SEARCH, + ToolNames.CONFIGURATION_SCANNER, + ToolNames.IMPORT_USAGE_ANALYZER, + } + assert set(CU_TOOL_GENERAL_DESCRIPTIONS.keys()) == expected_keys + + def test_values_non_empty(self): + for key, value in CU_TOOL_GENERAL_DESCRIPTIONS.items(): + assert isinstance(value, str), f"Value for '{key}' is not a string" + assert len(value.strip()) > 0, f"Value for '{key}' is empty" + + +class TestCUToolSelectionStrategy: + def test_has_5_ecosystems(self): + expected_ecosystems = {"python", "go", "java", "javascript", "c"} + assert set(CU_TOOL_SELECTION_STRATEGY.keys()) == expected_ecosystems + + def test_strategies_non_empty(self): + for ecosystem, strategy in CU_TOOL_SELECTION_STRATEGY.items(): + assert isinstance(strategy, str), f"Strategy for '{ecosystem}' is not a string" + assert len(strategy.strip()) > 0, f"Strategy for '{ecosystem}' is empty" + + def test_each_mentions_at_least_3_tool_names(self): + cu_tool_names = [ + ToolNames.DOCS_SEMANTIC_SEARCH, + ToolNames.CODE_KEYWORD_SEARCH, + ToolNames.CONFIGURATION_SCANNER, + ToolNames.IMPORT_USAGE_ANALYZER, + ] + + for ecosystem, strategy in CU_TOOL_SELECTION_STRATEGY.items(): + mentioned_tools = sum(1 for tool_name in cu_tool_names if tool_name in strategy) + assert mentioned_tools >= 3, f"Strategy for '{ecosystem}' mentions only {mentioned_tools} tool(s)" diff --git a/tests/test_configuration_scanner.py b/tests/test_configuration_scanner.py new file mode 100644 index 000000000..ca9a462c8 --- /dev/null +++ b/tests/test_configuration_scanner.py @@ -0,0 +1,420 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from vuln_analysis.tools.configuration_scanner import ( + _is_config_file, + _is_in_config_dir, + _collect_config_files, + search_config_content, +) + + +class TestIsConfigFile: + @pytest.mark.parametrize( + "file_path,expected", + [ + # Named config files — matched anywhere + ("config.yaml", True), + ("config.yml", True), + ("config.xml", True), + ("application.properties", True), + ("application.yml", True), + ("application.yaml", True), + ("settings.toml", True), + ("settings.yaml", True), + ("settings.yml", True), + ("beans.xml", True), + ("web.xml", True), + ("Dockerfile", True), + ("Dockerfile.prod", True), + ("Dockerfile.dev", True), + ("docker-compose.yml", True), + ("docker-compose.prod.yml", True), + # Config-specific extensions — matched anywhere + ("app.env", True), + (".env", True), + ("nginx.conf", True), + ("config.ini", True), + ("app.properties", True), + # Removed: build/dependency files (handled by other tools) + ("pom.xml", False), + ("build.gradle", False), + ("build.gradle.kts", False), + ("go.mod", False), + ("go.sum", False), + ("package.json", False), + ("requirements.txt", False), + ("setup.py", False), + ("setup.cfg", False), + ("pyproject.toml", False), + # Removed: catch-all extensions (only matched inside config dirs) + ("app.yml", False), + ("app.cfg", False), + ("data.yaml", False), + ("docker-compose-dev.yaml", False), + # Non-config files + ("main.py", False), + ("utils.go", False), + ("README.md", False), + ("app.js", False), + ("test.txt", False), + ("data.json", False), + ], + ) + def test_config_file_patterns(self, file_path, expected): + assert _is_config_file(file_path) == expected + + @pytest.mark.parametrize( + "file_path,expected", + [ + ("DOCKERFILE", True), + ("CONFIG.YAML", True), + ("APPLICATION.PROPERTIES", True), + ("DOCKER-COMPOSE.YML", True), + ("BEANS.XML", True), + ("SETTINGS.TOML", True), + ], + ) + def test_case_insensitive(self, file_path, expected): + assert _is_config_file(file_path) == expected + + def test_file_in_subdirectory(self): + assert _is_config_file("path/to/config.yaml") is True + assert _is_config_file("deep/nested/path/application.properties") is True + + +class TestIsInConfigDir: + @pytest.mark.parametrize( + "file_path,expected", + [ + ("config/app.txt", True), + ("conf/settings.txt", True), + ("conf.d/default.conf", True), + ("etc/nginx/nginx.conf", True), + ("src/main/resources/app.yml", True), + ("app/config/database.yml", True), + ("project/conf/server.xml", True), + ("src/main/java/App.java", False), + ("lib/utils.py", False), + ("tests/test_app.py", False), + ("data/sample.csv", False), + ], + ) + def test_config_dir_patterns(self, file_path, expected): + assert _is_in_config_dir(file_path) == expected + + @pytest.mark.parametrize( + "file_path,expected", + [ + ("Config/app.txt", True), + ("RESOURCES/app.yml", True), + ("ETC/nginx.conf", True), + ("CONF.D/default.conf", True), + ], + ) + def test_case_insensitive_dir(self, file_path, expected): + assert _is_in_config_dir(file_path) == expected + + +class TestCollectConfigFiles: + def test_finds_config_files(self, tmp_path): + (tmp_path / "config.yaml").write_text("key: value") + (tmp_path / "application.properties").write_text("app.name=test") + (tmp_path / "nginx.conf").write_text("server {}") + + result = _collect_config_files(str(tmp_path)) + + assert len(result) == 3 + paths = {path for path, _ in result} + assert "config.yaml" in paths + assert "application.properties" in paths + assert "nginx.conf" in paths + + for path, content in result: + assert len(content) > 0 + + def test_finds_files_in_config_dir(self, tmp_path): + config_dir = tmp_path / "config" + config_dir.mkdir() + (config_dir / "database.txt").write_text("db_config") + (config_dir / "server.txt").write_text("server_config") + + result = _collect_config_files(str(tmp_path)) + + assert len(result) == 2 + paths = {path for path, _ in result} + assert "config/database.txt" in paths + assert "config/server.txt" in paths + + def test_excludes_git_dir(self, tmp_path): + git_dir = tmp_path / ".git" + git_dir.mkdir() + (git_dir / "config").write_text("git config") + (tmp_path / "application.yml").write_text("app: config") + + result = _collect_config_files(str(tmp_path)) + + assert len(result) == 1 + paths = {path for path, _ in result} + assert "application.yml" in paths + assert ".git/config" not in paths + + def test_excludes_pycache(self, tmp_path): + pycache_dir = tmp_path / "__pycache__" + pycache_dir.mkdir() + (pycache_dir / "config.yaml").write_text("cached") + (tmp_path / "application.yml").write_text("app: config") + + result = _collect_config_files(str(tmp_path)) + + assert len(result) == 1 + paths = {path for path, _ in result} + assert "application.yml" in paths + assert "__pycache__/config.yaml" not in paths + + def test_excludes_node_modules(self, tmp_path): + node_modules_dir = tmp_path / "node_modules" + node_modules_dir.mkdir() + (node_modules_dir / "package.json").write_text("{}") + (tmp_path / "application.yml").write_text("app: config") + + result = _collect_config_files(str(tmp_path)) + + assert len(result) == 1 + paths = {path for path, _ in result} + assert "application.yml" in paths + assert "node_modules/package.json" not in paths + + def test_excludes_tox(self, tmp_path): + tox_dir = tmp_path / ".tox" + tox_dir.mkdir() + (tox_dir / "config.ini").write_text("[tox]") + (tmp_path / "application.yml").write_text("app: config") + + result = _collect_config_files(str(tmp_path)) + + assert len(result) == 1 + paths = {path for path, _ in result} + assert "application.yml" in paths + assert ".tox/config.ini" not in paths + + def test_skips_large_files(self, tmp_path): + large_content = "x" * 500_001 + (tmp_path / "large.properties").write_text(large_content) + (tmp_path / "small.properties").write_text("key=value") + + result = _collect_config_files(str(tmp_path)) + + assert len(result) == 1 + paths = {path for path, _ in result} + assert "small.properties" in paths + assert "large.properties" not in paths + + def test_empty_repo_returns_empty(self, tmp_path): + result = _collect_config_files(str(tmp_path)) + assert result == [] + + def test_nested_directory_structure(self, tmp_path): + src_dir = tmp_path / "src" / "main" / "resources" + src_dir.mkdir(parents=True) + (src_dir / "application.yml").write_text("app: test") + + conf_dir = tmp_path / "conf" + conf_dir.mkdir() + (conf_dir / "server.txt").write_text("server config") + + (tmp_path / "config.yaml").write_text("key: value") + + result = _collect_config_files(str(tmp_path)) + + assert len(result) == 3 + paths = {path for path, _ in result} + assert "src/main/resources/application.yml" in paths + assert "conf/server.txt" in paths + assert "config.yaml" in paths + + def test_handles_read_errors_gracefully(self, tmp_path): + (tmp_path / "good.properties").write_text("key=value") + bad_file = tmp_path / "bad.properties" + bad_file.write_bytes(b"\xff\xfe invalid utf-8") + bad_file.chmod(0o000) + + result = _collect_config_files(str(tmp_path)) + + paths = {path for path, _ in result} + assert "good.properties" in paths + + bad_file.chmod(0o644) + + def test_returns_relative_paths(self, tmp_path): + subdir = tmp_path / "config" + subdir.mkdir() + (subdir / "app.yml").write_text("app: config") + + result = _collect_config_files(str(tmp_path)) + + assert len(result) == 1 + path, _ = result[0] + assert path == "config/app.yml" + assert not path.startswith("/") + assert str(tmp_path) not in path + + def test_returns_file_contents(self, tmp_path): + expected_content = "database:\n host: localhost\n port: 5432" + (tmp_path / "config.yaml").write_text(expected_content) + + result = _collect_config_files(str(tmp_path)) + + assert len(result) == 1 + path, content = result[0] + assert path == "config.yaml" + assert content == expected_content + + +class TestSearchConfigContent: + def test_empty_config_files(self): + result = search_config_content([], ["keyword"]) + assert result == [] + + def test_no_matches(self): + config_files = [("app.yml", "database:\n host: localhost")] + result = search_config_content(config_files, ["xstream"]) + assert result == [] + + def test_single_match(self): + config_files = [("app.yml", "line1\nxstream_enabled: true\nline3")] + result = search_config_content(config_files, ["xstream"], context_lines=0) + assert len(result) == 1 + assert "app.yml" in result[0] + assert "xstream_enabled: true" in result[0] + + def test_source_label(self): + config_files = [("app.yml", "match_keyword")] + result = search_config_content(config_files, ["match"], source_label="git://repo", context_lines=0) + assert "source: git://repo" in result[0] + + def test_multiple_files(self): + config_files = [ + ("a.yml", "xstream: yes"), + ("b.yml", "no match here"), + ("c.xml", "xstream config"), + ] + result = search_config_content(config_files, ["xstream"], context_lines=0) + assert len(result) == 2 + assert "a.yml" in result[0] + assert "c.xml" in result[1] + + def test_max_results_cap(self): + config_files = [("big.yml", "\n".join(f"keyword_{i}" for i in range(50)))] + result = search_config_content(config_files, ["keyword"], max_results=5, context_lines=0) + assert len(result) == 5 + + def test_context_lines(self): + content = "line1\nline2\nMATCH_LINE\nline4\nline5" + config_files = [("app.yml", content)] + result = search_config_content(config_files, ["match_line"], context_lines=1) + assert len(result) == 1 + assert "line2" in result[0] + assert "> 3: MATCH_LINE" in result[0] + assert "line4" in result[0] + assert "line1" not in result[0] + assert "line5" not in result[0] + + def test_case_insensitive_keywords(self): + config_files = [("app.yml", "XStream_Config: enabled")] + result = search_config_content(config_files, ["xstream"], context_lines=0) + assert len(result) == 1 + + def test_multiple_keywords_match_any(self): + config_files = [ + ("a.yml", "xstream: true"), + ("b.yml", "deserialization: false"), + ("c.yml", "no relevant content"), + ] + result = search_config_content(config_files, ["xstream", "deserialization"], context_lines=0) + assert len(result) == 2 + + def test_line_number_in_output(self): + config_files = [("app.yml", "line1\nline2\nxstream: true")] + result = search_config_content(config_files, ["xstream"], context_lines=0) + assert "line 3" in result[0] + + def test_default_source_label(self): + config_files = [("app.yml", "match_keyword")] + result = search_config_content(config_files, ["match"], context_lines=0) + assert "source: unknown" in result[0] + + +class TestConfigScannerSourceAwareness: + """Tests for app/dep classification and source scoping in Configuration Scanner.""" + + def _make_configs(self): + return [ + ("config.yaml", "xstream: app-level"), + ("src/main/resources/application.yml", "xstream: app-config"), + ("dependencies-sources/xstream-1.4/config.properties", "xstream: dep-xstream"), + ("dependencies-sources/commons-io-2.6/config.properties", "xstream: dep-commons"), + ] + + def test_app_config_prioritized_over_dependency(self): + from vuln_analysis.utils.source_classification import ( + is_dependency_path, format_app_dep_output, + ) + configs = self._make_configs() + app_configs = [c for c in configs if not is_dependency_path(c[0])] + dep_configs = [c for c in configs if is_dependency_path(c[0])] + + app_matches = search_config_content(app_configs, ["xstream"], max_results=10, context_lines=0) + dep_matches = search_config_content(dep_configs, ["xstream"], max_results=10, context_lines=0) + + result = format_app_dep_output(app_matches, dep_matches, len(app_matches), len(dep_matches), + "No matches") + assert "Main application (2 of 2 results)" in result + assert "Application library dependencies (2 of 2 results)" in result + app_pos = result.index("config.yaml") + dep_pos = result.index("Application library dependencies") + assert app_pos < dep_pos + + def test_source_scope_filters_dependency_configs(self): + from vuln_analysis.utils.source_classification import ( + is_dependency_path, filter_by_source_scope, + ) + configs = self._make_configs() + dep_configs = [c for c in configs if is_dependency_path(c[0])] + + filtered = filter_by_source_scope(dep_configs, ["xstream"], lambda x: x[0]) + assert len(filtered) == 1 + assert "xstream-1.4" in filtered[0][0] + + def test_app_configs_unaffected_by_source_scope(self): + from vuln_analysis.utils.source_classification import is_dependency_path + configs = self._make_configs() + app_configs = [c for c in configs if not is_dependency_path(c[0])] + + app_matches = search_config_content(app_configs, ["xstream"], max_results=10, context_lines=0) + assert len(app_matches) == 2 + + def test_app_dep_headers_in_output(self): + from vuln_analysis.utils.source_classification import format_app_dep_output + result = format_app_dep_output(["match1"], ["match2"], 1, 1, "No matches") + assert "Main application" in result + assert "Application library dependencies" in result + + def test_no_matches_preserves_message(self): + from vuln_analysis.utils.source_classification import format_app_dep_output + result = format_app_dep_output([], [], 0, 0, "No configuration entries found matching: xstream") + assert result == "No configuration entries found matching: xstream" diff --git a/tests/test_dispatcher.py b/tests/test_dispatcher.py new file mode 100644 index 000000000..bdbeef344 --- /dev/null +++ b/tests/test_dispatcher.py @@ -0,0 +1,160 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from unittest.mock import AsyncMock +from pydantic import ValidationError +from langchain_core.messages import HumanMessage + +from vuln_analysis.functions.dispatcher import ( + QuestionRouting, + build_routing_prompt, + dispatch_question, +) + + +class TestQuestionRouting: + def test_valid_reachability_type(self): + routing = QuestionRouting(agent_type="reachability", reason="test reason") + assert routing.agent_type == "reachability" + assert routing.reason == "test reason" + + def test_valid_code_understanding_type(self): + routing = QuestionRouting(agent_type="code_understanding", reason="test reason") + assert routing.agent_type == "code_understanding" + assert routing.reason == "test reason" + + def test_invalid_type_rejected(self): + with pytest.raises(ValidationError) as exc_info: + QuestionRouting(agent_type="invalid", reason="test") + assert "agent_type" in str(exc_info.value) + + def test_missing_agent_type_rejected(self): + with pytest.raises(ValidationError) as exc_info: + QuestionRouting(reason="test") + assert "agent_type" in str(exc_info.value) + + def test_missing_reason_rejected(self): + with pytest.raises(ValidationError) as exc_info: + QuestionRouting(agent_type="reachability") + assert "reason" in str(exc_info.value) + + +class TestBuildRoutingPrompt: + def test_context_and_question_inserted(self): + context = "CVE-2024-1234, package: foo:bar" + question = "Is the function vulnerable()?" + result = build_routing_prompt(context, question) + + assert context in result + assert question in result + + def test_all_few_shot_examples_present(self): + result = build_routing_prompt("context", "question") + + expected_examples = [ + "XStream.fromXML()", + "XML parser", + "BeanUtils.populate()", + "commons-beanutils", + "parseXML()", + "external entity processing", + "newTransformer()", + "deserialize()", + ] + + for example in expected_examples: + assert example in result, f"Missing example: {example}" + + def test_both_agent_types_mentioned(self): + result = build_routing_prompt("context", "question") + + assert "reachability" in result + assert "code_understanding" in result + + def test_empty_inputs(self): + result = build_routing_prompt("", "") + assert isinstance(result, str) + assert len(result) > 0 + + def test_special_chars_in_inputs(self): + context = "CVE {test} with % and {{nested}}" + question = "Is {function}() reachable? % complete" + + result = build_routing_prompt(context, question) + + assert context in result + assert question in result + + +class TestDispatchQuestion: + @pytest.mark.asyncio + async def test_dispatch_returns_routing_result(self): + expected_result = QuestionRouting( + agent_type="reachability", + reason="Test reason" + ) + + mock_llm = AsyncMock() + mock_llm.ainvoke.return_value = expected_result + + result = await dispatch_question( + routing_llm=mock_llm, + question="Test question?", + context_block="Test context", + ) + + assert result == expected_result + assert result.agent_type == "reachability" + assert result.reason == "Test reason" + + @pytest.mark.asyncio + async def test_dispatch_passes_human_message(self): + mock_result = QuestionRouting( + agent_type="code_understanding", + reason="Config check" + ) + + mock_llm = AsyncMock() + mock_llm.ainvoke.return_value = mock_result + + question = "Is the XML parser configured?" + context = "CVE-2024-5678" + + await dispatch_question( + routing_llm=mock_llm, + question=question, + context_block=context, + ) + + mock_llm.ainvoke.assert_called_once() + + call_args = mock_llm.ainvoke.call_args[0][0] + assert len(call_args) == 1 + assert isinstance(call_args[0], HumanMessage) + assert question in call_args[0].content + assert context in call_args[0].content + + @pytest.mark.asyncio + async def test_dispatch_propagates_exception(self): + mock_llm = AsyncMock() + mock_llm.ainvoke.side_effect = ValueError("LLM error") + + with pytest.raises(ValueError, match="LLM error"): + await dispatch_question( + routing_llm=mock_llm, + question="Test question", + context_block="Test context", + ) diff --git a/tests/test_git_commit_searcher.py b/tests/test_git_commit_searcher.py new file mode 100644 index 000000000..9ad0d7848 --- /dev/null +++ b/tests/test_git_commit_searcher.py @@ -0,0 +1,462 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for git_commit_searcher and git_repo_manager: commit discovery in git repositories.""" + +import pytest +from pathlib import Path +from unittest.mock import MagicMock + +from vuln_analysis.utils.git_repo_manager import ( + GitRepoManager, + DEFAULT_CACHE_DIR, + DEFAULT_CLONE_DEPTH, + GIT_HASH_LENGTH_SHORT, + GIT_HASH_LENGTH_FULL, +) +from vuln_analysis.utils.git_commit_searcher import ( + GitCommitSearcher, + GIT_HASH_FULL_PATTERN, + GIT_HASH_SHORT_PATTERN, + SVN_REVISION_PATTERN, + SECURITY_KEYWORDS, + MAX_RESULTS_PER_STRATEGY, +) +from vuln_analysis.functions.code_agent_graph_defs import ( + CommitSearchResult, + GitSearchReport, + ReferenceHints, + SEARCH_METHOD_REVISION, + SEARCH_METHOD_FUNCTION_MESSAGE, + CONFIDENCE_REVISION_DIRECT, + CONFIDENCE_REVISION_BRANCH, + CONFIDENCE_FUNCTION_MESSAGE, + CONFIDENCE_FUNCTION_PICKAXE, + CONFIDENCE_THRESHOLD_MIN, +) + + +# --------------------------------------------------------------------------- +# GitRepoManager Tests +# --------------------------------------------------------------------------- + + +class TestGitRepoManagerCachePath: + """Tests for cache path generation.""" + + def test_github_url_to_cache_path(self): + """GitHub URL converts to proper cache path.""" + manager = GitRepoManager(cache_dir="/tmp/test_cache") + path = manager.get_repo_cache_path("https://github.com/apache/httpd") + + assert path == Path("/tmp/test_cache/github.com/apache/httpd") + + def test_gitlab_url_to_cache_path(self): + """GitLab URL converts to proper cache path.""" + manager = GitRepoManager(cache_dir="/tmp/test_cache") + path = manager.get_repo_cache_path("https://gitlab.gnome.org/GNOME/libxml2") + + assert path == Path("/tmp/test_cache/gitlab.gnome.org/GNOME/libxml2") + + def test_git_suffix_stripped(self): + """URL with .git suffix has it stripped.""" + manager = GitRepoManager(cache_dir="/tmp/test_cache") + path = manager.get_repo_cache_path("https://github.com/curl/curl.git") + + assert path == Path("/tmp/test_cache/github.com/curl/curl") + + def test_kernel_org_url(self): + """kernel.org URL converts properly.""" + manager = GitRepoManager(cache_dir="/tmp/test_cache") + path = manager.get_repo_cache_path( + "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" + ) + + assert "git.kernel.org" in str(path) + assert "linux" in str(path) + + +class TestGitRepoManagerValidation: + """Tests for repository validation.""" + + def test_is_valid_repo_with_git_dir(self, tmp_path): + """Valid repo has .git directory with HEAD.""" + manager = GitRepoManager(cache_dir=tmp_path) + repo_path = tmp_path / "test_repo" + git_dir = repo_path / ".git" + git_dir.mkdir(parents=True) + (git_dir / "HEAD").write_text("ref: refs/heads/main") + + assert manager._is_valid_repo(repo_path) is True + + def test_is_valid_repo_without_git_dir(self, tmp_path): + """Directory without .git is not valid.""" + manager = GitRepoManager(cache_dir=tmp_path) + repo_path = tmp_path / "not_a_repo" + repo_path.mkdir(parents=True) + + assert manager._is_valid_repo(repo_path) is False + + def test_is_valid_repo_missing_head(self, tmp_path): + """Directory with .git but no HEAD is not valid.""" + manager = GitRepoManager(cache_dir=tmp_path) + repo_path = tmp_path / "broken_repo" + git_dir = repo_path / ".git" + git_dir.mkdir(parents=True) + + assert manager._is_valid_repo(repo_path) is False + + +# --------------------------------------------------------------------------- +# Revision Pattern Tests +# --------------------------------------------------------------------------- + + +class TestRevisionPatterns: + """Tests for revision type classification patterns.""" + + def test_full_git_hash_pattern(self): + """40-character hex string matches full hash pattern.""" + full_hash = "a70753d2a61b7a43e11aaa217f8d2844c7d8cfab" + assert GIT_HASH_FULL_PATTERN.match(full_hash) is not None + + def test_short_git_hash_pattern(self): + """7-12 character hex string matches short hash pattern.""" + short_hashes = ["a70753d", "40326939", "a70753d2a61b"] + for h in short_hashes: + assert GIT_HASH_SHORT_PATTERN.match(h) is not None + + def test_svn_revision_with_r_prefix(self): + """SVN revision with r prefix matches.""" + match = SVN_REVISION_PATTERN.match("r1935008") + assert match is not None + assert match.group(1) == "1935008" + + def test_svn_revision_without_prefix(self): + """SVN revision without r prefix matches.""" + match = SVN_REVISION_PATTERN.match("1935008") + assert match is not None + assert match.group(1) == "1935008" + + def test_short_number_does_not_match_svn(self): + """Short numbers (< 5 digits) don't match SVN pattern.""" + assert SVN_REVISION_PATTERN.match("1234") is None + + def test_non_hex_does_not_match_git_hash(self): + """Non-hex strings don't match git hash patterns.""" + assert GIT_HASH_FULL_PATTERN.match("not-a-hash-string") is None + assert GIT_HASH_SHORT_PATTERN.match("xyz1234") is None + + +# --------------------------------------------------------------------------- +# Security Keyword Tests +# --------------------------------------------------------------------------- + + +class TestSecurityKeywords: + """Tests for security keyword boosting.""" + + def test_security_keywords_defined(self): + """Security keywords list is populated.""" + assert len(SECURITY_KEYWORDS) > 0 + + def test_security_keywords_have_boosts(self): + """Each keyword has a positive boost value.""" + for keyword, boost in SECURITY_KEYWORDS: + assert isinstance(keyword, str) + assert isinstance(boost, float) + assert boost > 0 + + def test_common_security_terms_present(self): + """Common security terms are in the list.""" + keyword_names = [k for k, _ in SECURITY_KEYWORDS] + assert "fix" in keyword_names + assert "security" in keyword_names + assert "CVE" in keyword_names + + +# --------------------------------------------------------------------------- +# CommitSearchResult Model Tests +# --------------------------------------------------------------------------- + + +class TestCommitSearchResultModel: + """Tests for CommitSearchResult Pydantic model.""" + + def test_create_valid_result(self): + """Valid CommitSearchResult can be created.""" + result = CommitSearchResult( + commit_hash="a70753d2a61b7a43e11aaa217f8d2844c7d8cfab", + commit_hash_short="a70753d", + commit_message="Fix buffer overflow in parser", + author="Test Author ", + date="2026-01-15T10:30:00+00:00", + files_changed=["src/parser.c", "include/parser.h"], + search_method=SEARCH_METHOD_REVISION, + confidence=0.95, + match_details="grep matched r1935008", + ) + + assert result.commit_hash_short == "a70753d" + assert result.confidence == 0.95 + assert len(result.files_changed) == 2 + + def test_confidence_bounds(self): + """Confidence must be between 0 and 1.""" + with pytest.raises(ValueError): + CommitSearchResult( + commit_hash="a" * 40, + commit_hash_short="a" * 7, + commit_message="test", + author="test", + date="2026-01-01", + search_method=SEARCH_METHOD_REVISION, + confidence=1.5, # Invalid + ) + + def test_search_method_literal(self): + """Search method must be a valid literal.""" + result = CommitSearchResult( + commit_hash="a" * 40, + commit_hash_short="a" * 7, + commit_message="test", + author="test", + date="2026-01-01", + search_method=SEARCH_METHOD_FUNCTION_MESSAGE, + confidence=0.75, + ) + assert result.search_method == "function_message" + + +# --------------------------------------------------------------------------- +# GitSearchReport Model Tests +# --------------------------------------------------------------------------- + + +class TestGitSearchReportModel: + """Tests for GitSearchReport Pydantic model.""" + + def test_create_empty_report(self): + """Empty GitSearchReport can be created.""" + report = GitSearchReport(repo_url="https://github.com/test/repo") + + assert report.repo_url == "https://github.com/test/repo" + assert report.search_success is False + assert report.best_result is None + assert len(report.all_results) == 0 + + def test_report_with_results(self): + """GitSearchReport with results.""" + result = CommitSearchResult( + commit_hash="a" * 40, + commit_hash_short="a" * 7, + commit_message="Fix security issue", + author="test", + date="2026-01-01", + search_method=SEARCH_METHOD_REVISION, + confidence=0.95, + ) + + report = GitSearchReport( + repo_url="https://github.com/test/repo", + best_result=result, + all_results=[result], + search_success=True, + strategies_tried=[SEARCH_METHOD_REVISION], + strategies_succeeded=[SEARCH_METHOD_REVISION], + ) + + assert report.search_success is True + assert report.best_result is not None + assert len(report.strategies_succeeded) == 1 + + +# --------------------------------------------------------------------------- +# GitCommitSearcher Tests (with mocks) +# --------------------------------------------------------------------------- + + +class TestGitCommitSearcherRanking: + """Tests for result ranking logic.""" + + def test_rank_boosts_security_keywords(self): + """Results with security keywords get boosted.""" + manager = MagicMock(spec=GitRepoManager) + searcher = GitCommitSearcher(manager) + + result_with_keyword = CommitSearchResult( + commit_hash="a" * 40, + commit_hash_short="a" * 7, + commit_message="Fix security vulnerability in parser", + author="test", + date="2026-01-01", + search_method=SEARCH_METHOD_FUNCTION_MESSAGE, + confidence=0.75, + ) + + result_without_keyword = CommitSearchResult( + commit_hash="b" * 40, + commit_hash_short="b" * 7, + commit_message="Update documentation", + author="test", + date="2026-01-01", + search_method=SEARCH_METHOD_FUNCTION_MESSAGE, + confidence=0.75, + ) + + ranked = searcher._rank_results( + [result_without_keyword, result_with_keyword], + file_hints=None, + function_hints=None, + ) + + # Result with security keyword should be ranked first + assert ranked[0].commit_hash.startswith("a") + assert ranked[0].confidence > 0.75 + + def test_rank_penalizes_merge_commits(self): + """Merge commits get penalized in ranking.""" + manager = MagicMock(spec=GitRepoManager) + searcher = GitCommitSearcher(manager) + + merge_commit = CommitSearchResult( + commit_hash="a" * 40, + commit_hash_short="a" * 7, + commit_message="Merge branch 'feature' into main", + author="test", + date="2026-01-01", + search_method=SEARCH_METHOD_REVISION, + confidence=0.95, + ) + + regular_commit = CommitSearchResult( + commit_hash="b" * 40, + commit_hash_short="b" * 7, + commit_message="Fix buffer overflow", + author="test", + date="2026-01-01", + search_method=SEARCH_METHOD_REVISION, + confidence=0.95, + ) + + ranked = searcher._rank_results( + [merge_commit, regular_commit], + file_hints=None, + function_hints=None, + ) + + # Regular commit should be ranked first after penalty + assert ranked[0].commit_hash.startswith("b") + + def test_rank_deduplicates_by_hash(self): + """Duplicate commits are removed.""" + manager = MagicMock(spec=GitRepoManager) + searcher = GitCommitSearcher(manager) + + same_hash = "a" * 40 + result1 = CommitSearchResult( + commit_hash=same_hash, + commit_hash_short="a" * 7, + commit_message="Fix issue", + author="test", + date="2026-01-01", + search_method=SEARCH_METHOD_REVISION, + confidence=0.95, + ) + result2 = CommitSearchResult( + commit_hash=same_hash, + commit_hash_short="a" * 7, + commit_message="Fix issue", + author="test", + date="2026-01-01", + search_method=SEARCH_METHOD_FUNCTION_MESSAGE, + confidence=0.75, + ) + + ranked = searcher._rank_results( + [result1, result2], + file_hints=None, + function_hints=None, + ) + + # Should only have one result + assert len(ranked) == 1 + + +# --------------------------------------------------------------------------- +# ReferenceHints Tests +# --------------------------------------------------------------------------- + + +class TestReferenceHints: + """Tests for ReferenceHints model used in searches.""" + + def test_create_hints_with_revision(self): + """ReferenceHints with single revision hint.""" + hints = ReferenceHints( + revision_hint="r1935008", + function_hints=["ap_proxy_cookie_revers"], + file_hints=["proxy_util.c"], + branch_hints=["2.4.x"], + ) + + assert hints.revision_hint == "r1935008" + assert len(hints.function_hints) == 1 + + def test_empty_hints(self): + """Empty ReferenceHints is valid.""" + hints = ReferenceHints() + + assert hints.revision_hint is None + assert len(hints.function_hints) == 0 + assert hints.hint_strength == "none" + + +# --------------------------------------------------------------------------- +# Confidence Constants Tests +# --------------------------------------------------------------------------- + + +class TestConfidenceConstants: + """Tests for confidence threshold constants.""" + + def test_revision_direct_highest(self): + """Direct revision lookup has highest confidence.""" + assert CONFIDENCE_REVISION_DIRECT > CONFIDENCE_REVISION_BRANCH + assert CONFIDENCE_REVISION_DIRECT > CONFIDENCE_FUNCTION_MESSAGE + + def test_function_message_higher_than_pickaxe(self): + """Function message search has higher confidence than pickaxe.""" + assert CONFIDENCE_FUNCTION_MESSAGE > CONFIDENCE_FUNCTION_PICKAXE + + def test_threshold_is_reasonable(self): + """Minimum threshold is a sensible value.""" + assert CONFIDENCE_THRESHOLD_MIN >= 0.0 + assert CONFIDENCE_THRESHOLD_MIN <= 1.0 + assert CONFIDENCE_THRESHOLD_MIN == 0.50 + + +# --------------------------------------------------------------------------- +# Integration Test Helpers +# --------------------------------------------------------------------------- + + +class TestDefaultConstants: + """Tests for default constant values.""" + + def test_default_cache_dir(self): + """Default cache directory is set.""" + assert DEFAULT_CACHE_DIR == ".cache/am_cache/checker/upstream_repos" + + def test_default_clone_depth(self): + """Default clone depth is reasonable.""" + assert DEFAULT_CLONE_DEPTH == 1000 + + def test_hash_lengths(self): + """Git hash lengths are correct.""" + assert GIT_HASH_LENGTH_SHORT == 7 + assert GIT_HASH_LENGTH_FULL == 40 + + def test_max_results(self): + """Max results per strategy is set.""" + assert MAX_RESULTS_PER_STRATEGY == 5 diff --git a/tests/test_import_usage_analyzer.py b/tests/test_import_usage_analyzer.py new file mode 100644 index 000000000..247408138 --- /dev/null +++ b/tests/test_import_usage_analyzer.py @@ -0,0 +1,407 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +from exploit_iq_commons.utils.dep_tree import Ecosystem +from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers_factory import get_language_function_parser +from vuln_analysis.tools.import_usage_analyzer import ( + _find_usage_in_file, + analyze_imports, +) + + +def _get_patterns(ecosystem: Ecosystem, package_name: str) -> list[re.Pattern]: + parser = get_language_function_parser(ecosystem, tree=None) + return parser.get_import_search_patterns(package_name) + + +class TestGoImportSearchPatterns: + """Go-specific pattern tests: single-line, grouped, aliased, dot, blank imports.""" + + def test_single_line_import(self): + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + assert any(p.search('import "github.com/quic-go/quic-go"') for p in patterns) + + def test_grouped_import_all_matches(self): + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + content = 'import (\n\t"github.com/quic-go/quic-go"\n\t"github.com/quic-go/quic-go/http3"\n\t"github.com/quic-go/quic-go/qlog"\n)' + matches = [] + for p in patterns: + matches.extend(m.group(1) for m in p.finditer(content)) + assert "github.com/quic-go/quic-go" in matches + assert "github.com/quic-go/quic-go/http3" in matches + assert "github.com/quic-go/quic-go/qlog" in matches + + def test_sub_packages_only(self): + """File imports only sub-packages, not the root module.""" + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + content = 'import (\n\t"github.com/quic-go/quic-go/http3"\n\t"github.com/quic-go/quic-go/qlog"\n)' + matches = [] + for p in patterns: + matches.extend(m.group(1) for m in p.finditer(content)) + assert len(matches) == 2 + assert "github.com/quic-go/quic-go/http3" in matches + assert "github.com/quic-go/quic-go/qlog" in matches + + def test_aliased_import(self): + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + content = '\tquic "github.com/quic-go/quic-go"' + assert any(p.search(content) for p in patterns) + + def test_blank_identifier_import(self): + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + content = '\t_ "github.com/quic-go/quic-go"' + assert any(p.search(content) for p in patterns) + + def test_dot_import(self): + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + content = '\t. "github.com/quic-go/quic-go"' + assert any(p.search(content) for p in patterns) + + def test_no_false_positive_on_string_literal(self): + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + content = '\tfmt.Println("using github.com/quic-go/quic-go")' + assert not any(p.search(content) for p in patterns) + + def test_no_false_positive_on_comment(self): + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + content = '\t// See github.com/quic-go/quic-go for details' + assert not any(p.search(content) for p in patterns) + + def test_unrelated_import_ignored(self): + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + content = 'import (\n\t"go.uber.org/zap"\n\t"net/http"\n)' + matches = [] + for p in patterns: + matches.extend(m.group(1) for m in p.finditer(content)) + assert len(matches) == 0 + + +class TestGetImportSearchPatterns: + def test_python_import_pattern(self): + patterns = _get_patterns(Ecosystem.PYTHON, "xml.etree") + + assert any(p.search("import xml.etree.ElementTree") for p in patterns) + assert any(p.search("from xml.etree import ElementTree") for p in patterns) + + def test_java_import_pattern(self): + patterns = _get_patterns(Ecosystem.JAVA, "com.thoughtworks.xstream") + + assert any(p.search("import com.thoughtworks.xstream.XStream;") for p in patterns) + assert any(p.search("import static com.thoughtworks.xstream.XStream;") for p in patterns) + + def test_go_import_pattern(self): + patterns = _get_patterns(Ecosystem.GO, "encoding/xml") + + assert any(p.search('import "encoding/xml"') for p in patterns) + assert any(p.search('import (\n "encoding/xml"\n)') for p in patterns) + + def test_javascript_require_pattern(self): + patterns = _get_patterns(Ecosystem.JAVASCRIPT, "xml2js") + + assert any(p.search("const xml = require('xml2js')") for p in patterns) + assert any(p.search('const xml = require("xml2js")') for p in patterns) + + def test_javascript_import_from_pattern(self): + patterns = _get_patterns(Ecosystem.JAVASCRIPT, "xml2js") + + assert any(p.search("import xml from 'xml2js'") for p in patterns) + assert any(p.search('import { parseString } from "xml2js"') for p in patterns) + + def test_c_include_pattern(self): + patterns = _get_patterns(Ecosystem.C_CPP, "libxml") + + assert any(p.search('#include ') for p in patterns) + assert any(p.search('#include "libxml/tree.h"') for p in patterns) + + def test_special_chars_escaped(self): + patterns = _get_patterns(Ecosystem.PYTHON, "foo.bar[baz]") + + assert len(patterns) > 0 + for p in patterns: + assert isinstance(p, re.Pattern) + + def test_case_insensitive(self): + patterns = _get_patterns(Ecosystem.PYTHON, "XmlParser") + + assert any(p.search("import xmlparser") for p in patterns) + assert any(p.search("import XMLPARSER") for p in patterns) + assert any(p.search("import XmlParser") for p in patterns) + + +class TestFindUsageInFile: + def test_finds_usage_sites(self): + content = """import xml.etree.ElementTree +tree = ElementTree.parse('data.xml') +root = tree.getroot() +for child in root: + print(child.tag) +""" + usages = _find_usage_in_file(content, ["xml.etree.ElementTree"]) + + assert len(usages) > 0 + assert any("ElementTree.parse" in u for u in usages) + + def test_skips_import_lines(self): + content = """import xml.etree.ElementTree +from xml.etree import ElementTree +tree = ElementTree.parse('data.xml') +""" + usages = _find_usage_in_file(content, ["ElementTree"]) + + assert len(usages) == 1 + assert "L3:" in usages[0] + assert "parse" in usages[0] + + def test_skips_from_lines(self): + content = """from xml.etree import ElementTree +tree = ElementTree.parse('data.xml') +""" + usages = _find_usage_in_file(content, ["ElementTree"]) + + assert len(usages) == 1 + assert "L2:" in usages[0] + + def test_skips_include_lines(self): + content = """#include +xmlDocPtr doc = xmlParseFile("data.xml"); +""" + usages = _find_usage_in_file(content, ["xmlParseFile"]) + + assert len(usages) == 1 + assert "L2:" in usages[0] + + def test_max_usages_cap(self): + content = "\n".join([f"var x{i} = XStream();" for i in range(20)]) + + usages = _find_usage_in_file(content, ["XStream"], max_usages=5) + + assert len(usages) == 5 + + def test_word_boundary_matching(self): + content = """class MyXStreamHelper: + pass +xs = XStream() +""" + usages = _find_usage_in_file(content, ["XStream"]) + + assert len(usages) == 1 + assert any("L3:" in u and "XStream()" in u for u in usages) + + def test_dotted_name_uses_short_component(self): + content = """import com.thoughtworks.xstream.XStream; +XStream xs = new XStream(); +""" + usages = _find_usage_in_file(content, ["com.thoughtworks.xstream.XStream"]) + + assert len(usages) == 1 + assert "L2:" in usages[0] + assert "XStream xs" in usages[0] + + def test_go_path_short_name_extraction(self): + """Go import paths use / separator — short name should be last path segment.""" + content = """import "github.com/quic-go/quic-go/http3" +server := http3.Server{} +""" + usages = _find_usage_in_file(content, ["github.com/quic-go/quic-go/http3"]) + + assert len(usages) == 1 + assert "http3.Server" in usages[0] + + def test_go_path_encoding_xml(self): + content = """import "encoding/xml" +data, err := xml.Marshal(v) +""" + usages = _find_usage_in_file(content, ["encoding/xml"]) + + assert len(usages) == 1 + assert "xml.Marshal" in usages[0] + + def test_java_dotted_still_works(self): + """Java dotted package names still extract correct short name.""" + content = """import com.thoughtworks.xstream.XStream; +XStream xs = new XStream(); +""" + usages = _find_usage_in_file(content, ["com.thoughtworks.xstream.XStream"]) + + assert len(usages) == 1 + assert "XStream" in usages[0] + + def test_no_usages_returns_empty(self): + content = """import xml.etree.ElementTree +from xml.etree import ElementTree +""" + usages = _find_usage_in_file(content, ["ElementTree"]) + + assert usages == [] + + def test_empty_content(self): + usages = _find_usage_in_file("", ["XStream"]) + + assert usages == [] + + def test_empty_names(self): + content = "xs = XStream();" + usages = _find_usage_in_file(content, []) + + assert usages == [] + + def test_multiple_names_same_line(self): + content = """import foo, bar +result = foo.process(bar.data) +""" + usages = _find_usage_in_file(content, ["foo", "bar"]) + + assert len(usages) == 2 + assert all("L2:" in u for u in usages) + assert any("foo.process" in u for u in usages) + assert any("bar.data" in u for u in usages) + + def test_line_number_format(self): + content = """import XStream +xs = XStream() +""" + usages = _find_usage_in_file(content, ["XStream"]) + + assert usages[0].startswith(" L2:") + + def test_strips_line_content(self): + content = """import XStream + xs = XStream() +""" + usages = _find_usage_in_file(content, ["XStream"]) + + assert "xs = XStream()" in usages[0] + assert usages[0].count(" " * 8) == 0 + + +class _MockSearchResult: + def __init__(self, hits): + self.hits = hits + + +class _MockSearcher: + def __init__(self, docs): + self.docs = docs + self.num_docs = len(docs) + + def doc(self, doc_address): + return self.docs[doc_address] + + def search(self, query, limit=10, count=True): + return _MockSearchResult([(0.0, i) for i in range(min(limit, len(self.docs)))]) + + +def _import_searcher(*docs): + return _MockSearcher(list(docs)) + + +class TestAnalyzeImportsSourceAwareness: + + def _java_patterns(self, package_name): + return _get_patterns(Ecosystem.JAVA, package_name) + + def test_app_imports_prioritized(self): + docs = [ + {"file_path": ["dependencies-sources/xstream-1.4/XStreamConverter.java"], + "content": ["import com.thoughtworks.xstream.XStream;\nXStream xs = new XStream();"]}, + {"file_path": ["src/main/java/App.java"], + "content": ["import com.thoughtworks.xstream.XStream;\nXStream converter = new XStream();"]}, + ] + patterns = self._java_patterns("com.thoughtworks.xstream") + result = analyze_imports(_import_searcher(*docs), patterns, "com.thoughtworks.xstream") + assert "Main application" in result + assert "Application library dependencies" in result + app_pos = result.index("Main application") + dep_pos = result.index("Application library dependencies") + assert app_pos < dep_pos + assert "src/main/java/App.java" in result + + def test_source_scope_filters_dep_imports(self): + docs = [ + {"file_path": ["dependencies-sources/xstream-1.4/Converter.java"], + "content": ["import com.thoughtworks.xstream.XStream;"]}, + {"file_path": ["dependencies-sources/commons-io/IOUtils.java"], + "content": ["import com.thoughtworks.xstream.XStream;"]}, + ] + patterns = self._java_patterns("com.thoughtworks.xstream") + result = analyze_imports(_import_searcher(*docs), patterns, "com.thoughtworks.xstream", + source_scope=["xstream"]) + assert "xstream-1.4" in result + assert "commons-io" not in result + + def test_no_imports_unchanged(self): + docs = [ + {"file_path": ["src/App.java"], "content": ["public class App {}"]}, + ] + patterns = self._java_patterns("com.thoughtworks.xstream") + result = analyze_imports(_import_searcher(*docs), patterns, "com.thoughtworks.xstream", + ecosystem_label="java") + assert "No imports of 'com.thoughtworks.xstream' found" in result + + def test_app_dep_headers_in_output(self): + docs = [ + {"file_path": ["src/App.java"], + "content": ["import com.thoughtworks.xstream.XStream;"]}, + {"file_path": ["vendor/lib/Helper.java"], + "content": ["import com.thoughtworks.xstream.XStream;"]}, + ] + patterns = self._java_patterns("com.thoughtworks.xstream") + result = analyze_imports(_import_searcher(*docs), patterns, "com.thoughtworks.xstream") + assert "Main application" in result + assert "Application library dependencies" in result + + def test_content_fallback_on_chunked_doc(self): + """When a chunked doc has the package name but no import keyword, fallback triggers.""" + docs = [ + {"file_path": ["src/listeners.go"], + "content": ['\t"github.com/quic-go/quic-go/http3"\n\t"github.com/quic-go/quic-go/qlog"\n)\n\nfunc Listen() {}']}, + ] + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + result = analyze_imports(_import_searcher(*docs), patterns, "github.com/quic-go/quic-go") + assert "listeners.go" in result + assert "No imports" not in result + + def test_content_fallback_skips_unrelated_doc(self): + """Fallback doesn't match docs that don't contain the package name.""" + docs = [ + {"file_path": ["src/main.go"], + "content": ["package main\nfunc main() { fmt.Println() }"]}, + ] + patterns = _get_patterns(Ecosystem.GO, "github.com/quic-go/quic-go") + result = analyze_imports(_import_searcher(*docs), patterns, "github.com/quic-go/quic-go", + ecosystem_label="go") + assert "No imports" in result + + def test_content_fallback_java_chunked(self): + """Java chunked doc with import line split from keyword.""" + docs = [ + {"file_path": ["src/Converter.java"], + "content": ["com.thoughtworks.xstream.XStream;\nXStream xs = new XStream();"]}, + ] + patterns = _get_patterns(Ecosystem.JAVA, "com.thoughtworks.xstream") + result = analyze_imports(_import_searcher(*docs), patterns, "com.thoughtworks.xstream") + assert "Converter.java" in result + + def test_only_dep_imports(self): + docs = [ + {"file_path": ["dependencies-sources/lib/Foo.java"], + "content": ["import com.thoughtworks.xstream.XStream;"]}, + ] + patterns = self._java_patterns("com.thoughtworks.xstream") + result = analyze_imports(_import_searcher(*docs), patterns, "com.thoughtworks.xstream") + assert "Main application (0 of 0 results)" in result + assert "Application library dependencies (1 of 1 results)" in result diff --git a/tests/test_intel_utils.py b/tests/test_intel_utils.py new file mode 100644 index 000000000..bb31ccee2 --- /dev/null +++ b/tests/test_intel_utils.py @@ -0,0 +1,295 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for intel_utils: build_critical_context RHSA candidate capping and patch enrichment.""" + +import pytest + +from exploit_iq_commons.data_models.cve_intel import CveIntel, CveIntelRhsa +from vuln_analysis.utils.intel_utils import ( + build_critical_context, + _MAX_RHSA_CANDIDATES, + extract_functions_from_parsed_patch, + enrich_vulnerable_functions_from_patch, +) +from vuln_analysis.utils.web_patch_fetcher import ParsedPatch, PatchFile, PatchHunk + + +class TestBuildCriticalContextRhsaCap: + """Tests for RHSA candidate package capping in build_critical_context.""" + + def _make_cve_intel_with_rhsa_packages(self, count): + """Create a CveIntel with `count` RHSA package_state entries.""" + package_states = [ + CveIntelRhsa.PackageState(package_name=f"rhsa-product-{i}") + for i in range(count) + ] + rhsa = CveIntelRhsa(package_state=package_states) + return CveIntel(vuln_id="CVE-2026-99999", rhsa=rhsa) + + def test_rhsa_cap_limits_candidates(self): + """RHSA with 100+ packages should only add _MAX_RHSA_CANDIDATES to candidates.""" + cve_intel = self._make_cve_intel_with_rhsa_packages(100) + _, candidates, _ = build_critical_context([cve_intel]) + + rhsa_candidates = [c for c in candidates if c["source"] == "rhsa"] + assert len(rhsa_candidates) == _MAX_RHSA_CANDIDATES + + def test_rhsa_below_cap_all_included(self): + """RHSA with fewer than _MAX_RHSA_CANDIDATES packages includes all.""" + cve_intel = self._make_cve_intel_with_rhsa_packages(5) + _, candidates, _ = build_critical_context([cve_intel]) + + rhsa_candidates = [c for c in candidates if c["source"] == "rhsa"] + assert len(rhsa_candidates) == 5 + + def test_rhsa_cap_with_1000_packages(self): + """Reproduces production scenario: 1000+ RHSA packages capped at limit.""" + cve_intel = self._make_cve_intel_with_rhsa_packages(1234) + _, candidates, _ = build_critical_context([cve_intel]) + + rhsa_candidates = [c for c in candidates if c["source"] == "rhsa"] + assert len(rhsa_candidates) == _MAX_RHSA_CANDIDATES + + def test_rhsa_cap_does_not_affect_ghsa(self): + """GHSA candidates are not affected by the RHSA cap.""" + cve_intel = self._make_cve_intel_with_rhsa_packages(100) + # Manually add GHSA data + from exploit_iq_commons.data_models.cve_intel import CveIntelGhsa + cve_intel.ghsa = CveIntelGhsa( + ghsa_id="GHSA-test-0001", + vulnerabilities=[{"package": {"name": "xstream", "ecosystem": "Maven"}}], + ) + _, candidates, _ = build_critical_context([cve_intel]) + + ghsa_candidates = [c for c in candidates if c["source"] == "ghsa"] + rhsa_candidates = [c for c in candidates if c["source"] == "rhsa"] + assert len(ghsa_candidates) == 1 + assert len(rhsa_candidates) == _MAX_RHSA_CANDIDATES + + def test_context_note_still_shows_total_count(self): + """The critical_context note should still mention the full package count.""" + cve_intel = self._make_cve_intel_with_rhsa_packages(50) + context, _, _ = build_critical_context([cve_intel]) + + affected_notes = [c for c in context if "Affected across" in c] + assert len(affected_notes) == 1 + assert "50" in affected_notes[0] + + +class TestExtractFunctionsFromParsedPatchNoiseFilter: + """Tests for test function filtering in extract_functions_from_parsed_patch.""" + + def _make_parsed_patch(self, filename, added_lines): + """Build a ParsedPatch with a single file and hunk.""" + return ParsedPatch(patch_filename="test.patch", files=[ + PatchFile( + source_path="a/" + filename, + target_path="b/" + filename, + hunks=[PatchHunk( + source_start=1, source_length=0, + target_start=1, target_length=len(added_lines), + added_lines=added_lines, removed_lines=[], context_lines=[], + )], + ), + ]) + + JAVA_TEST_LINES = [ + " public void testAllowAccessToClassPropertyFromBeanUtilsBean() {", + " // test body", + " }", + " public void testSuppressClassPropertyByDefaultFromPropertyUtilsBean() {", + " // test body", + " }", + " public String getName() {", + " return this.name;", + " }", + " public void setName(String name) {", + " this.name = name;", + " }", + ] + + def test_java_testFoo_methods_filtered(self): + """Java testFoo naming convention (lowercase test + uppercase letter) must be filtered.""" + pp = self._make_parsed_patch("PropertyUtilsBean.java", self.JAVA_TEST_LINES) + funcs = extract_functions_from_parsed_patch(pp, "java") + assert "testAllowAccessToClassPropertyFromBeanUtilsBean" not in funcs + assert "testSuppressClassPropertyByDefaultFromPropertyUtilsBean" not in funcs + + def test_java_real_methods_kept(self): + """Real methods (getName, setName) must not be filtered.""" + pp = self._make_parsed_patch("PropertyUtilsBean.java", self.JAVA_TEST_LINES) + funcs = extract_functions_from_parsed_patch(pp, "java") + assert "getName" in funcs + assert "setName" in funcs + + def test_go_Test_prefix_filtered(self): + """Go Test prefix (capital T) is filtered.""" + pp = self._make_parsed_patch("handler.go", [ + "func TestDoSAttack(t *testing.T) {", + "func convert(input string) string {", + ]) + funcs = extract_functions_from_parsed_patch(pp, "go") + assert "TestDoSAttack" not in funcs + assert "convert" in funcs + + def test_python_test_underscore_filtered(self): + """Python test_ prefix is filtered.""" + pp = self._make_parsed_patch("parser.py", [ + "def test_parse_input():", + "def parse_input(data):", + ]) + funcs = extract_functions_from_parsed_patch(pp, "python") + assert "test_parse_input" not in funcs + assert "parse_input" in funcs + + def test_java_testDoS_variants_filtered(self): + """All testXxx variants are filtered regardless of suffix.""" + pp = self._make_parsed_patch("XStream.java", [ + " public void testCannotInjectEventHandler() {", + " public void testCannotUseJaxwsInputStreamToDeleteFile() {", + " public void testDoSAttackWithCollections() {", + " public void unmarshal() {", + ]) + funcs = extract_functions_from_parsed_patch(pp, "java") + assert "testCannotInjectEventHandler" not in funcs + assert "testCannotUseJaxwsInputStreamToDeleteFile" not in funcs + assert "testDoSAttackWithCollections" not in funcs + assert "unmarshal" in funcs + + def test_noise_words_filtered(self): + """Built-in noise words (main, init, setup, etc.) are filtered.""" + pp = self._make_parsed_patch("main.go", [ + "func main() {", + "func init() {", + "func Setup() {", + "func realFunc() {", + ]) + funcs = extract_functions_from_parsed_patch(pp, "go") + assert "main" not in funcs + assert "init" not in funcs + assert "realFunc" in funcs + + +class TestEnrichVulnerableFunctionsFromPatch: + """Patch-extracted functions populate critical_context as hints; test files are skipped.""" + + def _make_patch_result(self, files, source="ghsa"): + """Build a WebPatchResult with the given ParsedPatch files.""" + from vuln_analysis.utils.web_patch_fetcher import WebPatchResult, ParsedPatch, PatchFile, PatchHunk + parsed_files = [] + for f in files: + hunks = [PatchHunk( + source_start=1, source_length=0, + target_start=1, target_length=len(f["added_lines"]), + added_lines=f["added_lines"], + removed_lines=f.get("removed_lines", []), + context_lines=[], + )] + parsed_files.append(PatchFile( + source_path="a/" + f["filename"], + target_path="b/" + f["filename"], + hunks=hunks, + )) + return WebPatchResult( + cve_id="CVE-2024-0000", + parsed_patch=ParsedPatch(patch_filename="test.patch", files=parsed_files), + source=source, + patch_url="https://example.com/patch", + fixed_commit="abc123", + repo_url="https://github.com/test/repo", + ) + + @pytest.mark.asyncio + async def test_source_functions_added_to_critical_context_as_hints(self): + """Functions from non-test source files are added to critical_context as hints.""" + critical_context: list[str] = [] + vulnerable_functions: set[str] = set() + patch_result = self._make_patch_result([{ + "filename": "src/main/java/org/apache/commons/beanutils/PropertyUtilsBean.java", + "added_lines": [ + " public Object getProperty(Object bean, String name) {", + " return null;", + " }", + ], + }]) + + await enrich_vulnerable_functions_from_patch( + [], critical_context, vulnerable_functions, "java", + patch_result=patch_result, + ) + + assert len(vulnerable_functions) == 0, "patch-extracted functions should not be added to vulnerable_functions" + patch_context = [c for c in critical_context if "remediation patch hint" in c] + assert len(patch_context) > 0 + assert "getProperty" in patch_context[0] + + @pytest.mark.asyncio + async def test_test_files_skipped(self): + """Files matching test patterns are skipped entirely.""" + critical_context: list[str] = [] + vulnerable_functions: set[str] = set() + patch_result = self._make_patch_result([ + { + "filename": "src/main/java/org/apache/commons/beanutils/PropertyUtilsBean.java", + "added_lines": [ + " public Object getProperty(Object bean, String name) {", + " return null;", + " }", + ], + }, + { + "filename": "src/test/java/org/apache/commons/beanutils/PropertyUtilsBeanTestCase.java", + "added_lines": [ + " public void testAllowAccessToClassProperty() {", + " }", + " public void testSuppressClassProperty() {", + " }", + ], + }, + ]) + + await enrich_vulnerable_functions_from_patch( + [], critical_context, vulnerable_functions, "java", + patch_result=patch_result, + ) + + assert len(vulnerable_functions) == 0, "patch-extracted functions should not be added to vulnerable_functions" + patch_context = [c for c in critical_context if "remediation patch hint" in c] + assert len(patch_context) > 0 + assert "getProperty" in patch_context[0] + assert "testAllowAccessToClassProperty" not in patch_context[0] + assert "testSuppressClassProperty" not in patch_context[0] + + @pytest.mark.asyncio + async def test_go_test_files_skipped(self): + """Go _test.go files are skipped.""" + critical_context: list[str] = [] + vulnerable_functions: set[str] = set() + patch_result = self._make_patch_result([ + { + "filename": "internal/ackhandler/sent_packet_handler.go", + "added_lines": [ + "func detectAndRemoveAckedPackets() {", + ], + }, + { + "filename": "internal/ackhandler/sent_packet_handler_test.go", + "added_lines": [ + "func TestDoSAttack(t *testing.T) {", + "func BenchmarkAckHandler(b *testing.B) {", + ], + }, + ]) + + await enrich_vulnerable_functions_from_patch( + [], critical_context, vulnerable_functions, "go", + patch_result=patch_result, + ) + + assert len(vulnerable_functions) == 0, "patch-extracted functions should not be added to vulnerable_functions" + patch_context = [c for c in critical_context if "remediation patch hint" in c] + assert len(patch_context) > 0 + assert "detectAndRemoveAckedPackets" in patch_context[0] + assert "TestDoSAttack" not in patch_context[0] + assert "BenchmarkAckHandler" not in patch_context[0] \ No newline at end of file diff --git a/tests/test_java_script_extended.py b/tests/test_java_script_extended.py deleted file mode 100644 index 7c7fd430f..000000000 --- a/tests/test_java_script_extended.py +++ /dev/null @@ -1,140 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest - -from vuln_analysis.utils.js_extended_parser import ExtendedJavaScriptSegmenter - -TEST_CASES = [ - { - "name": - "regular_script", - "code": - """ -function hello() { - console.log('Hello'); -} - -class MyClass { - constructor() { - this.value = 42; - } -} -""", - "expected_functions": - 2, # One function and one class - "should_parse": - True, - }, - { - "name": - "es_module", - "code": - """ -import { something } from './module'; -export function exportedFunc() { - return 'exported'; -} -export class ExportedClass { - method() {} -} -""", - "expected_functions": - 2, # One exported function and one exported class - "should_parse": - True, - }, - { - "name": - "optional_chaining", - "code": - """ -function processUser(user) { - return user?.profile?.name; -} -class UserManager { - getAddress() { - return this.user?.address?.street; - } -} -""", - "expected_functions": - 2, - "should_parse": - True, - }, - { - "name": "shebang_file", - "code": """#!/usr/bin/env node -function main() { - console.log('Main'); -} -""", - "expected_functions": 0, # Should skip due to shebang - "should_parse": False, - } -] - - -@pytest.mark.parametrize("test_case", TEST_CASES, ids=lambda x: x["name"]) -def test_function_extraction(test_case): - """Test that functions and classes are correctly extracted.""" - segmenter = ExtendedJavaScriptSegmenter(test_case["code"]) - functions = segmenter.extract_functions_classes() - assert len(functions) == test_case["expected_functions"] - if not test_case["should_parse"]: - assert segmenter.skip_file - - -@pytest.mark.parametrize("test_case", TEST_CASES, ids=lambda x: x["name"]) -def test_code_simplification(test_case): - """Test that code is correctly simplified.""" - segmenter = ExtendedJavaScriptSegmenter(test_case["code"]) - simplified = segmenter.simplify_code() - - # Basic checks - assert isinstance(simplified, str) - if not test_case["should_parse"]: - assert segmenter.skip_file - assert simplified == test_case["code"] # Return original code for unparseable files - else: - # Should have fewer or equal lines than original - assert len(simplified.splitlines()) <= len(test_case["code"].splitlines()) - # Should contain "// Code for:" for each function/class - if test_case["expected_functions"] > 0: - assert "// Code for:" in simplified - - -def test_optional_chaining_replacement(): - """Test that optional chaining is correctly replaced.""" - code = "const name = user?.profile?.name;" - segmenter = ExtendedJavaScriptSegmenter(code) - assert not segmenter.skip_file - assert "?." not in segmenter.code - assert "." in segmenter.code - - -def test_invalid_js(): - """Test handling of invalid JavaScript code.""" - code = "this is not valid javascript" - segmenter = ExtendedJavaScriptSegmenter(code) - assert not segmenter.skip_file # Should attempt to parse - - # Should handle errors gracefully - functions = segmenter.extract_functions_classes() - assert functions == [] - - simplified = segmenter.simplify_code() - assert simplified == code # Return original code for invalid JS diff --git a/tests/test_java_script_extended_segmenter.py b/tests/test_java_script_extended_segmenter.py new file mode 100644 index 000000000..41dfef7a5 --- /dev/null +++ b/tests/test_java_script_extended_segmenter.py @@ -0,0 +1,767 @@ +import pytest + +from exploit_iq_commons.utils.javascript_extended_segmenter import ExtendedJavaScriptSegmenter + +TEST_CASES = [ + { + "name": + "regular_script", + "code": + """ +function hello() { + console.log('Hello'); +} + + +class MyClass { + constructor() { + this.value = 42; + } +} +""", + "expected_functions": + 3, # One function and one class and one class function + "should_parse": + True, + }, + { + "name": + "es_module", + "code": + """ +import { something } from './module'; +export function exportedFunc() { + return 'exported'; +} +export class ExportedClass { + method() {} +} +""", + "expected_functions": + 3, # One exported function and one exported class + "should_parse": + True, + }, + { + "name": + "optional_chaining", + "code": + """ +function processUser(user) { + return user?.profile?.name; +} +class UserManager { + getAddress() { + return this.user?.address?.street; + } +} +""", + "expected_functions": + 3, + "should_parse": + True, + }, + { + "name": "shebang_file", + "code": """#!/usr/bin/env node +function main() { + console.log('Main'); +} +""", + "expected_functions": 0, # Should skip due to shebang + "should_parse": False, + } +] + + +@pytest.mark.parametrize("test_case", TEST_CASES, ids=lambda x: x["name"]) +def test_function_extraction(test_case): + """Test that functions and classes are correctly extracted.""" + segmenter = ExtendedJavaScriptSegmenter(test_case["code"]) + functions = segmenter.extract_functions_classes() + assert len(functions) == test_case["expected_functions"] + if not test_case["should_parse"]: + assert segmenter.skip_file + + +@pytest.mark.parametrize("test_case", TEST_CASES, ids=lambda x: x["name"]) +def test_code_simplification(test_case): + """Test that code is correctly simplified.""" + segmenter = ExtendedJavaScriptSegmenter(test_case["code"]) + simplified = segmenter.simplify_code() + + # Basic checks + assert isinstance(simplified, str) + if not test_case["should_parse"]: + assert segmenter.skip_file + assert simplified == test_case["code"] # Return original code for unparseable files + else: + # Should have fewer or equal lines than original + assert len(simplified.splitlines()) <= len(test_case["code"].splitlines()) + # Should contain "// Code for:" for each function/class + if test_case["expected_functions"] > 0: + assert "// Code for:" in simplified + + +def test_optional_chaining_preservation(): + """Test that optional chaining is preserved (tree-sitter handles it natively).""" + code = "const name = user?.profile?.name;" + segmenter = ExtendedJavaScriptSegmenter(code) + assert not segmenter.skip_file + assert "?." in segmenter.code + + +def test_invalid_js(): + """Test handling of invalid JavaScript code.""" + code = "this is not valid javascript" + segmenter = ExtendedJavaScriptSegmenter(code) + assert not segmenter.skip_file # Should attempt to parse + + # Should handle errors gracefully + functions = segmenter.extract_functions_classes() + assert functions == [] + + simplified = segmenter.simplify_code() + assert simplified == code # Return original code for invalid JS + +def test_extract_function_declaration(): + """Test extraction of regular function declarations.""" + code = """ +function hello(name) { + console.log('Hello ' + name); +} + +function goodbye() { + return 'Goodbye'; +} +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + assert len(functions) == 2 + assert 'function hello' in functions[0] + assert 'function goodbye' in functions[1] + + +def test_extract_function_expression(): + """Test extraction of function expressions.""" + code = """ +const greet = function(name) { + return 'Hello ' + name; +}; + +var farewell = function() { + return 'Goodbye'; +}; +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + assert len(functions) == 2 + assert 'greet' in functions[0] and 'function' in functions[0] + assert 'farewell' in functions[1] and 'function' in functions[1] + + +def test_extract_arrow_function(): + """Test extraction of arrow functions (T039).""" + code = """ +const add = (a, b) => { + return a + b; +}; + +const multiply = (x, y) => x * y; +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + assert len(functions) == 2 + assert 'add' in functions[0] + assert 'multiply' in functions[1] + + +def test_extract_async_function(): + """Test extraction of async functions (T040).""" + code = """ +async function fetchData(url) { + const response = await fetch(url); + return response.json(); +} + +const asyncArrow = async () => { + return await Promise.resolve('data'); +}; +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + assert len(functions) == 2 + assert 'fetchData' in functions[0] and 'async' in functions[0] + assert 'asyncArrow' in functions[1] + + + +def test_extract_generator_function(): + """Test extraction of generator functions (T041).""" + code = """ +function* numberGenerator() { + yield 1; + yield 2; + yield 3; +} + +function* fibonacci(n) { + let a = 0, b = 1; + while (n-- > 0) { + yield a; + [a, b] = [b, a + b]; + } +} +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + assert len(functions) == 2 + assert 'numberGenerator' in functions[0] + assert 'fibonacci' in functions[1] + + +def test_extract_class_definition(): + """Test extraction of class definitions.""" + code = """ +class Person { + constructor(name, age) { + this.name = name; + this.age = age; + } + + greet() { + return `Hello, I'm ${this.name}`; + } + + getAge() { + return this.age; + } +} +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + # Should extract: class definition + 3 methods (constructor, greet, getAge) + assert len(functions) == 4 + + # Verify class is extracted + assert any('class Person' in f for f in functions) + + +def test_extract_class_methods_with_annotation(): + """Test that class methods have proper //(class: ClassName) annotation (T042).""" + code = """ +class Calculator { + constructor(initial) { + this.value = initial; + } + + add(n) { + this.value += n; + return this; + } + + subtract(n) { + this.value -= n; + return this; + } +} +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + # Find method functions (not the class itself) + method_functions = [f for f in functions if '//(class: Calculator)' in f] + + # Should have constructor, add, and subtract methods with annotations + assert len(method_functions) == 3 + + # Verify annotation format matches Python pattern: //(class: ClassName) + for method_func in method_functions: + assert method_func.strip().endswith('//(class: Calculator)') + assert 'constructor' in method_func or 'add' in method_func or 'subtract' in method_func + + +def test_parse_es6_module(): + """Test parsing of ES6 import/export syntax.""" + code = """ +import { helper } from './utils'; +import defaultExport from './default'; + +export function publicFunction() { + return helper(); +} + +export class PublicClass { + method() { + return 'public'; + } +} + +export default function() { + return 'default export'; +} +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + # Should extract exported functions and classes + assert len(functions) == 4 + assert not segmenter.skip_file + + +def test_parse_commonjs_module(): + """Test parsing of CommonJS require/module.exports syntax.""" + code = """ +const path = require('path'); +const { helper } = require('./utils'); + +function internalFunction() { + return path.join('/tmp', 'file.txt'); +} + +module.exports = { + internalFunction, + publicFunction: function() { + return helper(); + } +}; +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + # Should extract functions + assert len(functions) >= 1 + assert not segmenter.skip_file + + +def test_destructured_import_resolution(): + """Test handling of destructured imports (T043).""" + code = """ +import { foo, bar as baz } from 'module'; +const { method1, method2: renamed } = require('package'); + +function useImports() { + foo(); + baz(); + method1(); + renamed(); +} +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + # Should handle destructured imports without errors + assert len(functions) == 1 + assert not segmenter.skip_file + assert 'useImports' in functions[0] + +def test_mixed_function_types(): + """Test file with mixed function declaration types.""" + code = """ +// Regular function +function regularFunc() { return 1; } + +// Arrow function +const arrowFunc = () => 2; + +// Async function +async function asyncFunc() { return 3; } + +// Generator +function* genFunc() { yield 4; } + +// Class with methods +class MyClass { + constructor() {} + method() { return 5; } + async asyncMethod() { return 6; } +} +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + # Should extract multiple types of functions + assert len(functions) >= 8 + +def test_empty_file(): + """Test handling of empty files.""" + code = "" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + assert len(functions) == 0 + assert not segmenter.skip_file + + +def test_comments_only(): + """Test handling of files with only comments.""" + code = """ +// This is a comment +/* Multi-line + comment */ +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + assert len(functions) == 0 + + +def test_shebang_file(): + """Test that files with shebang are skipped.""" + code = """#!/usr/bin/env node +function main() { + console.log('Should be skipped'); +} +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + assert segmenter.skip_file + assert len(functions) == 0 + + +OBJECT_LITERAL_TEST_CASES = [ + { + "name": "shorthand_methods", + "code": """ +const api = { + fetchUser(id) { + return fetch('/users/' + id); + }, + parseResponse(res) { + return res.json(); + } +}; +""", + "expected_count": 3, + "object_var": "const api", + "annotation_name": "api", + "expected_methods": 2, + }, + { + "name": "function_expressions", + "code": """ +const utils = { + format: function(data) { + return JSON.stringify(data); + }, + parse: function(str) { + return JSON.parse(str); + } +}; +""", + "expected_count": 3, + "object_var": "const utils", + "annotation_name": "utils", + "expected_methods": 2, + }, + { + "name": "arrow_functions", + "code": """ +const handlers = { + onClick: () => { + console.log('clicked'); + }, + onHover: (e) => e.target.style.color = 'red' +}; +""", + "expected_count": 3, + "object_var": "const handlers", + "annotation_name": "handlers", + "expected_methods": 2, + }, + { + "name": "shorthand_methods_with_annotation", + "code": """ +const calculator = { + add(a, b) { + return a + b; + }, + subtract(a, b) { + return a - b; + } +}; +""", + "expected_count": 3, + "object_var": "const calculator", + "annotation_name": "calculator", + "expected_methods": 2, + }, + { + "name": "exported_object", + "code": """ +export const config = { + getBaseUrl() { + return 'https://api.example.com'; + }, + getTimeout() { + return 5000; + } +}; +""", + "expected_count": 3, + "object_var": "const config", + "annotation_name": "config", + "expected_methods": 2, + }, + { + "name": "object_without_methods_not_extracted", + "code": """ +const data = { + name: 'John', + age: 30, + active: true +}; + +function processData() { + return data; +} +""", + "expected_count": 1, + "object_var": None, + "annotation_name": None, + "expected_methods": 0, + }, +] + + +@pytest.mark.parametrize("test_case", OBJECT_LITERAL_TEST_CASES, ids=lambda x: x["name"]) +def test_object_literal_extraction(test_case): + """Test extraction of object literals with various method syntaxes.""" + segmenter = ExtendedJavaScriptSegmenter(test_case["code"]) + functions = segmenter.extract_functions_classes() + + assert len(functions) == test_case["expected_count"], \ + f"Expected {test_case['expected_count']} items, got {len(functions)}" + + if test_case["object_var"]: + assert any(test_case["object_var"] in f for f in functions), \ + f"Object '{test_case['object_var']}' not found" + + if test_case["annotation_name"]: + annotation = f'//(class: {test_case["annotation_name"]})' + method_functions = [f for f in functions if annotation in f] + assert len(method_functions) == test_case["expected_methods"], \ + f"Expected {test_case['expected_methods']} methods with annotation, got {len(method_functions)}" + + +def test_mixed_classes_and_objects(): + """Test file with both classes and object literals.""" + code = """ +class UserService { + constructor() { + this.users = []; + } + + getUser(id) { + return this.users.find(u => u.id === id); + } +} + +const apiUtils = { + formatUrl(path) { + return '/api' + path; + }, + handleError(err) { + console.error(err); + } +}; +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + # Should extract: + # - Class declaration (1) + # - Class methods with annotation (2: constructor, getUser) + # - Object literal (1) + # - Object methods with annotation (2: formatUrl, handleError) + assert len(functions) == 6 + + # Verify class methods have class annotation + class_methods = [f for f in functions if '//(class: UserService)' in f] + assert len(class_methods) == 2 + + # Verify object methods also have class annotation (unified pattern) + object_methods = [f for f in functions if '//(class: apiUtils)' in f] + assert len(object_methods) == 2 + + +def test_anonymous_default_export_class_methods(): + """Test that methods from 'export default class { ... }' get //(class:) annotations.""" + code = """ +export default class { + connect() { + return this.db.connect(); + } + + disconnect() { + this.db.close(); + } +} +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + # Should extract: the class chunk + 2 methods with AnonymousClass annotation + method_functions = [f for f in functions if '//(class: AnonymousClass)' in f] + assert len(method_functions) == 2 + assert any('connect' in f for f in method_functions) + assert any('disconnect' in f for f in method_functions) + + +def test_named_default_export_class_methods(): + """Test that methods from 'export default class Foo { ... }' get //(class: Foo) annotations.""" + code = """ +export default class Foo { + bar() { + return 1; + } +} +""" + segmenter = ExtendedJavaScriptSegmenter(code) + functions = segmenter.extract_functions_classes() + + method_functions = [f for f in functions if '//(class: Foo)' in f] + assert len(method_functions) == 1 + assert 'bar' in method_functions[0] + + +# ============================================================================= +# should_skip too broad — app-level dist/ directories excluded +# ============================================================================= + +class TestShouldSkipEdgeCases: + """should_skip currently excludes any 'dist/' outside node_modules, even + when it's nested inside application source like 'src/dist/'.""" + + def test_node_modules_dist_preserved(self): + """Third-party dist/ inside node_modules should NOT be skipped.""" + assert not ExtendedJavaScriptSegmenter.should_skip( + "node_modules/lodash/dist/lodash.js" + ) + + def test_node_modules_nested_dist_preserved(self): + """Nested dist/ inside node_modules should NOT be skipped.""" + assert not ExtendedJavaScriptSegmenter.should_skip( + "node_modules/@babel/core/dist/index.js" + ) + + def test_root_dist_skipped(self): + """Root-level dist/ IS a build artifact — should be skipped.""" + assert ExtendedJavaScriptSegmenter.should_skip("dist/bundle.js") + + def test_root_build_static_skipped(self): + """Root-level build/static/ IS a build artifact — should be skipped.""" + assert ExtendedJavaScriptSegmenter.should_skip("build/static/js/main.js") + + def test_minified_file_skipped(self): + """.min.js files should always be skipped.""" + assert ExtendedJavaScriptSegmenter.should_skip("lib/utils.min.js") + + def test_src_dist_should_not_be_skipped(self): + """dist/ inside src/ is likely application source, not a build artifact. + Currently should_skip returns True here — this is the bug.""" + assert not ExtendedJavaScriptSegmenter.should_skip( + "src/dist/utils.js" + ), "src/dist/utils.js is likely app source, not a build artifact" + + def test_lib_dist_should_not_be_skipped(self): + """dist/ inside lib/ is likely a sub-module, not a build artifact.""" + assert not ExtendedJavaScriptSegmenter.should_skip( + "lib/dist/helpers.js" + ), "lib/dist/helpers.js is likely app source, not a build artifact" + + def test_regular_app_file_not_skipped(self): + """Regular application files should not be skipped.""" + assert not ExtendedJavaScriptSegmenter.should_skip("src/app/main.js") + + def test_bundle_file_skipped(self): + """.bundle.js files should always be skipped.""" + assert ExtendedJavaScriptSegmenter.should_skip("assets/app.bundle.js") + + +# ============================================================================= +# Thread-safe class-level init +# ============================================================================= + +class TestThreadSafeInit: + """Class-level caches (_js_language, _js_chunk_query, _js_error_query) + are initialized on first instantiation with a None-check but no lock. + Two threads hitting __init__ simultaneously could see _js_language set + but _js_chunk_query still None.""" + + def test_concurrent_init_produces_valid_segmenters(self): + """Create segmenters from multiple threads and verify they all parse correctly.""" + from concurrent.futures import ThreadPoolExecutor, as_completed + + code = "function hello() { return 42; }" + results = [] + errors = [] + + def create_and_extract(idx): + seg = ExtendedJavaScriptSegmenter(code) + funcs = seg.extract_functions_classes() + return idx, funcs + + with ThreadPoolExecutor(max_workers=8) as pool: + futures = [pool.submit(create_and_extract, i) for i in range(20)] + for f in as_completed(futures): + try: + idx, funcs = f.result() + results.append((idx, funcs)) + except Exception as exc: + errors.append(exc) + + assert not errors, f"Concurrent init raised: {errors}" + assert len(results) == 20 + for idx, funcs in results: + assert len(funcs) == 1, f"Thread {idx}: expected 1 function, got {len(funcs)}" + + def test_class_caches_are_set_after_init(self): + """After any instantiation, all three class caches must be non-None.""" + ExtendedJavaScriptSegmenter("var x = 1;") + assert ExtendedJavaScriptSegmenter._js_language is not None + assert ExtendedJavaScriptSegmenter._js_chunk_query is not None + assert ExtendedJavaScriptSegmenter._js_error_query is not None + + +# ============================================================================= +# Bug: should_skip .min.js check runs before node_modules path analysis +# Lines 102-104: .min.js suffix check returns True before we check node_modules +# ============================================================================= + +class TestShouldSkipMinJsInNodeModules: + """Third-party .min.js files inside node_modules should NOT be skipped.""" + + def test_minified_in_node_modules_should_not_be_skipped(self): + """node_modules/jquery/dist/jquery.min.js is legitimate third-party source.""" + assert not ExtendedJavaScriptSegmenter.should_skip( + "node_modules/jquery/dist/jquery.min.js" + ), ".min.js inside node_modules should not be skipped" + + def test_minified_bundle_in_node_modules_should_not_be_skipped(self): + """node_modules/lodash/lodash.min.js is legitimate.""" + assert not ExtendedJavaScriptSegmenter.should_skip( + "node_modules/lodash/lodash.min.js" + ) + + def test_bundle_in_node_modules_should_not_be_skipped(self): + """node_modules/chart.js/dist/chart.bundle.js is legitimate.""" + assert not ExtendedJavaScriptSegmenter.should_skip( + "node_modules/chart.js/dist/chart.bundle.js" + ) + + def test_app_level_minified_still_skipped(self): + """App-level .min.js files are build artifacts — should be skipped.""" + assert ExtendedJavaScriptSegmenter.should_skip("lib/utils.min.js") + + def test_app_level_bundle_still_skipped(self): + """App-level .bundle.js files are build artifacts — should be skipped.""" + assert ExtendedJavaScriptSegmenter.should_skip("assets/app.bundle.js") + diff --git a/tests/test_javascript_functions_parser.py b/tests/test_javascript_functions_parser.py new file mode 100644 index 000000000..fda6ea5bb --- /dev/null +++ b/tests/test_javascript_functions_parser.py @@ -0,0 +1,4502 @@ +import pytest +from langchain_core.documents import Document + +from exploit_iq_commons.utils.functions_parsers.javascript_functions_parser import JavaScriptFunctionsParser +from exploit_iq_commons.utils.javascript_extended_segmenter import ExtendedJavaScriptSegmenter + + +@pytest.fixture +def parser(): + """Create a JavaScriptFunctionsParser instance for testing.""" + return JavaScriptFunctionsParser() + + +@pytest.mark.parametrize("code,expected_name", [ + # Regular function declarations + ( + "function myFunc() {\n return 42;\n}", + "myFunc" + ), + ( + "function hello(name) {\n console.log(name);\n}", + "hello" + ), + ( + "function calculateSum(a, b) {\n return a + b;\n}", + "calculateSum" + ), + + # Async function declarations + ( + "async function fetchData() {\n return await getData();\n}", + "fetchData" + ), + ( + "async function processRequest(req) {\n const data = await req.json();\n return data;\n}", + "processRequest" + ), + + # Arrow functions with const (with parentheses) + ( + "const add = (a, b) => {\n return a + b;\n}", + "add" + ), + ( + "const greet = () => {\n console.log('hello');\n}", + "greet" + ), + ( + "const multiply = (x, y) => x * y", + "multiply" + ), + + # Arrow functions without parentheses (single parameter) + ( + "const last = arr => arr[arr.length - 1];", + "last" + ), + ( + "const double = n => n * 2;", + "double" + ), + ( + "const identity = x => x;", + "identity" + ), + + # Arrow functions with let + ( + "let divide = (a, b) => {\n return a / b;\n}", + "divide" + ), + + # Arrow functions with var + ( + "var subtract = (a, b) => {\n return a - b;\n}", + "subtract" + ), + + # Async arrow functions + ( + "const asyncFetch = async () => {\n return await fetch('/api');\n}", + "asyncFetch" + ), + ( + "const loadData = async (url) => {\n const response = await fetch(url);\n return response.json();\n}", + "loadData" + ), + + # Function expressions with const + ( + "const myFunction = function() {\n return 'hello';\n}", + "myFunction" + ), + ( + "const handler = function(event) {\n event.preventDefault();\n}", + "handler" + ), + + # Function expressions with let + ( + "let process = function(data) {\n return data.map(x => x * 2);\n}", + "process" + ), + + # Function expressions with var + ( + "var oldStyle = function() {\n return 'legacy';\n}", + "oldStyle" + ), + + # Async function expressions + ( + "const asyncProcess = async function() {\n return await doWork();\n}", + "asyncProcess" + ), + ( + "const handleAsync = async function(data) {\n await save(data);\n}", + "handleAsync" + ), + + # Class methods (without class declaration) + ( + "constructor() {\n this.value = 0;\n}//(class: MyClass)", + "constructor" + ), + ( + "getValue() {\n return this.value;\n}//(class: MyClass)", + "getValue" + ), + ( + "setValue(val) {\n this.value = val;\n}//(class: Calculator)", + "setValue" + ), + + # Async class methods + ( + "async fetchUser() {\n return await api.getUser();\n}//(class: UserService)", + "fetchUser" + ), + ( + "async save(data) {\n await db.save(data);\n}//(class: Repository)", + "save" + ), + + # Functions with extra whitespace + ( + "function spacedFunc () {\n return true;\n}", + "spacedFunc" + ), + ( + "const arrowSpaced = () => {\n return false;\n}", + "arrowSpaced" + ), + + # Generator functions + ( + "function* generator() {\n yield 1;\n}", + "generator" + ), + + # Curried arrow functions (returns another function) + ( + "const helper = minVersion => tpl => ({\n minVersion,\n ast: () => _template.default.program.ast(tpl)\n});", + "helper" + ), + ( + "const curry = a => b => c => a + b + c;", + "curry" + ), + + # Computed property methods (using symbols or expressions) + ( + "[Symbol.iterator]() {\n\t\treturn createHeadersIterator(this, 'key+value');\n\t}", + "Symbol.iterator" + ), + ( + "[Symbol.toStringTag]() {\n return 'CustomClass';\n}", + "Symbol.toStringTag" + ), + ( + "[computed]() {\n return this.value;\n}", + "computed" + ), + + # multiple parameters + ( + "function complexParams(a, b, c, d) {\n return a + b + c + d;\n}", + "complexParams" + ), + ( + "const arrowMultiParams = (x, y, z) => {\n return x + y + z;\n}", + "arrowMultiParams" + ), + ( + "", + "", + ), + # Multiline async function + ( + """async function processUserData(userId) { + const user = await fetchUser(userId); + const profile = await fetchProfile(user.profileId); + const settings = await fetchSettings(user.settingsId); + + return { + user, + profile, + settings + }; + }""", + "processUserData", + ), + # Complex arrow function + ( + """const calculateTotalPrice = (items) => { + const subtotal = items.reduce((sum, item) => sum + item.price, 0); + const tax = subtotal * 0.08; + const shipping = items.length > 0 ? 5.99 : 0; + return subtotal + tax + shipping; + }""", + "calculateTotalPrice", + ), + # Class method with annotation + ( + """async validateInput(input) { + if (!input) { + throw new Error('Input is required'); + } + const sanitized = this.sanitize(input); + return await this.validate(sanitized); + }//(class: InputValidator)""", + "validateInput", + ), + ( + "\n\nfunction myFunc() {\n return 42;\n}\n\n", + "myFunc" + ), + ( + " const indented = () => {\n return 'test';\n }", + "indented" + ), + # Test with tabs + ( + "\tfunction tabbedFunc() {\n\t\treturn true;\n\t}", + "tabbedFunc" + ), +]) +def test_get_function_name_valid_patterns(parser, code, expected_name): + """Test get_function_name with various valid function patterns.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + result = parser.get_function_name(doc) + assert result == expected_name + + +@pytest.mark.parametrize("code,expected_name", [ + # Object literal property with arrow function + ( + "handler: (req, res) => {\n res.send('ok');\n}", + "handler" + ), + ( + "onClick: async (event) => {\n await handleClick(event);\n}", + "onClick" + ), + # Object literal property with single-param arrow (no parens) + ( + "transform: item => item.toUpperCase()", + "transform" + ), + ( + "callback: async result => {\n await process(result);\n}", + "callback" + ), + # Object literal property with function expression + ( + "initialize: function() {\n this.setup();\n}", + "initialize" + ), + ( + "process: async function(data) {\n await save(data);\n}", + "process" + ), + # Property assignment with single-param arrow + ( + "module.exports.handler = event => {\n return event.body;\n}", + "handler" + ), + ( + "self.callback = async result => {\n await log(result);\n}", + "callback" + ), + # Dollar sign in identifiers + ( + "function $apply(scope) {\n scope.$digest();\n}", + "$apply" + ), + ( + "const $timeout = (fn, delay) => {\n setTimeout(fn, delay);\n}", + "$timeout" + ), + ( + "function not$(value) {\n return !value;\n}", + "not$" + ), + ( + "const in$ = (key, obj) => key in obj;", + "in$" + ), + # String-keyed object property + ( + '"content-type"(value) {\n return value.toLowerCase();\n}', + "content-type" + ), + ( + "'set-cookie': (value) => {\n return parse(value);\n}", + "set-cookie" + ), + # Static getter/setter with computed property + ( + "static [Symbol.species]() {\n return Array;\n}", + "Symbol.species" + ), + ( + "static async [Symbol.asyncIterator]() {\n yield 1;\n}", + "Symbol.asyncIterator" + ), + ( + "get [Symbol.toStringTag]() {\n return 'Custom';\n}", + "Symbol.toStringTag" + ), + # TypeScript generics in function signatures + ( + "function assert(value: T): asserts value {\n if (!value) throw new Error();\n}", + "assert" + ), + ( + "function identity(arg: T): T {\n return arg;\n}", + "identity" + ), + # Constructor with destructured default params + ( + "constructor({name = 'default', age = 0} = {}) {\n this.name = name;\n}//(class: Config)", + "constructor" + ), + # Wrapper call pattern (const name = wrapper(...)) + ( + "const memoized = memoize(computeValue)", + "memoized" + ), + ( + "const debounced = debounce(handleInput, 300)", + "debounced" + ), +]) +def test_get_function_name_new_patterns(parser, code, expected_name): + """Test get_function_name with generator and TypeScript patterns.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + result = parser.get_function_name(doc) + assert result == expected_name + + +@pytest.mark.parametrize("code", [ + # TypeScript interface declarations + "interface UserConfig {\n name: string;\n age: number;\n}", + "export interface ApiResponse {\n data: any;\n status: number;\n}", + "declare interface Window {\n customProp: string;\n}", + # TypeScript enum declarations + "enum Color {\n Red,\n Green,\n Blue\n}", + "export enum Direction {\n Up,\n Down,\n Left,\n Right\n}", + "const enum HttpStatus {\n OK = 200,\n NotFound = 404\n}", + "export const enum LogLevel {\n Debug,\n Info,\n Error\n}", + "declare enum Platform {\n Web,\n Mobile\n}", +]) +def test_is_function_rejects_typescript_constructs(parser, code): + """Test that is_function returns False for TypeScript interface/enum declarations.""" + doc = Document( + page_content=code, + metadata={'source': 'test.ts', 'content_type': 'functions_classes'} + ) + assert parser.is_function(doc) is False + + +@pytest.mark.parametrize("code", [ + # Class declarations should raise ValueError + "class MyClass {\n constructor() {}\n}", + "export class ExportedClass {\n method() {}\n}", + "class Calculator {\n add(a, b) {\n return a + b;\n }\n}", + " class IndentedClass {\n getValue() {}\n}", +]) +def test_get_function_name_raises_on_class_declaration(parser, code): + """Test that get_function_name raises ValueError for class declarations.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + with pytest.raises(ValueError, match="Only function document is supported"): + parser.get_function_name(doc) + + +@pytest.mark.parametrize("code", [ + # Code that doesn't match any function pattern - returns empty string + "const x = 5;", + "let variable = 'string';", + "// Just a comment", + "{ key: 'value' }", + "/* Multi-line\n comment */", +]) +def test_get_function_name_returns_empty_string(parser, code): + """Test that get_function_name returns empty string for non-function code. + + When no function pattern matches, the method logs a warning and returns ''(empty string). + """ + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + result = parser.get_function_name(doc) + assert result == '' + + +@pytest.mark.parametrize("source_path,expected_packages", [ + # Application code (no node_modules) - returns root_project + ( + "/app/src/utils.js", + ["root_project"] + ), + # Regular third-party package + ( + "/app/node_modules/lodash/template.js", + ["lodash"] + ), + # Scoped package (starts with @) + ( + "/app/node_modules/@babel/core/lib/index.js", + ["@babel/core"] + ), + # Another regular package + ( + "/app/node_modules/express/lib/router.js", + ["express"] + ), + # Another application code path + ( + "/home/user/project/controllers/user.js", + ["root_project"] + ), + ( + "/app/node_modules/package-a/node_modules/package-b/index.js", + ["package-b"], + ), + # Nested scoped packages + ( + "/app/node_modules/@angular/common/node_modules/@angular/core/index.js", + ["@angular/core"], + ), +]) +def test_get_package_names(parser, source_path, expected_packages): + doc = Document( + page_content="function test() {}", + metadata={'source': source_path, 'content_type': 'functions_classes'} + ) + result = parser.get_package_names(doc) + assert result == expected_packages + + + +# ============================================================================ +# Tests for _get_function_calls method +# ============================================================================ + +@pytest.mark.parametrize("caller_code,callee_name,expected_calls", [ + # Direct function calls + ( + "function myFunc() {\n template();\n}", + "template", + ["template"] + ), + # Multiple calls to the same function + ( + "function process() {\n parse();\n parse();\n parse();\n}", + "parse", + ["parse"] + ), + # Object method calls + ( + "function handler() {\n lodash.template();\n}", + "template", + ["lodash.template"] + ), + # Nested property access + ( + "function run() {\n obj.nested.func();\n}", + "func", + ["obj.nested.func"] + ), + # Multiple different call patterns + ( + "function complex() {\n template();\n _.template();\n utils.template();\n}", + "template", + ["template", "_.template", "utils.template"] + ), + # Function call with arguments + ( + "function withArgs() {\n compile('test', 42);\n}", + "compile", + ["compile"] + ), + # Function call with complex arguments + ( + "function nested() {\n process(getData(), getConfig());\n}", + "process", + ["process"] + ), + # Call with newlines and whitespace + ( + "function spaces() {\n execute \n ();\n}", + "execute", + ["execute"] + ), + # No calls to the target function + ( + "function other() {\n differentFunc();\n}", + "targetFunc", + [] + ), + # Function name as part of longer identifier + ( + "function test() {\n template();\n mytemplate();\n templateFunc();\n}", + "template", + ["template"] + ), + # Chained method calls + ( + "function chain() {\n obj.method1().method2();\n}", + "method2", + ["obj.method1().method2"] + ), + # Arrow function with call + ( + "const arrow = () => {\n helper();\n}", + "helper", + ["helper"] + ), + # Async function with await + ( + "async function asyncFunc() {\n await fetchData();\n}", + "fetchData", + ["fetchData"] + ), + # Call in conditional + ( + "function conditional() {\n if (condition) {\n action();\n }\n}", + "action", + ["action"] + ), + # Call in loop + ( + "function loop() {\n for (let i = 0; i < 10; i++) {\n process();\n }\n}", + "process", + ["process"] + ), + # Call with callback + ( + "function withCallback() {\n execute(function() {\n callback();\n });\n}", + "execute", + ["execute"] + ), + # Multiple objects calling same method + ( + "function multi() {\n obj1.save();\n obj2.save();\n}", + "save", + ["obj1.save", "obj2.save"] + ), + # Special characters in object name (underscore, dollar sign) + ( + "function special() {\n _lodash.template();\n $jquery.ajax();\n}", + "template", + ["_lodash.template"] + ), + # No whitespace before parenthesis + ( + "function noSpace() {\n func();\n}", + "func", + ["func"] + ), + # Call as part of expression + ( + "function expr() {\n const result = calculate() + 10;\n}", + "calculate", + ["calculate"] + ), + # Empty function body + ( + "function empty() {}", + "anyFunc", + [], + ), + # Function calls in comments + ( + """function test() { + // execute(); + /* execute(); */ + actualCall(); + }""", + "execute", + [], # The regex will match both commented calls + ), + # Function names in string literals + ( + """function test() { + const str = "execute()"; + execute(); + }""", + "execute", + ["execute"], # Will find both in string and actual call + ), + # Special regex characters in function names + ( + """function test() { + $jquery(); + _underscore(); + }""", + "$jquery", + ["$jquery"], + ), + ( + """function test() { + $jquery(); + _underscore(); + }""", + "_underscore", + ["_underscore"], + ), + # Case sensitivity + ( + """function test() { + myFunc(); + MyFunc(); + MYFUNC(); + }""", + "myFunc", + ["myFunc"], + ), + ( + """function test() { + myFunc(); + MyFunc(); + MYFUNC(); + }""", + "MyFunc", + ["MyFunc"], + ), + # Constructor calls with 'new' + ( + """function create() { + const obj = new MyClass(); + return obj; + }""", + "MyClass", + ["MyClass"], + ), + # Deeply nested property access + ( + """function deepNesting() { + app.services.user.auth.validate(); + }""", + "validate", + ["app.services.user.auth.validate"], + ), + # ======================================================================== + # Callback references without parentheses + # ======================================================================== + # setTimeout with direct function reference + ( + """function test() { + setTimeout(add, 1000, 10, 15); + }""", + "add", + ["add"], + ), + # setTimeout with qualified function reference + ( + """function test() { + setTimeout(math.add, 1000, 7, 8); + }""", + "add", + ["math.add"], + ), + # Array forEach with direct function reference + ( + """function test() { + [1,2,3].forEach(add); + }""", + "add", + ["add"], + ), + # Array forEach with qualified function reference + ( + """function test() { + [1,2,3].forEach(math.add); + }""", + "add", + ["math.add"], + ), + # addEventListener with qualified function reference + ( + """function test() { + document.getElementById('btn').addEventListener('click', math.add); + }""", + "add", + ["math.add"], + ), + # Array map with callback + ( + """function test() { + const result = items.map(transform); + }""", + "transform", + ["transform"], + ), + # Array filter with callback + ( + """function test() { + const filtered = data.filter(validator.check); + }""", + "check", + ["validator.check"], + ), + # Promise then/catch with callbacks + ( + """function test() { + promise.then(handleSuccess).catch(handleError); + }""", + "handleSuccess", + ["handleSuccess"], + ), + ( + """function test() { + promise.then(handleSuccess).catch(handleError); + }""", + "handleError", + ["handleError"], + ), + # Mixed: both direct call and callback reference + ( + """function test() { + add(1, 2); + setTimeout(add, 1000); + }""", + "add", + ["add"], + ), + # Mixed: qualified call and qualified callback + ( + """function test() { + math.add(5, 10); + [1,2,3].forEach(math.add); + }""", + "add", + ["math.add"], + ), + # Callback in nested call + ( + """function test() { + Promise.all(items.map(process)); + }""", + "process", + ["process"], + ), + # Multiple callbacks in same line + ( + """function test() { + promise.then(success, failure); + }""", + "success", + ["success"], + ), + ( + """function test() { + promise.then(success, failure); + }""", + "failure", + ["failure"], + ), + # Callback with whitespace variations + ( + """function test() { + setTimeout( add , 1000 ); + }""", + "add", + ["add"], + ), + # Should NOT match: function name in string + ( + """function test() { + setTimeout("add", 1000); + }""", + "add", + [], + ), + # Should NOT match: function name as object key + ( + """function test() { + const obj = {add: someFunc}; + }""", + "add", + [], + ), + # Callback with deeply nested qualifier + ( + """function test() { + items.forEach(utils.math.operations.add); + }""", + "add", + ["utils.math.operations.add"], + ), + # setInterval with callback + ( + """function test() { + setInterval(tick, 1000); + }""", + "tick", + ["tick"], + ), + # requestAnimationFrame with callback + ( + """function test() { + requestAnimationFrame(render); + }""", + "render", + ["render"], + ), + # Array method chaining with callbacks + ( + """function test() { + items.map(transform).filter(validate).forEach(process); + }""", + "transform", + ["transform"], + ), + ( + """function test() { + items.map(transform).filter(validate).forEach(process); + }""", + "validate", + ["validate"], + ), + ( + """function test() { + items.map(transform).filter(validate).forEach(process); + }""", + "process", + ["process"], + ), +]) +def test_get_function_calls_various_patterns(parser, caller_code, callee_name, expected_calls): + caller_doc = Document( + page_content=caller_code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + result = parser._get_function_calls(caller_doc, callee_name) + assert sorted(result) == sorted(expected_calls) + + +def test_get_function_calls_complex_real_world_example(parser): + """Test _get_function_calls with realistic complex code.""" + caller_code = """async function processUserData(userId) { + const user = await fetchUser(userId); + const profile = await fetchProfile(user.profileId); + + if (user.isActive) { + logger.log('Processing active user'); + const settings = await fetchSettings(user.id); + validateSettings(settings); + } + + return { + user: user, + profile: profile + }; +}""" + caller_doc = Document( + page_content=caller_code, + metadata={'source': 'users.js', 'content_type': 'functions_classes'} + ) + + # Test finding different function calls + assert parser._get_function_calls(caller_doc, "fetchUser") == ["fetchUser"] + assert parser._get_function_calls(caller_doc, "fetchProfile") == ["fetchProfile"] + assert parser._get_function_calls(caller_doc, "fetchSettings") == ["fetchSettings"] + assert parser._get_function_calls(caller_doc, "validateSettings") == ["validateSettings"] + assert parser._get_function_calls(caller_doc, "log") == ["logger.log"] + + +@pytest.mark.parametrize("func_content,source,expected_key,expected_vars,description", [ + # Regular function parameters and variables + ( + """function processData(id, name, options) { + const userId = id; + let result = null; + var temp = 'test'; + return userId; +}""", + "app/utils.js", + "processData@app/utils.js", + { + 'id': {"value": "parameter", "type": ""}, + 'name': {"value": "parameter", "type": ""}, + 'options': {"value": "parameter", "type": ""}, + 'userId': {"value": "id", "type": ""}, + 'result': {"value": "null", "type": ""}, + 'temp': {"value": "'test'", "type": ""}, + 'return_types': [] + }, + "Regular function with parameters and various variable declarations" + ), + ( + """function processData(id, + name, + options) { + const userId = id; + let result = null; + var temp = 'test'; + return userId; + }""", + "app/utils.js", + "processData@app/utils.js", + { + 'id': {"value": "parameter", "type": ""}, + 'name': {"value": "parameter", "type": ""}, + 'options': {"value": "parameter", "type": ""}, + 'userId': {"value": "id", "type": ""}, + 'result': {"value": "null", "type": ""}, + 'temp': {"value": "'test'", "type": ""}, + 'return_types': [] + }, + "Regular function with parameters in few lines and various variable declarations" + ), + # Arrow function + ( + """const calculate = (a, b, c) => { + const sum = a + b + c; + let multiplier = 2; + return sum * multiplier; +}""", + "app/math.js", + "calculate@app/math.js", + { + 'a': {"value": "parameter", "type": ""}, + 'b': {"value": "parameter", "type": ""}, + 'c': {"value": "parameter", "type": ""}, + 'sum': {"value": "a + b + c", "type": ""}, + 'multiplier': {"value": "2", "type": ""}, + 'return_types': [] + }, + "Arrow function with parameters and local variables" + ), + # Rest parameters + ( + """async function processItems(baseUrl, ...endpoints) { + const results = []; + let count = 0; + return results; +}""", + "app/api.js", + "processItems@app/api.js", + { + 'baseUrl': {"value": "parameter", "type": ""}, + 'endpoints': {"value": "parameter", "type": "Array"}, + 'results': {"value": "[]", "type": ""}, + 'count': {"value": "0", "type": ""}, + 'return_types': [] + }, + "Function with rest parameters (type Array)" + ), + # Object destructuring parameters + ( + """const handleRequest = ({method, url, headers}) => { + const response = fetch(url); + let status = 200; + return response; +}""", + "app/request.js", + "handleRequest@app/request.js", + { + 'method': {"value": "parameter", "type": ""}, + 'url': {"value": "parameter", "type": ""}, + 'headers': {"value": "parameter", "type": ""}, + 'response': {"value": "fetch(url)", "type": ""}, + 'status': {"value": "200", "type": ""}, + 'return_types': [] + }, + "Function with object destructuring parameters" + ), + # Array destructuring parameters + ( + """function processCoordinates([x, y, z]) { + const distance = Math.sqrt(x*x + y*y + z*z); + return distance; +}""", + "app/math.js", + "processCoordinates@app/math.js", + { + 'x': {"value": "parameter", "type": ""}, + 'y': {"value": "parameter", "type": ""}, + 'z': {"value": "parameter", "type": ""}, + 'distance': {"value": "Math.sqrt(x*x + y*y + z*z)", "type": ""}, + 'return_types': [] + }, + "Function with array destructuring parameters" + ), + # Default parameters + ( + """function fetchData(url, timeout = 5000, retries = 3) { + const config = { url, timeout, retries }; + return config; +}""", + "app/fetch.js", + "fetchData@app/fetch.js", + { + 'url': {"value": "parameter", "type": ""}, + 'timeout': {"value": "5000", "type": ""}, + 'retries': {"value": "3", "type": ""}, + 'config': {"value": "{ url, timeout, retries }", "type": ""}, + 'return_types': [] + }, + "Function with default parameters" + ), + # Multiple declarations on one line + ( + """function multiVars() { + const x = 1, y = 2, z = 3; + let a, b, c; + var m = 'm', n = 'n'; + return x + y; +}""", + "app/multi.js", + "multiVars@app/multi.js", + { + 'x': {"value": "1", "type": ""}, + 'y': {"value": "2", "type": ""}, + 'z': {"value": "3", "type": ""}, + 'a': {"value": "", "type": ""}, + 'b': {"value": "", "type": ""}, + 'c': {"value": "", "type": ""}, + 'm': {"value": "'m'", "type": ""}, + 'n': {"value": "'n'", "type": ""}, + 'return_types': [] + }, + "Function with multiple variable declarations on one line" + ), + # Class method/constructor + ( + """constructor(name, age) { + const greeting = 'Hello'; + this.name = name; + this.age = age; +}//(class: Person)""", + "app/Person.js", + "constructor@app/Person.js", + { + 'name': {"value": "parameter", "type": ""}, + 'age': {"value": "parameter", "type": ""}, + 'greeting': {"value": "'Hello'", "type": ""}, + 'this': {"value": "Person()", "type": "Person"}, + 'return_types': [] + }, + "Class constructor with 'this' reference" + ), + # Object destructuring local variables + ( + """function processUser(user) { + const {name, age, email} = user; + let status = 'active'; + return name; +}""", + "app/user.js", + "processUser@app/user.js", + { + 'user': {"value": "parameter", "type": ""}, + 'name': {"value": "user", "type": ""}, + 'age': {"value": "user", "type": ""}, + 'email': {"value": "user", "type": ""}, + 'status': {"value": "'active'", "type": ""}, + 'return_types': [] + }, + "Function with object destructuring in local variables" + ), + # Array destructuring local variables + ( + """function processArray(data) { + const [first, second, third] = data; + return first + second; +}""", + "app/array.js", + "processArray@app/array.js", + { + 'data': {"value": "parameter", "type": ""}, + 'first': {"value": "data", "type": ""}, + 'second': {"value": "data", "type": ""}, + 'third': {"value": "data", "type": ""}, + 'return_types': [] + }, + "Function with array destructuring in local variables" + ), + # Empty function + ( + """function emptyFunc() { + return 42; +}""", + "app/empty.js", + "emptyFunc@app/empty.js", + { + 'return_types': [] + }, + "Function with no variables" + ), + # Complex values + ( + """function complex() { + const obj = { key: 'value', nested: { deep: true } }; + const arr = [1, 2, 3, 4, 5]; + const func_call = fetch('/api/data'); + const chain = data.map(x => x * 2).filter(x => x > 10); + return obj; +}""", + "app/complex.js", + "complex@app/complex.js", + { + 'obj': {"value": "{ key: 'value', nested: { deep: true } }", "type": ""}, + 'arr': {"value": "[1, 2, 3, 4, 5]", "type": ""}, + 'func_call': {"value": "fetch('/api/data')", "type": ""}, + 'chain': {"value": "data.map(x => x * 2).filter(x => x > 10)", "type": ""}, + 'return_types': [] + }, + "Function with complex variable values" + ), + # Renamed destructuring + ( + """function renamed({oldName: newName, id: userId}) { + const value = newName + userId; + return value; +}""", + "app/rename.js", + "renamed@app/rename.js", + { + 'newName': {"value": "parameter", "type": ""}, + 'userId': {"value": "parameter", "type": ""}, + 'value': {"value": "newName + userId", "type": ""}, + 'return_types': [] + }, + "Function with renamed object destructuring" + ), + # declaration split to multiple lines + ( + """function func() { + const fruits = ["Apple", + "Banana", + "Cherry", + "Date" + ]; + return value; + }""", + "app/func.js", + "func@app/func.js", + { + 'fruits': {"value": '["Apple", "Banana", "Cherry", "Date" ]', "type": ""}, + 'return_types': [] + }, + "Function with renamed object destructuring" + ), +]) +def test_create_map_of_local_vars_single_function(parser, func_content, source, expected_key, expected_vars, description): + """Test create_map_of_local_vars with various single function scenarios.""" + func_doc = Document( + page_content=func_content, + metadata={'source': source, 'content_type': 'functions_classes'} + ) + + result = parser.create_map_of_local_vars([func_doc]) + + assert expected_key in result + vars_map = result[expected_key] + + for var_name, expected_value in expected_vars.items(): + assert vars_map[var_name] == expected_value, f"Variable '{var_name}' mismatch" + + +def test_create_map_of_local_vars_multiple_functions(parser): + """Test create_map_of_local_vars with multiple functions.""" + func_docs = [ + Document( + page_content="""function funcA(x) { + const result = x * 2; + return result; +}""", + metadata={'source': 'app/a.js', 'content_type': 'functions_classes'} + ), + Document( + page_content="""function funcB(y) { + let temp = y + 1; + return temp; +}""", + metadata={'source': 'app/b.js', 'content_type': 'functions_classes'} + ) + ] + + result = parser.create_map_of_local_vars(func_docs) + + # Check both functions are in the result + assert 'funcA@app/a.js' in result + assert 'funcB@app/b.js' in result + + # Check funcA + assert result['funcA@app/a.js']['x'] == {"value": "parameter", "type": ""} + assert result['funcA@app/a.js']['result'] == {"value": "x * 2", "type": ""} + + # Check funcB + assert result['funcB@app/b.js']['y'] == {"value": "parameter", "type": ""} + assert result['funcB@app/b.js']['temp'] == {"value": "y + 1", "type": ""} + + +@pytest.mark.parametrize("class_code,fields_to_include,fields_to_exclude", [ + # Test: Local variables in methods are not included as class fields + ( + """ +class UserService { + constructor() { + this.processor = new DataProcessor(); + this.name = 'service'; + } + + findUser(username) { + const localVar = 'local'; + let anotherLocal = 123; + return this.processor.executeQuery(username); + } + + checkUser(username) { + var temp = 'temporary'; + return this.processor.validateInput(username); + } +} +""", + ['processor', 'name'], + ['localVar', 'anotherLocal', 'temp', 'username'] + ), + # Test: Class field declarations (ES2022+) + ( + """ +class DataProcessor { + apiKey = 'default-key'; + #privateField = 'private'; + counter = 0; + + constructor(config) { + this.config = config; + const tempVar = 'temp'; + } + + process(data) { + const result = this.apiKey + data; + return result; + } +} +""", + ['apiKey', 'privateField', 'counter', 'config'], + ['tempVar', 'result', 'data'] + ), + # Test: Async methods, static methods, and getters/setters + ( + """ +class ComplexClass { + status = 'active'; + + constructor() { + this.id = 123; + } + + async fetchData() { + const response = await fetch('/api'); + return response; + } + + static create() { + const instance = new ComplexClass(); + return instance; + } + + get value() { + const computed = this.id * 2; + return computed; + } +} +""", + ['status', 'id'], + ['response', 'instance', 'computed'] + ), + # Test: this.field assignments in non-constructor methods are excluded + ( + """ +class StateManager { + constructor() { + this.initialState = {}; + } + + setState(newState) { + // This should NOT be extracted as a field + this.currentState = newState; + const temp = 'local'; + } + + updateValue(val) { + this.value = val; // Not in constructor, should be excluded + } +} +""", + ['initialState'], + ['currentState', 'value', 'temp'] + ), + # Test: Constructor with nested braces (if statements, loops) + ( + """ +class ComplexConstructor { + constructor(config) { + this.config = config; + + if (config.enabled) { + const tempEnabled = true; + this.status = 'active'; + } + + for (let i = 0; i < 10; i++) { + const loopVar = i; + } + } +} +""", + ['config', 'status'], + ['tempEnabled', 'loopVar', 'i'] + ), + # Test: Class with both class field declarations and constructor assignments + ( + """ +class MixedFields { + // Class field declarations + publicField = 'public'; + anotherField = 42; + + constructor(param1, param2) { + // Constructor assignments + this.param1 = param1; + this.param2 = param2; + + // Local variables (should be excluded) + const localInConstructor = 'local'; + let tempValue = 123; + } + + method1() { + const methodLocal = 'should not appear'; + return this.publicField; + } +} +""", + ['publicField', 'anotherField', 'param1', 'param2'], + ['localInConstructor', 'tempValue', 'methodLocal'] + ), + ( + """ + class EmptyClass { + doSomething() { + console.log('test'); + } + } + """, + [], + [] + ), +]) +def test_extract_fields(parser, class_code, fields_to_include, fields_to_exclude): + """Test _extract_class_fields with various class structures. + + Verifies that: + - Class field declarations are extracted + - Constructor this.field assignments are extracted + - Local variables in methods are NOT extracted + - Method parameters are NOT extracted + - this.field assignments in non-constructor methods are NOT extracted + """ + fields = parser._extract_class_fields(class_code) + field_names = [field[0] for field in fields] + + assert len(fields) == len(fields_to_include) + for field in fields_to_include: + assert field in field_names + + for field in fields_to_exclude: + assert field not in field_names + + +# def test_extract_fields_empty_class(parser): +# """Test with a class that has no fields.""" +# class_code = """ +# class EmptyClass { +# doSomething() { +# console.log('test'); +# } +# } +# """ +# +# fields = parser._extract_class_fields(class_code) +# +# # Should return empty list +# assert fields == [] + + +def test_extract_fields_all_field_types_return_any(parser): + """Test that all extracted fields have type 'any'.""" + class_code = """ +class TypeTest { + field1 = 'string'; + field2 = 123; + + constructor() { + this.field3 = true; + } +} +""" + + fields = parser._extract_class_fields(class_code) + + # All fields should have type 'any' + for field_name, field_type in fields: + assert field_type == 'any', f"Field '{field_name}' should have type 'any', got '{field_type}'" + + # Should have 3 fields + assert len(fields) == 3 + +@pytest.mark.parametrize("description,function_content,file_content,expected", [ + ( + "Constructor should be detected as exported when class is exported via module.exports = ClassName", + "constructor() {\n this.name = 'PackageURL';\n}\n//(class: PackageURL)", + """class PackageURL { + constructor() { + this.name = 'PackageURL'; + } + + static fromString(str) { + return new PackageURL(); + } + } + + module.exports = PackageURL; + """, + True, + ), + ( + "Class method should be detected as exported when class is exported via module.exports = ClassName", + "static fromString(str) {\n return new PackageURL();\n}\n//(class: PackageURL)", + """class PackageURL { + constructor() { + this.name = 'PackageURL'; + } + + static fromString(str) { + return new PackageURL(); + } + } + + module.exports = PackageURL; + """, + True, + ), + ( + "Class method should be detected as exported when class is in module.exports object", + "executeQuery(userInput) {\n const query = `SELECT * FROM users WHERE name = '${userInput}'`;\n return query;\n}\n//(class: DataProcessor)", + """class DataProcessor { + executeQuery(userInput) { + const query = `SELECT * FROM users WHERE name = '${userInput}'`; + return query; + } + } + + module.exports = { DataProcessor }; + """, + True, + ), + ( + "Class method should be detected as exported when class is exported via export class", + "process(data) {\n return data.toUpperCase();\n}\n//(class: Processor)", + """export class Processor { + process(data) { + return data.toUpperCase(); + } + } + """, + True, + ), + ( + "Class method should be detected as exported when class is exported via export default", + "save(item) {\n this.items.push(item);\n}\n//(class: Storage)", + """class Storage { + constructor() { + this.items = []; + } + + save(item) { + this.items.push(item); + } + } + + export default Storage; + """, + True, + ), + ( + "Function should be detected as exported", + "function myFunction() {\n return 'test';\n}", + """function myFunction() { + return 'test'; + } + + module.exports = myFunction; + """, + True, + ), + ( + "Function should be detected as exported when in module.exports object", + "function helper() {\n return 'help';\n}", + """function helper() { + return 'help'; + } + + function other() { + return 'other'; + } + + module.exports = { helper, other }; + """, + True, + ), + ( + "Class method should NOT be detected as exported when class is not exported", + "privateMethod() {\n return 'private';\n}\n//(class: InternalClass)", + """class InternalClass { + privateMethod() { + return 'private'; + } + } + + // No exports + """, + False, + ), + ( + "Function should NOT be detected as exported when not exported", + "function internalFunc() {\n return 'internal';\n}", + """function internalFunc() { + return 'internal'; + } + + // No exports + """, + False, + ), +]) +def test_is_exported_function(parser, description, function_content, file_content, expected): + source = "node_modules/packageurl-js/index.js" + func_doc = Document( + page_content=function_content, + metadata={'source': source, 'content_type': 'functions_classes'} + ) + full_file_doc = Document( + page_content=file_content, + metadata={'source': source, 'content_type': 'simplified_code'} + ) + documents_of_full_sources = { + source: full_file_doc + } + + # Test + result = parser.is_exported_function(func_doc, documents_of_full_sources) + assert result == expected, description + + +@pytest.mark.parametrize("description,code,expected", [ + ( + "Simple function with single line parameters", + """function add (a, b) { +a = 2; +b = 3; +// a+b +c = a+b; +}""", + [ + 'function add (a, b) {', + 'a = 2;', + 'b = 3;', + '// a+b', + 'c = a+b;', + '}' + ] + ), + ( + "Function with multiline parameters", + """function add (a, +b) { +a = 2; +b = 3; +c = a+b; +}""", + [ + 'function add (a, b) {', + 'a = 2;', + 'b = 3;', + 'c = a+b;', + '}' + ] + ), + ( + "Function with multiline parameters and extra spaces", + """function processData( + id, + name, + options +) { +const userId = id; +let result = null; +return userId; +}""", + [ + 'function processData( id, name, options ) {', + 'const userId = id;', + 'let result = null;', + 'return userId;', + '}' + ] + ), + ( + "Arrow function with multiline parameters", + """const calculate = ( + a, + b, + c +) => { +const sum = a + b + c; +return sum; +}""", + [ + 'const calculate = ( a, b, c ) => {', + 'const sum = a + b + c;', + 'return sum;', + '}' + ] + ), + ( + "Async function with multiline parameters", + """async function fetchData( + url, + timeout, + retries +) { +const response = await fetch(url); +return response; +}""", + [ + 'async function fetchData( url, timeout, retries ) {', + 'const response = await fetch(url);', + 'return response;', + '}' + ] + ), + ( + "Function with nested parentheses in default parameters", + """function test(a = (1 + 2), b) { +const x = a; +return x; +}""", + [ + 'function test(a = (1 + 2), b) {', + 'const x = a;', + 'return x;', + '}' + ] + ), + ( + "Multiple functions with multiline parameters", + """function first( + x, + y +) { +return x + y; +} +function second(a, b) { +return a * b; +}""", + [ + 'function first( x, y ) {', + 'return x + y;', + '}', + 'function second(a, b) {', + 'return a * b;', + '}' + ] + ), + ( + "Function with string containing semicolons", + """function test() { +const msg = "Hello; World"; +console.log(msg); +}""", + [ + 'function test() {', + 'const msg = "Hello; World";', + 'console.log(msg);', + '}' + ] + ), + ( + "Function with block comment", + """function test() { +/* This is a comment */ +const x = 1; +return x; +}""", + [ + 'function test() {', + '/* This is a comment */', + 'const x = 1;', + 'return x;', + '}' + ] + ), + ( + "Function with inline comment containing semicolons", + """function test() { +a = 1; +// This; is; a; comment +b = 2; +}""", + [ + 'function test() {', + 'a = 1;', + '// This; is; a; comment', + 'b = 2;', + '}' + ] + ), + ( + "Class with multiline constructor parameters", + """class MyClass { +constructor( + name, + age +) { +this.name = name; +this.age = age; +} +}""", + [ + 'class MyClass {', + 'constructor( name, age ) {', + 'this.name = name;', + 'this.age = age;', + '}', + '}' + ] + ), + ( + "Object destructuring with multiline", + """const handleRequest = ({ + method, + url, + headers +}) => { +const response = fetch(url); +return response; +}""", + [ + 'const handleRequest = ({ method, url, headers }) => {', + 'const response = fetch(url);', + 'return response;', + '}' + ] + ), + ( + "Multi-line block comment on its own", + """function test() { +/* comment with +few lines*/ +const x = 1; +}""", + [ + 'function test() {', + '/* comment with few lines*/', + 'const x = 1;', + '}' + ] + ), + ( + "Multi-line block comment on its own", + """function test() { + /* comment with + few lines*/ const x = 1; + }""", + [ + 'function test() {', + '/* comment with few lines*/', + 'const x = 1;', + '}' + ] + ), +]) +def test_split_by_semicolon(parser, description, code, expected): + """Test the _split_by_semicolon function with various code structures.""" + result = parser._split_into_statements(code) + assert result == expected, f"{description}: expected {expected}, got {result}" + + +@pytest.mark.parametrize("child_class,parent_class,code_documents_content,expected,description", [ + ( + "CustomHandler", + "VulnerableBase", + {"src/handlers/CustomHandler.js": "class CustomHandler extends VulnerableBase {\n constructor() { super(); }\n}"}, + True, + "Simple direct inheritance: class X extends Y" + ), + ( + "CustomHandler", + "VulnerableBase", + {"src/handlers/CustomHandler.js": "class CustomHandler extends LoggerMixin(VulnerableBase) {\n constructor() { super(); }\n}"}, + True, + "Inheritance with mixin: class X extends Mixin(Y)" + ), + ( + "CustomHandler", + "VulnerableBase", + {"src/handlers/CustomHandler.js": "class CustomHandler extends Mixin1(Mixin2(VulnerableBase)) {\n constructor() { super(); }\n}"}, + True, + "Chained mixins: class X extends Mixin1(Mixin2(Y))" + ), + ( + "MyClass", + "BaseClass", + {"src/MyClass.js": "class MyClass extends A(B(C(BaseClass))) {\n}"}, + True, + "Triple chained mixins: class X extends A(B(C(Y)))" + ), + ( + "CustomHandler", + "VulnerableBase", + {"src/handlers/CustomHandler.js": "class CustomHandler extends OtherClass {\n constructor() { super(); }\n}"}, + False, + "No match: child extends different parent" + ), + ( + "CustomHandler", + "VulnerableBase", + {"src/handlers/CustomHandler.js": "class CustomHandler {\n constructor() {}\n}"}, + False, + "No match: class without extends" + ), + ( + "NonExistentClass", + "VulnerableBase", + {"src/handlers/CustomHandler.js": "class CustomHandler extends VulnerableBase {\n}"}, + False, + "No match: child class not found in documents" + ), + ( + "MyHandler", + "Base", + {"src/MyHandler.js": "class MyHandler extends BaseExtended {\n}"}, + False, + "Word boundary: Base should not match BaseExtended" + ), + ( + "ChildClass", + "ParentClass", + { + "src/other.js": "class Other extends Unrelated {\n}", + "src/child.js": "class ChildClass extends ParentClass {\n}" + }, + True, + "Multiple files: inheritance found in second file" + ), + ( + "MyService", + "BaseService", + {"src/MyService.js": "export class MyService extends BaseService {\n doSomething() {}\n}"}, + True, + "Export class with inheritance" + ), + ( + "", + "VulnerableBase", + {"src/file.js": "class Handler extends VulnerableBase {}"}, + False, + "Empty child class name" + ), + ( + "CustomHandler", + "", + {"src/file.js": "class CustomHandler extends VulnerableBase {}"}, + False, + "Empty parent class name" + ), + ( + "Handler", + "EventEmitter", + {"src/Handler.js": "class Handler extends EventEmitter(Object) {\n}"}, + True, + "Mixin with parent at the start of the chain" + ), + ( + "GrandChild", + "GrandParent", + { + "src/GrandChild.js": "class GrandChild extends Child {\n}", + "src/Child.js": "class Child extends GrandParent {\n}", + "src/GrandParent.js": "class GrandParent {\n}" + }, + True, + "2-level transitive: GrandChild extends Child extends GrandParent" + ), + ( + "A", + "D", + { + "src/A.js": "class A extends B {\n}", + "src/B.js": "class B extends C {\n}", + "src/C.js": "class C extends D {\n}", + "src/D.js": "class D {\n}" + }, + True, + "3-level transitive: A extends B extends C extends D" + ), + ( + "MyHandler", + "BaseClass", + { + "src/MyHandler.js": "class MyHandler extends Mixin(MiddleClass) {\n}", + "src/MiddleClass.js": "class MiddleClass extends BaseClass {\n}" + }, + True, + "Transitive with mixin: MyHandler extends Mixin(MiddleClass), MiddleClass extends BaseClass" + ), + ( + "A", + "C", + { + "src/A.js": "class A extends B {\n}", + "src/B.js": "class B extends A {\n}" + }, + False, + "Circular reference: A extends B extends A - should not infinite loop" + ), + ( + "Child", + "Unrelated", + { + "src/Child.js": "class Child extends Parent {\n}", + "src/Parent.js": "class Parent extends GrandParent {\n}", + "src/Unrelated.js": "class Unrelated {\n}" + }, + False, + "Transitive non-match: Child is not a subclass of Unrelated" + ), + ( + "Child", + "Parent", + {"src/legacy.js": "function Child() {}\nfunction Parent() {}\nutil.inherits(Child, Parent);"}, + True, + "util.inherits(Child, Parent)" + ), + ( + "MyClass", + "BaseClass", + {"src/legacy.js": "function MyClass() {}\nfunction BaseClass() {}\ninherits(MyClass, BaseClass);"}, + True, + "inherits(Child, Parent) - standalone inherits package" + ), + ( + "Derived", + "Base", + {"src/proto.js": "function Derived() {}\nfunction Base() {}\nDerived.prototype = Object.create(Base.prototype);"}, + True, + "Child.prototype = Object.create(Parent.prototype)" + ), + ( + "Sub", + "Super", + {"src/proto.js": "function Sub() {}\nfunction Super() {}\nObject.setPrototypeOf(Sub.prototype, Super.prototype);"}, + True, + "Object.setPrototypeOf(Child.prototype, Parent.prototype)" + ), + ( + "Child", + "WrongParent", + {"src/legacy.js": "function Child() {}\nfunction Parent() {}\nutil.inherits(Child, Parent);"}, + False, + "util.inherits - non-matching parent" + ), + ( + "GrandChild", + "GrandParent", + { + "src/legacy.js": """function GrandChild() {} +function Child() {} +function GrandParent() {} +util.inherits(GrandChild, Child); +util.inherits(Child, GrandParent);""" + }, + True, + "Transitive util.inherits: GrandChild -> Child -> GrandParent" + ), + ( + "ModernClass", + "LegacyBase", + { + "src/modern.js": "class ModernClass extends LegacyWrapper {\n}", + "src/legacy.js": "function LegacyWrapper() {}\nfunction LegacyBase() {}\nutil.inherits(LegacyWrapper, LegacyBase);" + }, + True, + "Mixed: ES6 class extends prototype-based class" + ), +]) +def test_is_subclass_of(parser, child_class, parent_class, code_documents_content, expected, description): + """Test _is_subclass_of with various inheritance patterns including mixins, transitive, and prototype-based.""" + code_documents = { + source: Document( + page_content=content, + metadata={'source': source, 'content_type': 'simplified_code'} + ) + for source, content in code_documents_content.items() + } + + result = parser._is_subclass_of(child_class, parent_class, code_documents) + assert result == expected, f"{description}: expected {expected}, got {result}" + + +@pytest.mark.parametrize("function_name,file_content,expected,description", [ + # ES6: export { functionName } + ( + "myFunc", + """function myFunc() { return 1; } +export { myFunc };""", + True, + "ES6 named export with braces", + ), + # ES6: export { func as functionName } (aliased export) + ( + "myFunc", + """function internalFunc() { return 1; } +export { internalFunc as myFunc };""", + True, + "ES6 named export with alias", + ), + # ES6: export default functionName + ( + "myFunc", + """function myFunc() { return 1; } +export default myFunc;""", + True, + "ES6 export default", + ), + # CommonJS: module.exports = functionName + ( + "myFunc", + """function myFunc() { return 1; } +module.exports = myFunc;""", + True, + "CommonJS direct module.exports assignment", + ), + # CommonJS: module.exports = { functionName } + ( + "myFunc", + """function myFunc() { return 1; } +function otherFunc() { return 2; } +module.exports = { myFunc, otherFunc };""", + True, + "CommonJS module.exports object shorthand", + ), + # CommonJS: module.exports = { functionName: functionName } + ( + "myFunc", + """function myFunc() { return 1; } +module.exports = { myFunc: myFunc };""", + True, + "CommonJS module.exports object with explicit key-value", + ), + # CommonJS: module.exports.functionName = ... + ( + "myFunc", + """function myFunc() { return 1; } +module.exports.myFunc = myFunc;""", + True, + "CommonJS module.exports property assignment", + ), + # CommonJS: exports.functionName = ... + ( + "myFunc", + """function myFunc() { return 1; } +exports.myFunc = myFunc;""", + True, + "CommonJS exports property assignment", + ), + # Not exported - should return False + ( + "myFunc", + """function myFunc() { return 1; } +function otherFunc() { return 2; } +// No exports""", + False, + "Function not exported", + ), + # Function name as substring of another - should NOT match + ( + "parse", + """function parseAsync() { return 1; } +function parseSomething() { return 2; } +module.exports = { parseAsync, parseSomething };""", + False, + "Function name as prefix of other functions - word boundary check", + ), + ( + "divide", + """const add = (a, b) => a + b; +const multiply = (a, b) => a * b; +const divide = (a, b) => a / b; +export { add, multiply, divide };""", + True, + "multiple functions exporting", + ), +]) +def test_is_exportable_function(parser, function_name, file_content, expected, description): + """Test _is_exportable_function with various export patterns.""" + result = parser._is_exportable_function(function_name, file_content) + assert result == expected, description + + +@pytest.mark.parametrize("description,code_content,identifier,callee_package,expected", [ + # ======================================================================== + # Dynamic imports with relative paths + # ======================================================================== + ( + "Top-level await dynamic import with relative path", + """// main.mjs (or in a script tag with type="module") + +// Dynamically import the "utils" module at the top level +const utils = await import('./utils.mjs'); +console.log(utils.calculateArea(5));""", + "utils", + "utils", + True, + ), + ( + "Async function with destructuring and default export", + """async function loadSpecifics() { + const { hi, default: say } = await import('./say.js'); + + hi(); // Hello! + say(); // Module loaded (export default)! +} + +loadSpecifics();""", + "hi", + "say", + True, + ), + ( + "Async function - check default export renamed to 'say'", + """async function loadSpecifics() { + const { hi, default: say } = await import('./say.js'); + + hi(); // Hello! + say(); // Module loaded (export default)! +} + +loadSpecifics();""", + "say", + "say", + True, + ), + ( + "Async function with try/catch block", + """async function loadAndCalculate() { + try { + // Dynamically import the module + const module = await import('./math.js'); + + // Use the exported functions + const sum = module.add(5, 3); + const difference = module.subtract(10, 4); + + console.log(`Sum: ${sum}`); // Output: Sum: 8 + console.log(`Difference: ${difference}`); // Output: Difference: 6 + + } catch (error) { + console.error('Failed to load the math module:', error); + } +} + +// Call the async function +loadAndCalculate();""", + "module", + "math", + True, + ), + # ======================================================================== + # Dynamic imports with NPM packages + # ======================================================================== + ( + "Top-level await dynamic import with method call", + """const utils = await import('lodash'); + console.log(utils.template('hello'));""", + "utils", + "lodash", + True, + ), + ( + "Async function with destructuring and default export - NPM package", + """async function loadSpecifics() { + const { parse, default: babylon } = await import('babylon'); + + parse(); // Parse code! + babylon(); // Module loaded (export default)! +} + +loadSpecifics();""", + "parse", + "babylon", + True, + ), + ( + "Async function - check default export renamed - NPM package", + """async function loadSpecifics() { + const { parse, default: babylon } = await import('babylon'); + + parse(); + babylon(); +}""", + "babylon", + "babylon", + True, + ), + ( + "Async function with try/catch and module property access - NPM package", + """async function loadAndCalculate() { + try { + // Dynamically import the module + const module = await import('mathjs'); + + // Use the exported functions + const sum = module.add(5, 3); + const difference = module.subtract(10, 4); + + console.log(`Sum: ${sum}`); // Output: Sum: 8 + console.log(`Difference: ${difference}`); // Output: Difference: 6 + + } catch (error) { + console.error('Failed to load the math module:', error); + } +} + +// Call the async function +loadAndCalculate();""", + "module", + "mathjs", + True, + ), + # ======================================================================== + # Dynamic imports with NPM packages + # ======================================================================== + ( + "Dynamic import of NPM package - lodash", + """const lodash = await import("lodash"); +lodash.template();""", + "lodash", + "lodash", + True, + ), + ( + "Dynamic import of NPM package with destructuring", + """const { template } = await import('lodash'); +template();""", + "template", + "lodash", + True, + ), + ( + "Dynamic import of scoped NPM package", + """const babel = await import('@babel/core'); +babel.transform();""", + "babel", + "@babel/core", + True, + ), + ( + "Dynamic import of express", + """const express = await import('express'); +express();""", + "express", + "express", + True, + ), + ( + "Dynamic import with multiple spaces", + """const lodash = await import ('lodash');""", + "lodash", + "lodash", + True, + ), + ( + "Dynamic import with space between import and parenthesis", + """const axios = await import ('axios');""", + "axios", + "axios", + True, + ), + ( + "Dynamic import without await - .then pattern (empty identifier → False)", + """import('lodash').then(mod => mod.template());""", + "", + "lodash", + False, + ), + ( + "Dynamic import side-effect only (empty identifier → False)", + """await import('side-effect-pkg');""", + "", + "side-effect-pkg", + False, + ), + ( + "Dynamic import with backticks", + """const moment = await import(`moment`);""", + "moment", + "moment", + True, + ), + # ======================================================================== + # Variable-based dynamic imports (with tracing) + # ======================================================================== + ( + "Variable-based import - NPM package with const", + """const pkgName = 'lodash'; +const lodash = await import(pkgName);""", + "lodash", + "lodash", + True, + ), + ( + "Variable-based import - NPM package with let", + """let pkgName = 'express'; +const express = await import(pkgName);""", + "express", + "express", + True, + ), + ( + "Variable-based import - NPM package with let", + """let pkgName = 'express'; + const express_pakc = await import(pkgName);""", + "express_pakc", + "express", + True, + ), + ( + "Variable-based import - with destructuring", + """const moduleName = 'express'; +const { Router } = await import(moduleName);""", + "Router", + "express", + True, + ), + ( + "Variable-based import - scoped package", + """const pkg = '@babel/core'; +const babel = await import(pkg);""", + "babel", + "@babel/core", + True, + ), + # ======================================================================== + # Recursive variable tracing - variable chains + # ======================================================================== + ( + "Variable chain 2-level", + """const original = 'lodash'; +const pkg = original; +const lib = await import(pkg);""", + "lib", + "lodash", + True, + ), + ( + "Variable chain 3-level", + """const original = 'axios'; +const alias1 = original; +const alias2 = alias1; +const lib = await import(alias2);""", + "lib", + "axios", + True, + ), + ( + "Variable chain with mixed let/const/var", + """var pkgName = 'moment'; +let temp = pkgName; +const final = temp; +const lib = await import(final);""", + "lib", + "moment", + True, + ), + # ======================================================================== + # Negative cases - should NOT match + # ======================================================================== + ( + "Dynamic import with wrong package name", + """const lodash = await import('lodash');""", + "lodash", + "express", + False, + ), + ( + "Dynamic import with wrong identifier", + """const lodash = await import('lodash');""", + "wrongIdentifier", + "lodash", + False, + ), + ( + "Dynamic import in comment - should not match", + """// const lodash = await import('lodash'); +const other = require('express');""", + "lodash", + "lodash", + False, + ), + ( + "Dynamic import in string literal - should not match", + """const code = "const lodash = await import('lodash');";""", + "lodash", + "lodash", + False, + ), + # ======================================================================== + # Variable tracing - non-traceable cases + # ======================================================================== + ( + "Dynamic import with environment variable (not traceable)", + """const module = await import(process.env.MODULE_PATH);""", + "module", + "lodash", + False, + ), + ( + "Dynamic import with config object property (not traceable)", + """const mod = await import(config.modulePath);""", + "mod", + "lodash", + False, + ), + ( + "Dynamic import with function call (not traceable)", + """const utils = await import(getModulePath());""", + "utils", + "lodash", + False, + ), +# ( +# "Dynamic import with computed expression (not traceable)", +# """const prefix = 'lod'; +# const lib = await import(prefix + 'ash');""", +# "lib", +# "lodash", +# False, +# ), + ( + "Variable not found in code (not traceable)", + """const lib = await import(unknownVariable);""", + "lib", + "lodash", + False, + ), + ( + "Variable chain that ends in non-traceable expression", + """const base = getPackageName(); +const pkg = base; +const lib = await import(pkg);""", + "lib", + "lodash", + False, + ), + # ======================================================================== + # Mixed static and dynamic imports + # ======================================================================== + ( + "Both static and dynamic imports - check dynamic", + """import { staticFunc } from 'other-package'; +const lodash = await import('lodash');""", + "lodash", + "lodash", + True, + ), + ( + "Both static and dynamic imports - check static", + """import { template } from 'lodash'; +const express = await import('express');""", + "template", + "lodash", + True, + ), + # ======================================================================== + # Edge cases + # ======================================================================== + ( + "Dynamic import with whitespace variations", + """const lodash = await import( 'lodash' );""", + "lodash", + "lodash", + True, + ), + ( + "Dynamic import without assignment (empty identifier → False)", + """(await import('lodash')).template();""", + "", + "lodash", + False, + ), + ( + "Multiple dynamic imports - check first", + """const lodash = await import('lodash'); +const express = await import('express');""", + "lodash", + "lodash", + True, + ), + ( + "Multiple dynamic imports - check second", + """const lodash = await import('lodash'); +const express = await import('express');""", + "express", + "express", + True, + ), + ( + "Dynamic import in async function", + """async function loadModule() { + const { template } = await import('lodash'); + return template; +}""", + "template", + "lodash", + True, + ), + ( + "Dynamic import with Promise.then and destructuring", + """import('lodash').then(({ template }) => template());""", + "template", + "lodash", + True, + ), +]) +def test_is_package_imported_dynamic_imports(parser, description, code_content, identifier, callee_package, expected): + """Test is_package_imported with dynamic import() syntax including variable tracing.""" + result = parser.is_package_imported(code_content, identifier, callee_package) + assert result == expected, f"{description}: expected {expected}, got {result}" + + +# ============================================================================ +# Tests for is_package_imported - Multi-function imports and word boundary edge cases +# ============================================================================ + +# ============================================================================ +# Tests for empty-name CCA guards +# ============================================================================ + +def test_get_function_calls_empty_name_returns_empty(parser): + """_get_function_calls must return [] when callee_function_name is empty.""" + caller = Document( + page_content="function test() {\n anything();\n something.method();\n}", + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser._get_function_calls(caller, '') == [] + assert parser._get_function_calls(caller, None) == [] + + +def test_is_package_imported_empty_identifier_returns_false(parser): + """is_package_imported must return False when identifier is empty.""" + code = "import { template } from 'lodash';\nconst x = require('express');" + assert parser.is_package_imported(code, '', 'lodash') is False + assert parser.is_package_imported(code, '', '') is False + + +def test_filter_docs_by_func_pkg_name_empty_name_returns_empty(parser): + """filter_docs_by_func_pkg_name must return [] when function_name is empty.""" + docs = [ + Document(page_content="function template() {}", metadata={'source': 'node_modules/lodash/template.js'}), + Document(page_content="function parse() {}", metadata={'source': 'node_modules/lodash/parse.js'}), + ] + assert parser.filter_docs_by_func_pkg_name('', 'lodash', docs) == [] + + +def test_search_for_called_function_empty_callee_name(parser): + """search_for_called_function returns False when callee_function_name is empty. + + Without the guard on _get_function_calls, an empty callee name would + cause the regex to match spuriously and return True for any caller that imports + the callee package. + """ + caller = Document( + page_content="function handler() {\n lodash.template('hello');\n}", + metadata={'source': 'app/handler.js', 'content_type': 'functions_classes'} + ) + callee = Document( + page_content="function template(str) { return str; }", + metadata={'source': 'node_modules/lodash/template.js', 'content_type': 'functions_classes'} + ) + code_docs = { + 'app/handler.js': Document( + page_content="import { template } from 'lodash';\nfunction handler() { lodash.template('hello'); }", + metadata={'source': 'app/handler.js', 'content_type': 'simplified_code'} + ) + } + + result = parser.search_for_called_function( + caller_function=caller, + callee_function_name='', + callee_function=callee, + callee_function_package='lodash', + code_documents=code_docs, + type_documents=[], + callee_function_file_name='node_modules/lodash/template.js', + fields_of_types={}, + functions_local_variables_index={}, + documents_of_functions=[], + type_inheritance={} + ) + assert result is False + + +# ============================================================================ +# Tests for dotted call chain RHS in variable declarations +# ============================================================================ + +@pytest.mark.parametrize("code,expected_name", [ + ( + "var changes = ts.textChanges.ChangeTracker.with(context, function(tracker) {\n tracker.apply();\n})", + "changes" + ), + ( + "const result = obj.method.call(this, arg)", + "result" + ), + ( + "let parser = JSON.parse(data)", + "parser" + ), +]) +def test_get_function_name_dotted_rhs(parser, code, expected_name): + """Variable declarations with dotted call chains on the RHS should extract the variable name.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == expected_name + + +# ============================================================================ +# Tests for bare assignment function +# ============================================================================ + +@pytest.mark.parametrize("code,expected_name", [ + ( + "name = function(x, y) {\n return x + y;\n}", + "name" + ), + ( + "callback = function() {\n return 1;\n}", + "callback" + ), + ( + "handler = async function(req, res) {\n await handle(req);\n}", + "handler" + ), + ( + "gen = function*() {\n yield 1;\n}", + "gen" + ), +]) +def test_get_function_name_bare_assignment_function(parser, code, expected_name): + """Bare assignment to function expression (no const/let/var) should extract the LHS name.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == expected_name + + +# ============================================================================ +# Tests for bare assignment arrow +# ============================================================================ + +@pytest.mark.parametrize("code,expected_name", [ + ( + "resolve = (id, options) => {\n return lookup(id);\n}", + "resolve" + ), + ( + "process = async (data) => {\n await save(data);\n}", + "process" + ), + ( + "transform = x => x * 2", + "transform" + ), +]) +def test_get_function_name_bare_assignment_arrow(parser, code, expected_name): + """Bare assignment to arrow function (no const/let/var) should extract the LHS name.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == expected_name + + +# ============================================================================ +# Tests for anonymous export default +# ============================================================================ + +@pytest.mark.parametrize("code", [ + "export default function() {\n return 42;\n}", + "export default async function() {\n await fetch('/api');\n}", + "export default function*() {\n yield 1;\n}", +]) +def test_get_function_name_anonymous_export_default(parser, code): + """Anonymous export default should return empty string.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == '' + + +# ============================================================================ +# Tests for generator computed property +# ============================================================================ + +def test_get_function_name_generator_computed_property(parser): + """Generator computed property with * prefix should extract the symbol expression.""" + doc = Document( + page_content="*[Symbol.iterator]() {\n for (const item of this.items) {\n yield item;\n }\n}", + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == 'Symbol.iterator' + + +# ============================================================================ +# Tests for keyword rejection in fallback pattern +# ============================================================================ + +@pytest.mark.parametrize("code,expected_name", [ + ( + "doStuff() {\n if (x) {\n return;\n }\n}", + "doStuff" + ), + ( + "render() {\n for (let i = 0; i < 10; i++) {\n items.push(i);\n }\n}", + "render" + ), + ( + "handle() {\n while (queue.length) {\n process(queue.shift());\n }\n}", + "handle" + ), + ( + "execute() {\n switch(type) {\n case('a'):\n break;\n }\n}", + "execute" + ), +]) +def test_get_function_name_keyword_rejection(parser, code, expected_name): + """Keyword filtering should skip JS keywords and find the real method name.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == expected_name + + +@pytest.mark.parametrize("code", [ + "var reporter = {\n onTestStart: function(test) { if (test.pending) return; }\n}", + "if (x) {\n for (y in z) {\n while (true) { break; }\n }\n}", +]) +def test_get_function_name_all_keywords_returns_empty(parser, code): + """When only keywords match the fallback pattern, return empty string.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == '' + + +# ============================================================================ +# Tests for keyword set trimming: removed keywords that are valid method names +# ============================================================================ + +@pytest.mark.parametrize("code, expected_name", [ + # 'default' removed from _JS_KEYWORDS — valid method in commander.js etc. + ("default(value, description) {\n this._defaultValue = value;\n}", "default"), + # 'case' removed — can be a method name (e.g., case(value) in switch builders) + ("case(value) {\n this.cases.push(value);\n}", "case"), + # 'export' removed — can be a method name (e.g., export(data) in serializers) + ("export(data) {\n return JSON.stringify(data);\n}", "export"), + # 'import' removed — can be a method name + ("import(module) {\n return require(module);\n}", "import"), + # 'in' removed — can be a method name (e.g., in(collection) in query builders) + ("in(collection) {\n return this.where(collection);\n}", "in"), + # 'of' removed — can be a method name + ("of(items) {\n return new Collection(items);\n}", "of"), + # 'super' removed — can be used as method name in some patterns + ("super(args) {\n return parent.call(this, args);\n}", "super"), + # 'this' removed — can be used as method name + ("this(args) {\n return self.init(args);\n}", "this"), +]) +def test_get_function_name_removed_keywords_are_valid_methods(parser, code, expected_name): + """Keywords removed from _JS_KEYWORDS should be extractable as method names.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == expected_name + + +@pytest.mark.parametrize("code", [ + # Keywords still in _JS_KEYWORDS should NOT be extracted (no non-keyword calls in body) + "if (x) { return (y) }", + "for (var y in z) { break; }", + "while (true) { continue; }", + "switch (x) { break; }", + "return (x)", + "throw (x)", + "new (x)", + "typeof (x)", + "delete (x)", +]) +def test_get_function_name_retained_keywords_still_rejected(parser, code): + """Keywords still in _JS_KEYWORDS should return empty string.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == '' + + +# ============================================================================ +# Tests for variable declaration matching restricted to header +# ============================================================================ + +@pytest.mark.parametrize("code, expected_name", [ + # Should NOT match const/let/var assignments inside function body + ("async method() {\n const x = obj.call(y);\n return x;\n}", "method"), + ("render() {\n const sanitized = this.sanitize(input);\n return sanitized;\n}", "render"), + # Should match in header (no body yet) + ("const result = helper.create(args)", "result"), + # Multi-line header before body + ("const handler =\n middleware.wrap(\n fn\n )", "handler"), +]) +def test_get_function_name_pattern5_body_restriction(parser, code, expected_name): + """Variable declaration matching should only search before the first '{' to avoid body matches.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == expected_name + + +# ============================================================================ +# Tests for body-matching prevention: patterns should not match inside bodies +# ============================================================================ + +@pytest.mark.parametrize("code, expected_name", [ + # Function declaration inside body should not hijack name + ("method() { function inner() {} }", "method"), + ("render() { async function fetchData() {} }", "render"), + # Const single-param arrow inside body + ("method() { const helper = x => x }", "method"), + # Property function assignment inside body + ("method() { this.handler = function() {} }", "method"), + # Property single-param arrow inside body + ("method() { this.cb = val => val }", "method"), + # Bare assignment function inside body + ("method() { callback = function() {} }", "method"), + # Bare single-param arrow inside body + ("method() { transform = x => x * 2 }", "method"), +]) +def test_get_function_name_body_matching_prevention(parser, code, expected_name): + """Patterns restricted to header should not match declarations inside function bodies.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == expected_name + + +# ============================================================================ +# Tests for generator name without space: function*name() +# ============================================================================ + +@pytest.mark.parametrize("code, expected_name", [ + ("function*myGen() { yield 1; }", "myGen"), + ("function *myGen() { yield 1; }", "myGen"), + ("function * myGen() { yield 1; }", "myGen"), + ("async function*streamGen() { yield 1; }", "streamGen"), +]) +def test_get_function_name_generator_no_space(parser, code, expected_name): + """Generator declarations should match function*name() without space after *.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == expected_name + + +def test_get_function_name_no_functionName_false_match(parser): + """Word boundary should not match 'functionName(' as 'Name'.""" + doc = Document( + page_content="functionName(x) { return x; }", + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == 'functionName' + + +# ============================================================================ +# Tests for function_called_from_caller_body regex fixes +# ============================================================================ + +def test_function_called_from_caller_body_finds_call(): + """Should return truthy when function is called (not declared) in body.""" + from exploit_iq_commons.utils.chain_of_calls_retriever import ChainOfCallsRetriever + from exploit_iq_commons.utils.functions_parsers.javascript_functions_parser import JavaScriptFunctionsParser + + parser = JavaScriptFunctionsParser() + retriever = ChainOfCallsRetriever.__new__(ChainOfCallsRetriever) + retriever.language_parser = parser + + doc = Document( + page_content="function handler(req) {\n const result = process(req.body);\n return result;\n}", + metadata={'source': 'app/handler.js', 'content_type': 'functions_classes'} + ) + assert retriever.function_called_from_caller_body(doc, "process") + + +# ============================================================================ +# Tests for function_called_from_caller_body empty-name pass-through +# ============================================================================ + +def test_function_called_from_caller_body_empty_name_passthrough(): + """function_called_from_caller_body returns True for empty function_to_search (pass-through). + + Empty function_to_search is used by __find_initial_function as a "no filter" — + all documents pass through. Empty-name defense is handled by BFS guards. + """ + from exploit_iq_commons.utils.chain_of_calls_retriever import ChainOfCallsRetriever + from exploit_iq_commons.utils.functions_parsers.javascript_functions_parser import JavaScriptFunctionsParser + + parser = JavaScriptFunctionsParser() + retriever = ChainOfCallsRetriever.__new__(ChainOfCallsRetriever) + retriever.language_parser = parser + + doc = Document( + page_content="function handler() {\n someLib.process(data);\n}", + metadata={'source': 'app/handler.js', 'content_type': 'functions_classes'} + ) + assert retriever.function_called_from_caller_body(doc, "") is True + assert retriever.function_called_from_caller_body(doc, " ") is True + + +# ============================================================================ +# Tests for _breadth_first_search empty-name guard +# ============================================================================ + +def test_bfs_skips_empty_name_docs(): + """BFS should skip documents whose get_function_name returns empty string.""" + from unittest.mock import MagicMock, patch + from exploit_iq_commons.utils.chain_of_calls_retriever import ChainOfCallsRetriever, _SearchCtx + + parser = JavaScriptFunctionsParser() + retriever = ChainOfCallsRetriever.__new__(ChainOfCallsRetriever) + retriever.language_parser = parser + retriever.documents = [] + retriever.sort_docs = {} + + empty_name_doc = Document( + page_content="var reporter = {\n if (x) {}\n}", + metadata={'source': 'test.js', 'content_type': 'functions_classes', 'state': None} + ) + + ctx = _SearchCtx() + result, found = retriever._breadth_first_search([], empty_name_doc, 'test-pkg', ctx) + assert found is False + assert result == [] + + +# ============================================================================ +# Tests for __find_caller_functions_bfs candidate empty-name guard +# ============================================================================ + +def test_bfs_caller_skips_empty_name_candidates(): + """__find_caller_functions_bfs should skip candidate docs with empty function names.""" + from unittest.mock import MagicMock, patch + from collections import defaultdict + from exploit_iq_commons.utils.chain_of_calls_retriever import ChainOfCallsRetriever, _SearchCtx + + parser = JavaScriptFunctionsParser() + retriever = ChainOfCallsRetriever.__new__(ChainOfCallsRetriever) + retriever.language_parser = parser + retriever.ecosystem = 'javascript' + + target_doc = Document( + page_content="function vulnerable() {\n return 'exploit';\n}", + metadata={'source': 'node_modules/lib/index.js', 'content_type': 'functions_classes'} + ) + + empty_name_candidate = Document( + page_content="var reporter = {\n if (x) {}\n}", + metadata={'source': 'node_modules/parent/index.js', 'content_type': 'functions_classes', 'state': None} + ) + + retriever.sort_docs = defaultdict(list, {'parent-pkg': [empty_name_candidate]}) + retriever.documents = [empty_name_candidate] + retriever.documents_of_full_sources = {} + retriever.documents_of_types = [] + retriever.types_classes_fields_mapping = {} + retriever.functions_local_variables_index = {} + retriever.documents_of_functions = [] + + ctx = _SearchCtx() + + with patch.object(retriever, '_get_parents', return_value=['parent-pkg']): + result = retriever._ChainOfCallsRetriever__find_caller_functions_bfs( + document_function=target_doc, + function_package='lib', + ctx=ctx + ) + + assert result == [] + + +@pytest.mark.parametrize("description,code_content,identifier,callee_package,expected", [ + # ======================================================================== + # Multi-function imports - should match exact identifiers + # ======================================================================== + ( + "Multi-function import - exact match for function1", + """import { function1, function2 } from "./utils.js";""", + "function1", + "./utils.js", + True, + ), + ( + "Multi-function import - exact match for function2", + """import { function1, function2 } from "./utils.js";""", + "function2", + "./utils.js", + True, + ), + # ======================================================================== + # Substring edge cases - should NOT match partial identifiers + # ======================================================================== + ( + "Multi-function import - 'func' should NOT match 'function1'", + """import { function1, function2 } from "./utils.js";""", + "func", + "./utils.js", + False, + ), + ( + "Multi-function import - 'add' should NOT match 'addHandler'", + """import { addHandler, removeHandler } from "./utils.js";""", + "add", + "./utils.js", + False, + ), + ( + "Single function import - 'parse' should NOT match 'parseAsync'", + """import { parseAsync } from "some-lib";""", + "parse", + "some-lib", + False, + ), + # ======================================================================== + # CommonJS require - substring edge cases + # ======================================================================== + ( + "CommonJS destructured require - exact match", + """const { function1, function2 } = require('./utils.js');""", + "function1", + "./utils.js", + True, + ), + ( + "CommonJS destructured require - 'func' should NOT match 'function1'", + """const { function1, function2 } = require('./utils.js');""", + "func", + "./utils.js", + False, + ), + # ======================================================================== + # Re-exports - substring edge cases + # ======================================================================== + ( + "Re-export - exact match", + """export { function1, function2 } from "./utils.js";""", + "function1", + "./utils.js", + True, + ), + ( + "Re-export - 'func' should NOT match 'function1'", + """export { function1, function2 } from "./utils.js";""", + "func", + "./utils.js", + False, + ), +]) +def test_is_package_imported_word_boundaries(parser, description, code_content, identifier, callee_package, expected): + """Test is_package_imported with word boundary edge cases to prevent substring false positives.""" + result = parser.is_package_imported(code_content, identifier, callee_package) + assert result == expected, f"{description}: expected {expected}, got {result}" + + +@pytest.mark.parametrize("function_content,expected_name", [ + # Object methods now use //(class: name) pattern + ( + "fetchData(url) {\n return fetch(url);\n}\n//(class: apiUtils)", + "apiUtils" + ), + ( + "format: function(data) {\n return JSON.stringify(data);\n}\n//(class: helpers)", + "helpers" + ), + # Standalone function (no annotation) + ( + "function regularFunc() {\n return true;\n}", + None + ), +]) +def test_get_class_name_for_object_methods(parser, function_content, expected_name): + """Test that object methods use the same //(class: name) pattern as classes.""" + doc = Document( + page_content=function_content, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + result = parser.get_class_name_from_class_function(doc) + assert result == expected_name + + +@pytest.mark.parametrize("description,function_content,file_content,expected", [ + ( + "Object method should be exported when object is in module.exports", + "fetchData(url) {\n return fetch(url);\n}\n//(class: apiUtils)", + """const apiUtils = { + fetchData(url) { + return fetch(url); + } +}; + +module.exports = { apiUtils }; +""", + True, + ), + ( + "Object method should be exported with ES6 named export", + "format(data) {\n return JSON.stringify(data);\n}\n//(class: helpers)", + """const helpers = { + format(data) { + return JSON.stringify(data); + } +}; + +export { helpers }; +""", + True, + ), + ( + "Object method should NOT be exported when object is not exported", + "internalMethod() {\n return 'internal';\n}\n//(class: privateUtils)", + """const privateUtils = { + internalMethod() { + return 'internal'; + } +}; + +// No exports +""", + False, + ), +]) +def test_is_exported_function_for_objects(parser, description, function_content, file_content, expected): + """Test is_exported_function correctly handles object methods with //(class: name) annotation.""" + source = "node_modules/test-pkg/index.js" + func_doc = Document( + page_content=function_content, + metadata={'source': source, 'content_type': 'functions_classes'} + ) + full_file_doc = Document( + page_content=file_content, + metadata={'source': source, 'content_type': 'simplified_code'} + ) + documents_of_full_sources = {source: full_file_doc} + + result = parser.is_exported_function(func_doc, documents_of_full_sources) + assert result == expected, f"{description}: expected {expected}, got {result}" + + +# ============================================================================= +# should_skip — node_modules/dist should NOT be skipped +# ============================================================================= + +@pytest.mark.parametrize("path,expected", [ + ("node_modules/express/dist/router.js", False), + ("node_modules/@babel/core/dist/index.js", False), + ("node_modules/pkg/dist/build/static/app.js", False), + ("dist/bundle.js", True), + ("src/dist/helper.js", False), + ("build/static/app.js", True), + ("app.min.js", True), + ("src/index.js", False), + ("coverage/report.js", True), +]) +def test_should_skip_node_modules_dist(path, expected): + """dist/ inside node_modules/ is legitimate source, not a build artifact.""" + assert ExtendedJavaScriptSegmenter.should_skip(path) == expected + + +# ============================================================================= +# Optional chaining in _get_function_calls +# ============================================================================= + +def test_get_function_calls_optional_chaining(parser): + """_get_function_calls should detect calls through optional chaining (?.).""" + caller = Document( + page_content="function test() {\n obj?.method(arg);\n}", + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + calls = parser._get_function_calls(caller, "method") + assert any("method" in c for c in calls), f"Expected 'method' call, got {calls}" + + +def test_get_function_calls_chained_optional(parser): + """_get_function_calls should handle chained optional access a?.b?.method().""" + caller = Document( + page_content="function test() {\n a?.b?.method(x);\n}", + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + calls = parser._get_function_calls(caller, "method") + assert any("method" in c for c in calls), f"Expected 'method' call, got {calls}" + + +# ============================================================================= +# Position check — destructuring defaults inside bodies +# ============================================================================= + +@pytest.mark.parametrize("code,expected_name", [ + ( + "method({callback = (x) => x}) {\n return callback;\n}", + "method", + ), + ( + "const handler = ({a, b}) => { return a + b; }", + "handler", + ), + ( + "resolve = ({id}) => { doStuff(); }", + "resolve", + ), + ( + "function outer() {\n const inner = (x) => x;\n}", + "outer", + ), +]) +def test_position_check_destructuring_in_body(parser, code, expected_name): + """Patterns with [^)]* should not match destructuring defaults inside bodies.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == expected_name + + +# ============================================================================= +# Function expression with destructuring uses header +# ============================================================================= + +def test_pattern5_function_expr_with_destructuring(parser): + """Function expressions with destructuring params should extract the variable name.""" + doc = Document( + page_content="const fn = function({a, b}) { return a + b; }", + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == "fn" + + +# ============================================================================= +# Bare assignment and property patterns +# ============================================================================= + +@pytest.mark.parametrize("code,expected_name", [ + ( + "obj.method = function() { return 1; }", + "method", + ), + ( + "obj.handler = (x) => x * 2", + "handler", + ), + ( + "exports.default = async function(req, res) { res.send(); }", + "default", + ), + ( + "module.exports.init = function() {}", + "init", + ), +]) +def test_property_assignment_after_pattern_removal(parser, code, expected_name): + """Property assignments (obj.name = ...) should still work after removing patterns 6-8.""" + doc = Document( + page_content=code, + metadata={'source': 'test.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == expected_name + + +# ============================================================================= +# Optional chaining '?' leaks into identifiers in _get_function_calls +# ============================================================================= + +class TestOptionalChainingInFunctionCalls: + """_get_function_calls regex captures '?' from optional chaining, which + then leaks into search_for_called_function's identifier split, preventing + import resolution and local-var lookup.""" + + def test_optional_chaining_direct_call_detected(self, parser): + """_get_function_calls should detect calls through optional chaining.""" + caller = Document( + page_content="function process(res) {\n return res?.json();\n}", + metadata={'source': 'app/handler.js', 'content_type': 'functions_classes'} + ) + calls = parser._get_function_calls(caller, "json") + assert len(calls) >= 1, "Should detect json() through optional chaining" + + def test_optional_chaining_identifier_has_no_question_mark(self, parser): + """After splitting a qualified optional-chaining call by '.', identifiers + should NOT contain '?'. The '?' is syntax, not part of the name.""" + caller = Document( + page_content="function process(res) {\n return res?.json();\n}", + metadata={'source': 'app/handler.js', 'content_type': 'functions_classes'} + ) + calls = parser._get_function_calls(caller, "json") + for call in calls: + parts = call.split('.') + for part in parts: + clean = part.rstrip('(') + assert '?' not in clean, ( + f"Identifier '{clean}' still contains '?' from optional chaining" + ) + + def test_optional_chaining_search_for_called_function(self, parser): + """search_for_called_function should find calls through optional chaining + when the identifier matches an imported package.""" + caller = Document( + page_content=( + "import axios from 'axios';\n" + "function fetchData() {\n" + " return axios?.get('/api/data');\n" + "}" + ), + metadata={'source': 'app/api.js', 'content_type': 'functions_classes'} + ) + callee = Document( + page_content="get(url) {\n return this.request('GET', url);\n}\n//(class: axios)", + metadata={'source': 'node_modules/axios/index.js', 'content_type': 'functions_classes'} + ) + code_docs = { + 'app/api.js': Document( + page_content="import axios from 'axios';\nfunction fetchData() { return axios?.get('/api/data'); }", + metadata={'source': 'app/api.js', 'content_type': 'simplified_code'} + ) + } + result = parser.search_for_called_function( + caller_function=caller, + callee_function_name="get", + callee_function=callee, + callee_function_package="axios", + code_documents=code_docs, + type_documents=[], + callee_function_file_name='node_modules/axios/index.js', + fields_of_types={}, + functions_local_variables_index={}, + documents_of_functions=[], + ) + assert result is True, ( + "search_for_called_function should match 'axios?.get()' — " + "the '?' is syntax, not part of the identifier" + ) + + +# ============================================================================= +# Named export chunks lose 'export' prefix — is_exported_function +# fallback doesn't check for 'export function name' pattern +# ============================================================================= + +class TestNamedExportWithoutPrefix: + """Tree-sitter extracts 'function compile(...)' without the 'export' prefix. + is_exported_function should still detect it as exported via the fallback.""" + + def test_named_export_function_without_prefix(self, parser): + """A chunk 'function compile(input) {...}' from an 'export function compile' + should be detected as exported by checking the full source.""" + func_doc = Document( + page_content="function compile(input) {\n return transform(input);\n}", + metadata={'source': 'node_modules/handlebars/index.js', 'content_type': 'functions_classes'} + ) + full_source = Document( + page_content=( + "import { transform } from './transform';\n" + "export function compile(input) {\n" + " return transform(input);\n" + "}\n" + "export function precompile(input) {\n" + " return parse(input);\n" + "}\n" + ), + metadata={'source': 'node_modules/handlebars/index.js', 'content_type': 'simplified_code'} + ) + docs_of_full_sources = {'node_modules/handlebars/index.js': full_source} + result = parser.is_exported_function(func_doc, docs_of_full_sources) + assert result is True, ( + "Function 'compile' is exported via 'export function compile' in source, " + "even though the chunk itself lacks the 'export' prefix" + ) + + def test_named_export_async_function_without_prefix(self, parser): + """Async export: 'async function fetchData(...)' without 'export' in chunk.""" + func_doc = Document( + page_content="async function fetchData(url) {\n return await fetch(url);\n}", + metadata={'source': 'node_modules/my-lib/api.js', 'content_type': 'functions_classes'} + ) + full_source = Document( + page_content="export async function fetchData(url) {\n return await fetch(url);\n}\n", + metadata={'source': 'node_modules/my-lib/api.js', 'content_type': 'simplified_code'} + ) + docs_of_full_sources = {'node_modules/my-lib/api.js': full_source} + result = parser.is_exported_function(func_doc, docs_of_full_sources) + assert result is True, ( + "Async function 'fetchData' is exported via 'export async function fetchData' in source" + ) + + def test_non_exported_function_still_false(self, parser): + """A function that is genuinely not exported should still return False.""" + func_doc = Document( + page_content="function internal() {\n return 'private';\n}", + metadata={'source': 'node_modules/pkg/utils.js', 'content_type': 'functions_classes'} + ) + full_source = Document( + page_content=( + "function internal() {\n return 'private';\n}\n" + "function external() {\n return internal();\n}\n" + "module.exports = external;\n" + ), + metadata={'source': 'node_modules/pkg/utils.js', 'content_type': 'simplified_code'} + ) + docs_of_full_sources = {'node_modules/pkg/utils.js': full_source} + result = parser.is_exported_function(func_doc, docs_of_full_sources) + assert result is False + + +# ============================================================================= +# BFS ValueError on class docs — get_function_name raises for classes +# ============================================================================= + +class TestGetFunctionNameOnClassDocs: + """get_function_name raises ValueError on class documents. In BFS, this + aborts the entire search loop.""" + + def test_named_class_raises_valueerror(self, parser): + """get_function_name should raise ValueError for named class docs.""" + doc = Document( + page_content="class Router {\n constructor() {}\n route(path) {}\n}", + metadata={'source': 'app/router.js', 'content_type': 'functions_classes'} + ) + assert not parser.is_function(doc) + with pytest.raises(ValueError, match="Only function document"): + parser.get_function_name(doc) + + def test_anonymous_class_is_function_returns_false(self, parser): + """is_function should return False for anonymous class expressions, + preventing ValueError in get_function_name downstream.""" + doc = Document( + page_content="export default class {\n connect() {}\n disconnect() {}\n}", + metadata={'source': 'node_modules/pkg/db.js', 'content_type': 'functions_classes'} + ) + assert not parser.is_function(doc), ( + "Anonymous class 'export default class { ... }' should NOT pass is_function. " + "The regex requires class\\s+[\\w$]+ (a named identifier after 'class')." + ) + + +# ============================================================================= +# Nested parens in arrow function default parameters +# ============================================================================= + +class TestNestedParensInArrowDefaults: + """Arrow functions with nested parens in default params (e.g. getDefaults()) should extract the variable name.""" + + def test_arrow_with_function_call_default(self, parser): + """Arrow function with a function call as default parameter value.""" + doc = Document( + page_content="const handler = (options = getDefaults()) => {\n process(options);\n}", + metadata={'source': 'app/handler.js', 'content_type': 'functions_classes'} + ) + name = parser.get_function_name(doc) + assert name == "handler", ( + f"Expected 'handler' but got '{name}'. [^)]* in arrow regex stops at " + "the inner ')' of getDefaults(), causing fallthrough to catch-all." + ) + + def test_arrow_with_nested_method_call_default(self, parser): + """Arrow with obj.method() call in default parameter.""" + doc = Document( + page_content="const processor = (cfg = Config.load()) => {\n return cfg.run();\n}", + metadata={'source': 'app/process.js', 'content_type': 'functions_classes'} + ) + name = parser.get_function_name(doc) + assert name == "processor", f"Expected 'processor' but got '{name}'" + + def test_arrow_with_simple_default_still_works(self, parser): + """Ensure simple defaults (no nested parens) still work after the fix.""" + doc = Document( + page_content="const greet = (name = 'world') => {\n console.log(name);\n}", + metadata={'source': 'app/greet.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == "greet" + + def test_arrow_with_numeric_default_still_works(self, parser): + """Numeric default: no nested parens.""" + doc = Document( + page_content="const delay = (ms = 1000) => {\n return new Promise(r => setTimeout(r, ms));\n}", + metadata={'source': 'app/delay.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == "delay" + + +# ============================================================================= +# Anonymous export default arrow returns inner function name +# ============================================================================= + +class TestAnonymousExportDefaultArrow: + """export default (x) => {...} with inner named functions should return ''.""" + + def test_anonymous_default_arrow_with_inner_function(self, parser): + """An anonymous export default arrow should return '' even when the body + contains named functions.""" + doc = Document( + page_content=( + "export default async (req) => {\n" + " function validate(r) { return r.ok; }\n" + " return validate(req);\n" + "}" + ), + metadata={'source': 'node_modules/pkg/handler.js', 'content_type': 'functions_classes'} + ) + name = parser.get_function_name(doc) + assert name == '', ( + f"Expected '' for anonymous export default arrow, but got '{name}'. " + "The catch-all matched an inner function name from the body." + ) + + def test_anonymous_default_arrow_simple(self, parser): + """Simple anonymous export default arrow with no inner functions.""" + doc = Document( + page_content="export default (x, y) => {\n return x + y;\n}", + metadata={'source': 'node_modules/pkg/add.js', 'content_type': 'functions_classes'} + ) + name = parser.get_function_name(doc) + assert name == '', f"Expected '' for anonymous arrow, got '{name}'" + + def test_named_default_function_still_works(self, parser): + """Named export default function should still return the name.""" + doc = Document( + page_content="export default function handler(req) {\n return req.body;\n}", + metadata={'source': 'node_modules/pkg/handler.js', 'content_type': 'functions_classes'} + ) + assert parser.get_function_name(doc) == "handler" + + +# ============================================================================= +# is_function returns True for anonymous class expressions +# ============================================================================= + +class TestGetFunctionNamePerformance: + """get_function_name must not catastrophically backtrack on large function bodies + containing many parenthesized expressions (e.g., compiled TypeScript output).""" + + def test_large_prototype_method_completes_fast(self, parser): + """Reproduces XMLSerializerImpl.js — prototype method assignment with a + 17K+ char body full of var declarations and parenthesized calls. + Before fix, get_function_name took 34.8s on this content because + arrow-function regex patterns searched the full content.""" + import time + body_lines = [] + for i in range(300): + body_lines.append(f" var markup{i} = this._serialize(node, (opts{i} || {{}}));") + body_lines.append(f" if (requireWellFormed && (node.localName.indexOf(':') !== -1 ||") + body_lines.append(f" !algorithm_1.xml_isName(node.localName))) {{") + body_lines.append(f" throw new Error('not well-formed: ' + node.localName);") + body_lines.append(f" }}") + body = "\n".join(body_lines) + content = ( + f"XMLSerializerImpl.prototype._serializeElementNS = " + f"function (node, namespace, prefixMap, prefixIndex, requireWellFormed) {{\n" + f" var e_1, _a;\n{body}\n }}" + ) + assert len(content) > 15000 + doc = Document( + page_content=content, + metadata={'source': 'node_modules/@oozcitak/dom/lib/serializer/XMLSerializerImpl.js', + 'content_type': 'functions_classes'} + ) + t0 = time.monotonic() + name = parser.get_function_name(doc) + elapsed = time.monotonic() - t0 + assert name == "_serializeElementNS" + assert elapsed < 1.0, f"get_function_name took {elapsed:.2f}s (should be <1s)" + + def test_arrow_with_destructured_params_still_works(self, parser): + """Arrow function with destructured params — first '{' is in params, + not in the body. Ensures arrow_header covers the full signature.""" + import time + content = ( + "const handleRequest = ({method, url, headers}) => {\n" + " const response = fetch(url);\n" + " let status = 200;\n" + " return response;\n" + "}" + ) + doc = Document( + page_content=content, + metadata={'source': 'app/request.js', 'content_type': 'functions_classes'} + ) + t0 = time.monotonic() + name = parser.get_function_name(doc) + elapsed = time.monotonic() - t0 + assert name == "handleRequest" + assert elapsed < 1.0 + + def test_small_doc_with_parens_in_body(self, parser): + """Reproduces lodash LazyWrapper — anonymous function assigned to + computed property, body has var = (expr) patterns that triggered + catastrophic regex backtracking even on 556 chars.""" + import time + content = ( + "LazyWrapper.prototype[methodName] = function(n) {\n" + " n = n === undefined ? 1 : nativeMax(toInteger(n), 0);\n" + " var result = (this.__filtered__ && !index)\n" + " ? new LazyWrapper(this)\n" + " : this.clone();\n" + " if (result.__filtered__) {\n" + " result.__takeCount__ = nativeMin(n, result.__takeCount__);\n" + " } else {\n" + " result.__views__.push({\n" + " 'size': nativeMin(n, MAX_ARRAY_LENGTH),\n" + " 'type': methodName + (result.__dir__ < 0 ? 'Right' : '')\n" + " });\n" + " }\n" + " return result;\n" + " };" + ) + doc = Document( + page_content=content, + metadata={'source': 'node_modules/lodash/lodash.js', + 'content_type': 'functions_classes'} + ) + t0 = time.monotonic() + name = parser.get_function_name(doc) + elapsed = time.monotonic() - t0 + assert elapsed < 1.0, f"get_function_name took {elapsed:.2f}s (should be <1s)" + + def test_babel_iife_wrapped_arrow_completes_fast(self, parser): + """Reproduces babel package.js — IIFE-wrapped arrow with nested parens + in the header like (0, _utils.makeStaticFileCache)((filepath, content) => + caused catastrophic regex backtracking (38.8s on 662 chars).""" + import time + content = ( + "const readConfigPackage = " + "(0, _utils.makeStaticFileCache)((filepath, content) => {\n" + " let options;\n" + " try {\n" + " options = (0, _json.parse)(content);\n" + " } catch (err) {\n" + " throw new _configError.default(\n" + " `Error while parsing JSON - ${err.message}`, filepath);\n" + " }\n" + " if (!options) throw new _configError.default(\n" + " `No config detected in ${filepath}`, filepath);\n" + " if (typeof options !== 'object') throw new _configError.default(\n" + " `Config returned typeof ${typeof options}`, filepath);\n" + " if (Array.isArray(options)) throw new _configError.default(\n" + " `Config returned an array`, filepath);\n" + " delete options['$schema'];\n" + " return {\n" + " filepath,\n" + " dirname: _path().dirname(filepath),\n" + " options\n" + " };\n" + "});" + ) + doc = Document( + page_content=content, + metadata={'source': 'node_modules/@babel/core/lib/config/files/package.js', + 'content_type': 'functions_classes'} + ) + t0 = time.monotonic() + name = parser.get_function_name(doc) + elapsed = time.monotonic() - t0 + assert elapsed < 1.0, f"get_function_name took {elapsed:.2f}s (should be <1s)" + assert name == "readConfigPackage", f"Expected 'readConfigPackage', got '{name}'" + + def test_constructor_with_inner_arrow_not_confused(self, parser): + """Reproduces PackageURL constructor — body contains arrow functions + like `key => {` which caused arrow_header to extend into the body, + then P9 matched `required =` instead of extracting `constructor`.""" + content = ( + "constructor(type, namespace, name, version, qualifiers, subpath) {\n" + " let required = { 'type': type, 'name': name };\n" + " Object.keys(required).forEach(key => {\n" + " if (!required[key]) {\n" + " throw new Error('Invalid purl: \"' + key + '\" is a required field.');\n" + " }\n" + " });\n" + " }\n" + "//(class: PackageURL)" + ) + doc = Document( + page_content=content, + metadata={'source': 'node_modules/packageurl-js/lib/package-url.js', + 'content_type': 'functions_classes'} + ) + name = parser.get_function_name(doc) + assert name == "constructor", f"Expected 'constructor', got '{name}'" + + +class TestIsFunctionAnonymousClass: + """is_function regex requires class\\s+[\\w$]+ (a named class). Anonymous + classes like 'export default class { ... }' bypass the check.""" + + def test_anonymous_class_is_not_function(self, parser): + """Anonymous default export class should not be considered a function.""" + doc = Document( + page_content="export default class {\n connect() {}\n}", + metadata={'source': 'node_modules/pkg/db.js', 'content_type': 'functions_classes'} + ) + assert not parser.is_function(doc), ( + "Anonymous class should not pass is_function" + ) + + def test_anonymous_class_with_methods_is_not_function(self, parser): + """Anonymous class with multiple methods.""" + doc = Document( + page_content=( + "export default class {\n" + " constructor() { this.x = 1; }\n" + " render() { return this.x; }\n" + "}" + ), + metadata={'source': 'node_modules/pkg/component.js', 'content_type': 'functions_classes'} + ) + assert not parser.is_function(doc) + + def test_named_class_still_not_function(self, parser): + """Named classes should still return False (existing behavior).""" + doc = Document( + page_content="class MyClass {\n constructor() {}\n}", + metadata={'source': 'app/my.js', 'content_type': 'functions_classes'} + ) + assert not parser.is_function(doc) + + def test_regular_function_still_is_function(self, parser): + """Regular functions should still return True.""" + doc = Document( + page_content="function doStuff() {\n return 1;\n}", + metadata={'source': 'app/util.js', 'content_type': 'functions_classes'} + ) + assert parser.is_function(doc) + + +# ============================================================================= +# Generator export regex misses function* (no space) +# ============================================================================= + +class TestGeneratorExportDetection: + """_is_exportable_function must detect generator functions exported as + 'export function* genName()' where * immediately follows 'function'.""" + + def test_export_function_star_no_space(self, parser): + result = parser._is_exportable_function( + "genValues", + "export function* genValues() { yield 1; }" + ) + assert result is True, "function* (no space) should be detected as exported" + + def test_export_function_space_star(self, parser): + result = parser._is_exportable_function( + "genValues", + "export function *genValues() { yield 1; }" + ) + assert result is True, "function * (space before star) should be detected" + + def test_export_function_star_space(self, parser): + result = parser._is_exportable_function( + "genValues", + "export function * genValues() { yield 1; }" + ) + assert result is True, "function * genValues (space around star) should be detected" + + def test_export_async_function_star(self, parser): + result = parser._is_exportable_function( + "genValues", + "export async function* genValues() { yield 1; }" + ) + assert result is True, "async function* should be detected" + + def test_non_generator_still_works(self, parser): + result = parser._is_exportable_function( + "myFunc", + "export function myFunc() { return 1; }" + ) + assert result is True, "regular export function should still work" + + +# ============================================================================= +# Anonymous export-default arrow without parens +# ============================================================================= + +class TestAnonymousExportDefaultArrowNoParens: + """export default x => { innerFunc(); } should return '' not 'innerFunc'.""" + + def test_single_param_no_parens(self, parser): + doc = Document( + page_content="export default x => {\n innerFunc();\n}", + metadata={'source': 'node_modules/pkg/index.js', 'content_type': 'functions_classes'} + ) + name = parser.get_function_name(doc) + assert name == '', f"Expected '' for anonymous single-param arrow, got '{name}'" + + def test_single_param_async_no_parens(self, parser): + doc = Document( + page_content="export default async req => {\n validate(req);\n return respond();\n}", + metadata={'source': 'node_modules/pkg/handler.js', 'content_type': 'functions_classes'} + ) + name = parser.get_function_name(doc) + assert name == '', f"Expected '' for async single-param arrow, got '{name}'" + + def test_single_param_expression_body(self, parser): + doc = Document( + page_content="export default x => x + 1", + metadata={'source': 'node_modules/pkg/inc.js', 'content_type': 'functions_classes'} + ) + name = parser.get_function_name(doc) + assert name == '', f"Expected '' for expression-body arrow, got '{name}'" + + +# ============================================================================= +# arrow_header extends into inner arrows in destructured params +# ============================================================================= + +class TestArrowHeaderInnerArrowExtension: + """Method definitions with destructured params containing inner arrows + should not have arrow_header extend into the params.""" + + def test_method_with_inner_arrow_default(self, parser): + doc = Document( + page_content="method({cb = x => {return x}}) {\n doStuff();\n}", + metadata={'source': 'node_modules/pkg/util.js', 'content_type': 'functions_classes'} + ) + name = parser.get_function_name(doc) + assert name == 'method', f"Expected 'method' but got '{name}'" + + def test_method_with_complex_arrow_default(self, parser): + doc = Document( + page_content="handle({onError = err => {log(err)}, timeout = 5000}) {\n process();\n}", + metadata={'source': 'node_modules/pkg/handler.js', 'content_type': 'functions_classes'} + ) + name = parser.get_function_name(doc) + assert name == 'handle', f"Expected 'handle' but got '{name}'" + + +# ============================================================================= +# Anonymous function name collisions in local-var map +# ============================================================================= + +class TestLocalVarMapCollisions: + """Multiple anonymous functions from same file should not overwrite each other.""" + + def test_multiple_anonymous_same_file_no_collision(self, parser): + """Anonymous functions (empty name) should be skipped rather than collide.""" + docs = [ + Document( + page_content="(x) => {\n const a = x + 1;\n return a;\n}", + metadata={'source': 'app/utils.js', 'content_type': 'functions_classes'} + ), + Document( + page_content="(y) => {\n const b = y * 2;\n return b;\n}", + metadata={'source': 'app/utils.js', 'content_type': 'functions_classes'} + ), + ] + mappings = parser.create_map_of_local_vars(docs) + assert len(mappings) == 0, ( + f"Expected 0 entries (anonymous functions skipped), got {len(mappings)}. " + "Empty function names cannot be looked up and should not be indexed." + ) + + def test_named_functions_still_indexed(self, parser): + """Named functions should still be indexed normally.""" + docs = [ + Document( + page_content="function add(x) {\n const result = x + 1;\n return result;\n}", + metadata={'source': 'app/utils.js', 'content_type': 'functions_classes'} + ), + Document( + page_content="function multiply(y) {\n const product = y * 2;\n return product;\n}", + metadata={'source': 'app/utils.js', 'content_type': 'functions_classes'} + ), + ] + mappings = parser.create_map_of_local_vars(docs) + assert len(mappings) == 2 + assert 'add@app/utils.js' in mappings + assert 'multiply@app/utils.js' in mappings + + +# ============================================================================= +# _is_exportable duplication +# ============================================================================= + +class TestIsExportableSharedLogic: + """_is_exportable_class and _is_exportable_function should share CommonJS checks.""" + + def test_class_commonjs_module_exports_direct(self, parser): + assert parser._is_exportable_class("MyClass", "module.exports = MyClass;") + + def test_class_commonjs_module_exports_object(self, parser): + assert parser._is_exportable_class("MyClass", "module.exports = { MyClass };") + + def test_class_commonjs_module_exports_property(self, parser): + assert parser._is_exportable_class("MyClass", "module.exports.MyClass = MyClass;") + + def test_class_commonjs_exports_property(self, parser): + assert parser._is_exportable_class("MyClass", "exports.MyClass = MyClass;") + + def test_function_commonjs_module_exports_direct(self, parser): + assert parser._is_exportable_function("myFunc", "module.exports = myFunc;") + + def test_function_commonjs_module_exports_object(self, parser): + assert parser._is_exportable_function("myFunc", "module.exports = { myFunc };") + + def test_function_commonjs_module_exports_property(self, parser): + assert parser._is_exportable_function("myFunc", "module.exports.myFunc = myFunc;") + + def test_function_commonjs_exports_property(self, parser): + assert parser._is_exportable_function("myFunc", "exports.myFunc = myFunc;") + + +# ============================================================================= +# Bug: is_package_imported split('from') corrupts identifiers containing "from" +# Lines 799, 822: line.split('from')[0] splits on substrings like "fromString" +# ============================================================================= + +class TestIsPackageImportedFromSplit: + """split('from') breaks identifiers that start with or contain 'from'.""" + + def test_es6_import_identifier_starting_with_from(self, parser): + """import { fromString } from 'packageurl-js' should detect 'fromString'.""" + code = "import { fromString } from 'packageurl-js';" + assert parser.is_package_imported(code, "fromString", "packageurl-js") + + def test_es6_import_identifier_from_inside_name(self, parser): + """import { transformData } from 'utils' should detect 'transformData'.""" + code = "import { transformData } from 'utils';" + assert parser.is_package_imported(code, "transformData", "utils") + + def test_reexport_identifier_starting_with_from(self, parser): + """export { fromBuffer } from 'uuid' should detect 'fromBuffer'.""" + code = "export { fromBuffer } from 'uuid';" + assert parser.is_package_imported(code, "fromBuffer", "uuid") + + def test_normal_import_still_works(self, parser): + """import { template } from 'lodash' should still work after fix.""" + code = "import { template } from 'lodash';" + assert parser.is_package_imported(code, "template", "lodash") + + +# ============================================================================= +# Bug: _get_function_calls called without code_documents, alias resolution dead +# Line 236: search_for_called_function doesn't pass code_documents +# ============================================================================= + +class TestAliasResolutionPassthrough: + """search_for_called_function must pass code_documents so alias resolution works.""" + + def test_aliased_import_detected_via_search_for_called_function(self, parser): + """When an aliased import exists, search_for_called_function should detect it.""" + caller_code = "function handler() {\n myAlias();\n}" + caller_doc = Document( + page_content=caller_code, + metadata={"source": "handlers/main.js"} + ) + + callee_code = "function original() { return 1; }" + callee_doc = Document( + page_content=callee_code, + metadata={"source": "node_modules/somelib/index.js"} + ) + + full_file_code = "import { original as myAlias } from 'somelib';\n\n" + caller_code + full_file_doc = Document( + page_content=full_file_code, + metadata={"source": "handlers/main.js"} + ) + + code_documents = {"handlers/main.js": full_file_doc} + + result = parser.search_for_called_function( + caller_function=caller_doc, + callee_function_name="original", + callee_function=callee_doc, + callee_function_package="somelib", + code_documents=code_documents, + type_documents=[], + callee_function_file_name="index.js", + fields_of_types={}, + functions_local_variables_index={}, + documents_of_functions=[], + ) + assert result is True, "Alias 'myAlias' for 'original' should be detected" + + +# ============================================================================= +# Bug: $ missing from regex lookbehind in _get_function_calls +# Lines 196, 199: (? 0, "template() should match" + + def test_jquery_dollar_qualified_call_detected(self, parser): + """$.ajax() is a call to ajax through qualifier $ — should be detected.""" + code = "function fetch() {\n $.ajax('/api');\n}" + doc = Document(page_content=code, metadata={"source": "app.js"}) + calls = parser._get_function_calls(doc, "ajax") + # $.ajax() is a legitimate call to ajax. The $ qualifier isn't in the + # qualifier char class [\w.?()], so the match is 'ajax' not '$.ajax'. + # This is correct — the call IS to ajax. + assert len(calls) > 0, "$.ajax() should be detected as a call to ajax" + + +# ============================================================================= +# Bug: get_package_names returns single-element list for JS +# chain_of_calls_retriever.py:593 accesses [1] → IndexError +# ============================================================================= + +class TestGetPackageNamesSingleElement: + """JS get_package_names returns single-element list, so [1] index crashes.""" + + def test_third_party_package_returns_single_element(self, parser): + """Third-party packages return ['package_name'] — only one element.""" + doc = Document(page_content="function foo() {}", + metadata={"source": "node_modules/lodash/index.js"}) + result = parser.get_package_names(doc) + assert result == ["lodash"] + assert len(result) == 1 + + def test_third_party_scoped_returns_single_element(self, parser): + """Scoped packages return ['@scope/pkg'] — still one element.""" + doc = Document(page_content="function foo() {}", + metadata={"source": "node_modules/@babel/core/index.js"}) + result = parser.get_package_names(doc) + assert result == ["@babel/core"] + assert len(result) == 1 + + def test_root_project_returns_single_element(self, parser): + """Root project files return ['root_project'] — single element.""" + doc = Document(page_content="function foo() {}", + metadata={"source": "src/app.js"}) + result = parser.get_package_names(doc) + assert result == ["root_project"] + assert len(result) == 1 + + def test_index_0_always_valid(self, parser): + """Accessing [0] should always work for any document.""" + for source in ["node_modules/express/index.js", "src/app.js", + "node_modules/@types/node/index.d.ts"]: + doc = Document(page_content="var x;", metadata={"source": source}) + names = parser.get_package_names(doc) + assert len(names) >= 1 + _ = names[0] # Should never raise + + def test_index_1_raises_for_js(self, parser): + """Accessing [1] should raise IndexError — this is the bug CCA hits.""" + doc = Document(page_content="function foo() {}", + metadata={"source": "node_modules/lodash/index.js"}) + result = parser.get_package_names(doc) + with pytest.raises(IndexError): + _ = result[1] + + +# ============================================================================= +# Bug: print_call_hierarchy doesn't handle empty function_name string +# chain_of_calls_retriever.py:702 — ValueError is caught but empty string isn't +# ============================================================================= + +class TestPrintCallHierarchyEmptyName: + """print_call_hierarchy catches ValueError but doesn't check for empty string.""" + + def test_get_function_name_empty_string_handled(self, parser): + """Documents where get_function_name returns '' should not crash hierarchy.""" + doc = Document( + page_content="// just a comment block\n/* nothing here */", + metadata={"source": "utils.js"} + ) + try: + name = parser.get_function_name(doc) + except ValueError: + name = None + + # The bug: if get_function_name returns '' instead of raising, + # print_call_hierarchy formats it as (package=...,function=,depth=0) + # which is a meaningless entry in the call hierarchy. + # The fix in chain_of_calls_retriever.py guards against empty strings. + assert name is None or isinstance(name, str) + + +# ============================================================================= +# _build_class_hierarchy index and _is_subclass_of optimization +# ============================================================================= + +class TestBuildClassHierarchy: + """Verify the class hierarchy index is built correctly from code_documents.""" + + def test_simple_es6_extends(self, parser): + docs = { + "a.js": Document(page_content="class Dog extends Animal { bark() {} }", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert "Dog" in hierarchy + extends_clause, parent = hierarchy["Dog"] + assert parent == "Animal" + assert "Animal" in extends_clause + + def test_mixin_extends(self, parser): + docs = { + "a.js": Document(page_content="class MyComponent extends EventEmitter(Base) { }", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert "MyComponent" in hierarchy + extends_clause, parent = hierarchy["MyComponent"] + assert parent == "Base" + assert "EventEmitter" in extends_clause + + def test_chained_mixin(self, parser): + docs = { + "a.js": Document(page_content="class Widget extends Draggable(Resizable(Component)) { }", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + extends_clause, parent = hierarchy["Widget"] + assert parent == "Component" + assert "Draggable" in extends_clause + assert "Resizable" in extends_clause + + def test_prototype_inherits(self, parser): + docs = { + "a.js": Document(page_content="util.inherits(ReadStream, EventEmitter);", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert "ReadStream" in hierarchy + extends_clause, parent = hierarchy["ReadStream"] + assert extends_clause is None + assert parent == "EventEmitter" + + def test_object_create_prototype(self, parser): + docs = { + "a.js": Document(page_content="Child.prototype = Object.create(Parent.prototype);", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert "Child" in hierarchy + assert hierarchy["Child"] == (None, "Parent") + + def test_set_prototype_of(self, parser): + docs = { + "a.js": Document(page_content="Object.setPrototypeOf(Sub.prototype, Super.prototype);", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert "Sub" in hierarchy + assert hierarchy["Sub"] == (None, "Super") + + def test_multiple_classes_across_files(self, parser): + docs = { + "a.js": Document(page_content="class A extends B { }", metadata={"source": "a.js"}), + "b.js": Document(page_content="class B extends C { }", metadata={"source": "b.js"}), + "c.js": Document(page_content="class C { }", metadata={"source": "c.js"}), + } + hierarchy = parser._build_class_hierarchy(docs) + assert hierarchy["A"] == ("B", "B") + assert hierarchy["B"] == ("C", "C") + assert "C" not in hierarchy + + def test_es6_takes_precedence_over_prototype(self, parser): + """If both ES6 and prototype patterns exist, ES6 wins (indexed first).""" + docs = { + "a.js": Document( + page_content="class X extends Y { }\nutil.inherits(X, Z);", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert hierarchy["X"][1] == "Y" + + def test_empty_documents(self, parser): + hierarchy = parser._build_class_hierarchy({}) + assert hierarchy == {} + + +class TestIsSubclassOfOptimized: + """Verify _is_subclass_of correctness with the hierarchy index.""" + + def test_direct_subclass(self, parser): + docs = { + "a.js": Document(page_content="class Dog extends Animal { }", + metadata={"source": "a.js"}) + } + assert parser._is_subclass_of("Dog", "Animal", docs) + + def test_not_subclass(self, parser): + docs = { + "a.js": Document(page_content="class Dog extends Animal { }", + metadata={"source": "a.js"}) + } + assert not parser._is_subclass_of("Dog", "Vehicle", docs) + + def test_transitive_chain(self, parser): + docs = { + "a.js": Document(page_content="class A extends B { }", metadata={"source": "a.js"}), + "b.js": Document(page_content="class B extends C { }", metadata={"source": "b.js"}), + } + assert parser._is_subclass_of("A", "C", docs) + + def test_mixin_match(self, parser): + docs = { + "a.js": Document(page_content="class X extends Mixin(Base) { }", + metadata={"source": "a.js"}) + } + assert parser._is_subclass_of("X", "Mixin", docs) + assert parser._is_subclass_of("X", "Base", docs) + + def test_circular_reference_no_infinite_loop(self, parser): + docs = { + "a.js": Document(page_content="class A extends B { }", metadata={"source": "a.js"}), + "b.js": Document(page_content="class B extends A { }", metadata={"source": "b.js"}), + } + assert not parser._is_subclass_of("A", "Z", docs) + + def test_empty_child_or_parent(self, parser): + docs = {"a.js": Document(page_content="class X extends Y { }", metadata={"source": "a.js"})} + assert not parser._is_subclass_of("", "Y", docs) + assert not parser._is_subclass_of("X", "", docs) + assert not parser._is_subclass_of(None, "Y", docs) + + def test_prototype_inheritance(self, parser): + docs = { + "a.js": Document(page_content="util.inherits(ReadStream, EventEmitter);", + metadata={"source": "a.js"}) + } + assert parser._is_subclass_of("ReadStream", "EventEmitter", docs) + + def test_hierarchy_cache_reused(self, parser): + """Same code_documents dict should reuse cached hierarchy.""" + docs = { + "a.js": Document(page_content="class A extends B { }", metadata={"source": "a.js"}) + } + parser._is_subclass_of("A", "B", docs) + cache_key_1 = parser._class_hierarchy_cache_key + + parser._is_subclass_of("A", "C", docs) + cache_key_2 = parser._class_hierarchy_cache_key + + assert cache_key_1 == cache_key_2 + + def test_different_docs_rebuilds_cache(self, parser): + docs1 = {"a.js": Document(page_content="class A extends B { }", metadata={"source": "a.js"})} + docs2 = {"a.js": Document(page_content="class X extends Y { }", metadata={"source": "a.js"})} + + parser._is_subclass_of("A", "B", docs1) + key1 = parser._class_hierarchy_cache_key + + parser._is_subclass_of("X", "Y", docs2) + key2 = parser._class_hierarchy_cache_key + + assert key1 != key2 + + +class TestGetParentOptimized: + """Verify _get_direct_parent, _get_prototype_parent, _get_parent with index.""" + + def test_get_direct_parent(self, parser): + docs = {"a.js": Document(page_content="class X extends Y { }", metadata={"source": "a.js"})} + assert parser._get_direct_parent("X", docs) == "Y" + + def test_get_direct_parent_mixin(self, parser): + docs = {"a.js": Document(page_content="class X extends Mixin(Base) { }", metadata={"source": "a.js"})} + assert parser._get_direct_parent("X", docs) == "Base" + + def test_get_direct_parent_not_found(self, parser): + docs = {"a.js": Document(page_content="class X { }", metadata={"source": "a.js"})} + assert parser._get_direct_parent("X", docs) is None + + def test_get_prototype_parent(self, parser): + docs = {"a.js": Document(page_content="util.inherits(Child, Parent);", metadata={"source": "a.js"})} + assert parser._get_prototype_parent("Child", docs) == "Parent" + + def test_get_prototype_parent_not_found(self, parser): + docs = {"a.js": Document(page_content="class X extends Y { }", metadata={"source": "a.js"})} + assert parser._get_prototype_parent("X", docs) is None + + def test_get_parent_prefers_es6(self, parser): + docs = {"a.js": Document( + page_content="class X extends Y { }\nutil.inherits(X, Z);", + metadata={"source": "a.js"})} + assert parser._get_parent("X", docs) == "Y" + + +# ============================================================================= +# Bug: _build_class_hierarchy uses \w+ which misses $ in JS identifiers +# ============================================================================= + +class TestDollarSignInClassHierarchy: + """$ is a valid JS identifier char — hierarchy builder must handle it.""" + + def test_dollar_prefixed_es6_class(self, parser): + """class $Component extends React.Component should be in the hierarchy.""" + docs = { + "a.js": Document(page_content="class $Component extends Base { }", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert "$Component" in hierarchy, f"$Component not found in hierarchy: {hierarchy}" + _, parent = hierarchy["$Component"] + assert parent == "Base" + + def test_dollar_prefixed_parent(self, parser): + """class Child extends $Base should resolve $Base as parent.""" + docs = { + "a.js": Document(page_content="class Child extends $Base { }", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert "Child" in hierarchy + _, parent = hierarchy["Child"] + assert parent == "$Base" + + def test_dollar_prefixed_mixin_parent(self, parser): + """class Foo extends Mixin($Bar) should resolve $Bar as innermost parent.""" + docs = { + "a.js": Document(page_content="class Foo extends Mixin($Bar) { }", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert "Foo" in hierarchy + _, parent = hierarchy["Foo"] + assert parent == "$Bar" + + def test_dollar_prefixed_prototype_inherits(self, parser): + """util.inherits($Stream, EventEmitter) should capture $Stream.""" + docs = { + "a.js": Document(page_content="util.inherits($Stream, EventEmitter);", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert "$Stream" in hierarchy + assert hierarchy["$Stream"] == (None, "EventEmitter") + + def test_is_subclass_of_with_dollar(self, parser): + """_is_subclass_of should work with $-prefixed class names.""" + docs = { + "a.js": Document(page_content="class $Widget extends Component { }", + metadata={"source": "a.js"}) + } + assert parser._is_subclass_of("$Widget", "Component", docs) + + def test_dollar_in_both_child_and_parent(self, parser): + """Both child and parent have $ prefix.""" + docs = { + "a.js": Document(page_content="class $Child extends $Parent { }", + metadata={"source": "a.js"}) + } + hierarchy = parser._build_class_hierarchy(docs) + assert "$Child" in hierarchy + _, parent = hierarchy["$Child"] + assert parent == "$Parent" + + +# ============================================================================= +# Bug: recursive _get_function_calls omits code_documents — chained alias broken +# ============================================================================= + +class TestChainedAliasResolution: + """Recursive _get_function_calls must pass code_documents for chained aliases.""" + + def test_alias_of_alias_es6(self, parser): + """import { orig as mid } then import { mid as final } — final() should match orig.""" + caller_code = "function handler() {\n final();\n}" + caller_doc = Document( + page_content=caller_code, + metadata={"source": "handler.js"} + ) + + full_file_code = ( + "import { orig as mid } from 'pkg';\n" + "import { mid as final } from './re-export';\n" + "\n" + caller_code + ) + full_file_doc = Document( + page_content=full_file_code, + metadata={"source": "handler.js"} + ) + + code_documents = {"handler.js": full_file_doc} + + calls = parser._get_function_calls(caller_doc, "orig", code_documents) + assert "final" in calls, ( + f"Chained alias orig→mid→final should resolve. Got: {calls}" + ) + + def test_alias_commonjs_chain(self, parser): + """const { orig: mid } = require('a') then const { mid: local } = require('b').""" + caller_code = "function run() {\n local();\n}" + caller_doc = Document( + page_content=caller_code, + metadata={"source": "run.js"} + ) + + full_file_code = ( + "const { orig: mid } = require('a');\n" + "const { mid: local } = require('b');\n" + "\n" + caller_code + ) + full_file_doc = Document( + page_content=full_file_code, + metadata={"source": "run.js"} + ) + + code_documents = {"run.js": full_file_doc} + + calls = parser._get_function_calls(caller_doc, "orig", code_documents) + assert "local" in calls, ( + f"Chained CommonJS alias orig→mid→local should resolve. Got: {calls}" + ) + diff --git a/tests/test_package_identifier.py b/tests/test_package_identifier.py new file mode 100644 index 000000000..844192f88 --- /dev/null +++ b/tests/test_package_identifier.py @@ -0,0 +1,461 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest + +from exploit_iq_commons.data_models.checker_status import EnumIdentifyResult, PackageCheckerStatus +from exploit_iq_commons.data_models.common import TargetPackage +from exploit_iq_commons.data_models.cve_intel import CveIntel, CveIntelRhsa, CveIntelNvd + +from vuln_analysis.utils.package_identifier import ( + PackageIdentifier, + _extract_rhel_version, + _interpret_fix_state, + _match_package_state_for_distro, +) + + +class TestExtractRhelVersion: + """Tests for _extract_rhel_version helper.""" + + def test_el7_extracts_7(self): + assert _extract_rhel_version("el7") == "7" + + def test_el8_extracts_8(self): + assert _extract_rhel_version("el8") == "8" + + def test_el10_extracts_10(self): + assert _extract_rhel_version("el10") == "10" + + def test_none_returns_none(self): + assert _extract_rhel_version(None) is None + + def test_empty_string_returns_none(self): + assert _extract_rhel_version("") is None + + def test_invalid_format_returns_none(self): + assert _extract_rhel_version("rhel7") is None + assert _extract_rhel_version("centos7") is None + + +class TestInterpretFixState: + """Tests for _interpret_fix_state helper.""" + + def test_not_affected_returns_no(self): + assert _interpret_fix_state("Not affected") == EnumIdentifyResult.NO + assert _interpret_fix_state("not affected") == EnumIdentifyResult.NO + assert _interpret_fix_state("NOT AFFECTED") == EnumIdentifyResult.NO + + def test_will_not_fix_returns_no(self): + assert _interpret_fix_state("Will not fix") == EnumIdentifyResult.NO + assert _interpret_fix_state("will not fix") == EnumIdentifyResult.NO + + def test_out_of_support_scope_returns_no(self): + assert _interpret_fix_state("Out of support scope") == EnumIdentifyResult.NO + assert _interpret_fix_state("out of support scope") == EnumIdentifyResult.NO + + def test_affected_returns_yes(self): + assert _interpret_fix_state("Affected") == EnumIdentifyResult.YES + assert _interpret_fix_state("affected") == EnumIdentifyResult.YES + + def test_fix_deferred_returns_yes(self): + assert _interpret_fix_state("Fix deferred") == EnumIdentifyResult.YES + assert _interpret_fix_state("fix deferred") == EnumIdentifyResult.YES + + def test_under_investigation_returns_yes(self): + assert _interpret_fix_state("Under investigation") == EnumIdentifyResult.YES + + def test_none_returns_none(self): + assert _interpret_fix_state(None) is None + + def test_empty_returns_none(self): + assert _interpret_fix_state("") is None + + def test_unknown_state_returns_none(self): + assert _interpret_fix_state("Fixed") is None + assert _interpret_fix_state("Unknown") is None + assert _interpret_fix_state("Pending") is None + + +class TestMatchPackageStateForDistro: + """Tests for _match_package_state_for_distro helper.""" + + @pytest.fixture + def rhel6_package_state(self): + return CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 6", + fix_state="Not affected", + package_name="libarchive", + cpe="cpe:/o:redhat:enterprise_linux:6", + ) + + @pytest.fixture + def rhel7_package_state(self): + return CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 7", + fix_state="Will not fix", + package_name="libarchive", + cpe="cpe:/o:redhat:enterprise_linux:7", + ) + + @pytest.fixture + def rhel8_package_state(self): + return CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 8", + fix_state="Affected", + package_name="libarchive", + cpe="cpe:/o:redhat:enterprise_linux:8", + ) + + def test_matches_by_cpe(self, rhel6_package_state, rhel7_package_state): + states = [rhel6_package_state, rhel7_package_state] + + result = _match_package_state_for_distro(states, "libarchive", "el7") + assert result is not None + assert result.fix_state == "Will not fix" + + def test_matches_el6(self, rhel6_package_state, rhel7_package_state): + states = [rhel6_package_state, rhel7_package_state] + + result = _match_package_state_for_distro(states, "libarchive", "el6") + assert result is not None + assert result.fix_state == "Not affected" + + def test_no_distro_returns_first_name_match(self, rhel6_package_state, rhel7_package_state): + states = [rhel6_package_state, rhel7_package_state] + + result = _match_package_state_for_distro(states, "libarchive", None) + assert result is not None + assert result == rhel6_package_state + + def test_no_package_match_returns_none(self, rhel7_package_state): + states = [rhel7_package_state] + + result = _match_package_state_for_distro(states, "curl", "el7") + assert result is None + + def test_no_distro_match_returns_none(self, rhel7_package_state): + states = [rhel7_package_state] + + result = _match_package_state_for_distro(states, "libarchive", "el9") + assert result is None + + def test_matches_by_product_name(self): + state = CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 7", + fix_state="Will not fix", + package_name="libarchive", + cpe=None, # No CPE, should match by product_name + ) + + result = _match_package_state_for_distro([state], "libarchive", "el7") + assert result is not None + assert result.fix_state == "Will not fix" + + def test_fedora_profile_skips_rhsa_matching(self, rhel6_package_state, rhel7_package_state, monkeypatch): + """When RPM_USER_TYPE=external (Fedora), RHSA matching is skipped.""" + monkeypatch.setenv("RPM_USER_TYPE", "external") + states = [rhel6_package_state, rhel7_package_state] + + result = _match_package_state_for_distro(states, "libarchive", None) + assert result is None + + +class TestPackageIdentifierWithFixState: + """Integration tests for PackageIdentifier with RHSA fix_state.""" + + def test_cve_2016_8687_rhel7_will_not_fix(self): + """RHSA Will not fix on RHEL 7 -> NOT_VUL even when NVD version would match.""" + target = TargetPackage( + name="libarchive", + version="3.1.2", + release="14.el7_9.1", + arch="x86_64", + ) + + intel = CveIntel( + vuln_id="CVE-2016-8687", + nvd=CveIntelNvd( + cve_id="CVE-2016-8687", + configurations=[ + CveIntelNvd.Configuration( + package="libarchive", + vendor="libarchive", + versionStartIncluding="3.2.1", + versionEndIncluding="3.2.1", + ), + ], + ), + rhsa=CveIntelRhsa( + package_state=[ + CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 6", + fix_state="Not affected", + package_name="libarchive", + cpe="cpe:/o:redhat:enterprise_linux:6", + ), + CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 7", + fix_state="Will not fix", + package_name="libarchive", + cpe="cpe:/o:redhat:enterprise_linux:7", + ), + ], + ), + ) + + identifier = PackageIdentifier(target) + status, result = identifier.identify(intel) + + assert result.is_target_package_affected == EnumIdentifyResult.NO + assert status == PackageCheckerStatus.PKG_IDENT_NOT_VUL + assert "Will not fix" in result.conclusion_reason + + def test_cve_2016_8687_rhel6_not_affected_continues_investigation(self): + """CVE-2016-8687 on RHEL 6 'Not affected' should continue investigation (status=OK). + + When RHSA says 'not affected', we continue investigation to verify the assessment + and catch potential RHSA errors. The rhsa_fix_state is preserved for downstream + conflict detection. + """ + target = TargetPackage( + name="libarchive", + version="3.0.3", + release="6.el6_10", + arch="x86_64", + ) + + intel = CveIntel( + vuln_id="CVE-2016-8687", + nvd=CveIntelNvd( + cve_id="CVE-2016-8687", + configurations=[ + CveIntelNvd.Configuration( + package="libarchive", + vendor="libarchive", + versionStartIncluding="3.2.1", + versionEndIncluding="3.2.1", + ), + ], + ), + rhsa=CveIntelRhsa( + package_state=[ + CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 6", + fix_state="Not affected", + package_name="libarchive", + cpe="cpe:/o:redhat:enterprise_linux:6", + ), + CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 7", + fix_state="Will not fix", + package_name="libarchive", + cpe="cpe:/o:redhat:enterprise_linux:7", + ), + ], + ), + ) + + identifier = PackageIdentifier(target) + status, result = identifier.identify(intel) + + assert result.is_target_package_affected == EnumIdentifyResult.NO + # "not affected" continues investigation instead of early exit + assert status == PackageCheckerStatus.OK + # rhsa_fix_state is preserved for conflict detection + assert result.rhsa_fix_state == "not affected" + # Verify conclusion_reason is populated with RHSA details + assert "RHSA fix_state" in result.conclusion_reason + assert "Not affected" in result.conclusion_reason + assert "libarchive" in result.conclusion_reason + assert "el6" in result.conclusion_reason + + def test_out_of_support_scope_early_exits(self): + """RHSA 'out of support scope' should early-exit with PKG_IDENT_NOT_VUL. + + Unlike 'not affected', 'out of support scope' means Red Hat won't provide + further assessment, so we don't continue investigation. + """ + target = TargetPackage( + name="oldlib", + version="1.0.0", + release="1.el5", + arch="x86_64", + ) + + intel = CveIntel( + vuln_id="CVE-2024-0002", + nvd=None, + rhsa=CveIntelRhsa( + package_state=[ + CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 5", + fix_state="Out of support scope", + package_name="oldlib", + cpe="cpe:/o:redhat:enterprise_linux:5", + ), + ], + ), + ) + + identifier = PackageIdentifier(target) + status, result = identifier.identify(intel) + + assert result.is_target_package_affected == EnumIdentifyResult.NO + # "out of support scope" still triggers early exit + assert status == PackageCheckerStatus.PKG_IDENT_NOT_VUL + assert result.rhsa_fix_state == "out of support scope" + + def test_falls_back_to_nvd_when_no_fix_state(self): + """When RHSA has no fix_state, should fall back to NVD version check.""" + target = TargetPackage( + name="somelib", + version="1.0.0", + release="1.el8", + arch="x86_64", + ) + + intel = CveIntel( + vuln_id="CVE-2024-0001", + nvd=CveIntelNvd( + cve_id="CVE-2024-0001", + configurations=[ + CveIntelNvd.Configuration( + package="somelib", + vendor="somevendor", + versionStartIncluding="1.0.0", + versionEndIncluding="1.5.0", + ), + ], + ), + rhsa=CveIntelRhsa( + package_state=[ + CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 8", + fix_state=None, # No fix_state, should fall back to NVD + package_name="somelib", + cpe="cpe:/o:redhat:enterprise_linux:8", + ), + ], + ), + ) + + identifier = PackageIdentifier(target) + status, result = identifier.identify(intel) + + # Should be YES because 1.0.0 is in NVD range [1.0.0, 1.5.0] + assert result.is_target_package_affected == EnumIdentifyResult.YES + assert status == PackageCheckerStatus.OK + + def test_nvd_version_outside_range_populates_conclusion_reason(self): + """When target version is outside NVD range, conclusion_reason should explain.""" + target = TargetPackage( + name="somelib", + version="2.0.0", # Outside the affected range + release="1.el8", + arch="x86_64", + ) + + intel = CveIntel( + vuln_id="CVE-2024-0001", + nvd=CveIntelNvd( + cve_id="CVE-2024-0001", + configurations=[ + CveIntelNvd.Configuration( + package="somelib", + vendor="somevendor", + versionStartIncluding="1.0.0", + versionEndExcluding="1.5.0", + ), + ], + ), + rhsa=CveIntelRhsa( + package_state=[ + CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 8", + fix_state=None, # No fix_state, should fall back to NVD + package_name="somelib", + cpe="cpe:/o:redhat:enterprise_linux:8", + ), + ], + ), + ) + + identifier = PackageIdentifier(target) + status, result = identifier.identify(intel) + + # Should be NO because 2.0.0 is outside NVD range [1.0.0, 1.5.0) + assert result.is_target_package_affected == EnumIdentifyResult.NO + assert status == PackageCheckerStatus.PKG_IDENT_NOT_VUL + # Verify conclusion_reason is populated with NVD version range details + assert "NVD" in result.conclusion_reason + assert "version" in result.conclusion_reason.lower() + assert "somelib" in result.conclusion_reason + assert "2.0.0" in result.conclusion_reason + assert ">=1.0.0" in result.conclusion_reason + assert "<1.5.0" in result.conclusion_reason + + +class TestPackageIdentifierRhsaScope: + """Step 1: RHSA package_state + affected_release scope gate.""" + + def test_affected_release_only_passes_scope(self): + target = TargetPackage( + name="curl", + version="7.76.1", + release="26.el9", + arch="x86_64", + ) + intel = CveIntel( + vuln_id="CVE-2024-TEST", + rhsa=CveIntelRhsa( + package_state=[], + affected_release=[ + {"package": "curl-7.76.1-23.el9", "product_name": "Red Hat Enterprise Linux 9"}, + ], + ), + ) + identifier = PackageIdentifier(target) + status, result = identifier.identify(intel) + + assert status == PackageCheckerStatus.OK + assert result.fixed_rpm_list + + def test_not_in_either_rhsa_bucket_is_cve_mismatch(self): + target = TargetPackage( + name="curl", + version="7.76.1", + release="26.el9", + arch="x86_64", + ) + intel = CveIntel( + vuln_id="CVE-2024-TEST", + rhsa=CveIntelRhsa( + package_state=[ + CveIntelRhsa.PackageState( + package_name="webkit2gtk3", + fix_state="Affected", + product_name="Red Hat Enterprise Linux 9", + ), + ], + affected_release=[ + {"package": "webkit2gtk3-2.42.5-1.el9"}, + ], + ), + ) + identifier = PackageIdentifier(target) + status, result = identifier.identify(intel) + + assert status == PackageCheckerStatus.PKG_IDENT_CVE_MISMATCH + assert result.is_target_package_affected == EnumIdentifyResult.NO diff --git a/tests/test_process_steps.py b/tests/test_process_steps.py new file mode 100644 index 000000000..a19f174d3 --- /dev/null +++ b/tests/test_process_steps.py @@ -0,0 +1,378 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Unit tests for _process_steps in cve_agent.py. + +Focus: tracker/graph alignment on fallback, initial_state construction, +semaphore usage, concurrent step processing. +""" + +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from langchain_core.messages import HumanMessage + +from vuln_analysis.functions.cve_agent import _process_steps +from vuln_analysis.functions.dispatcher import QuestionRouting +from vuln_analysis.functions.react_internals import ReachabilityRulesTracker +from vuln_analysis.functions.code_understanding_internals import CodeUnderstandingRulesTracker + + +@pytest.fixture +def mock_workflow_state(): + state = MagicMock() + state.cve_intel = [] + return state + + +@pytest.fixture +def mock_graph(): + graph = AsyncMock() + graph.ainvoke = AsyncMock(return_value={"input": "q", "output": "answer"}) + return graph + + +@pytest.fixture +def patch_externals(mock_workflow_state): + """Patch ctx_state, build_critical_context, dispatch_question, enrich_vulnerable_functions_from_patch, and AGENT_TRACER.""" + with ( + patch("vuln_analysis.functions.cve_agent.ctx_state") as mock_ctx, + patch("vuln_analysis.functions.cve_agent.build_critical_context") as mock_bcc, + patch("vuln_analysis.functions.cve_agent.dispatch_question") as mock_dispatch, + patch("vuln_analysis.functions.cve_agent.enrich_vulnerable_functions_from_patch", new_callable=AsyncMock) as mock_enrich, + patch("vuln_analysis.functions.cve_agent.AGENT_TRACER") as mock_tracer, + ): + mock_ctx.get.return_value = mock_workflow_state + mock_bcc.return_value = (["ctx_line"], [], ["vuln_func"]) + mock_tracer.push_active_function.return_value.__enter__ = MagicMock() + mock_tracer.push_active_function.return_value.__exit__ = MagicMock(return_value=False) + yield { + "ctx_state": mock_ctx, + "build_critical_context": mock_bcc, + "dispatch_question": mock_dispatch, + "enrich_patch": mock_enrich, + "tracer": mock_tracer, + } + + +class TestFallbackAlignment: + """When routed agent_type is not in agents dict, both graph AND tracker + must fall back to reachability.""" + + @pytest.mark.asyncio + async def test_cu_fallback_uses_reachability_graph(self, patch_externals, mock_graph): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="code_understanding", reason="config question" + ) + agents = {"reachability": mock_graph} + + await _process_steps(agents, MagicMock(), ["Is it configured?"], None, vuln_id="CVE-test") + + mock_graph.ainvoke.assert_called_once() + + @pytest.mark.asyncio + async def test_cu_fallback_uses_reachability_tracker(self, patch_externals, mock_graph): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="code_understanding", reason="config question" + ) + agents = {"reachability": mock_graph} + + await _process_steps(agents, MagicMock(), ["Is it configured?"], None, vuln_id="CVE-test") + + call_args = mock_graph.ainvoke.call_args[0][0] + tracker = call_args["rules_tracker"] + assert isinstance(tracker, ReachabilityRulesTracker) + assert not isinstance(tracker, CodeUnderstandingRulesTracker) + + @pytest.mark.asyncio + async def test_direct_route_uses_correct_tracker(self, patch_externals): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="code_understanding", reason="config question" + ) + cu_graph = AsyncMock() + cu_graph.ainvoke = AsyncMock(return_value={"input": "q", "output": "a"}) + reach_graph = AsyncMock() + agents = {"reachability": reach_graph, "code_understanding": cu_graph} + + await _process_steps(agents, MagicMock(), ["Is it configured?"], None, vuln_id="CVE-test") + + call_args = cu_graph.ainvoke.call_args[0][0] + tracker = call_args["rules_tracker"] + assert isinstance(tracker, CodeUnderstandingRulesTracker) + reach_graph.ainvoke.assert_not_called() + + @pytest.mark.asyncio + async def test_reachability_route_uses_reachability_tracker(self, patch_externals, mock_graph): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="reachability", reason="function call check" + ) + agents = {"reachability": mock_graph} + + await _process_steps(agents, MagicMock(), ["Is func reachable?"], None, vuln_id="CVE-test") + + call_args = mock_graph.ainvoke.call_args[0][0] + tracker = call_args["rules_tracker"] + assert isinstance(tracker, ReachabilityRulesTracker) + + +class TestInitialState: + """Verify initial_state dict has correct keys and values.""" + + @pytest.mark.asyncio + async def test_initial_state_keys(self, patch_externals, mock_graph): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="reachability", reason="test" + ) + agents = {"reachability": mock_graph} + + await _process_steps(agents, MagicMock(), ["test question"], None, vuln_id="CVE-test") + + call_args = mock_graph.ainvoke.call_args[0][0] + assert call_args["input"] == "test question" + assert call_args["step"] == 0 + assert call_args["max_steps"] == 10 + assert call_args["thought"] is None + assert call_args["observation"] is None + assert call_args["output"] == "waiting for the agent to respond" + assert isinstance(call_args["messages"][0], HumanMessage) + assert call_args["messages"][0].content == "test question" + + @pytest.mark.asyncio + async def test_precomputed_intel_passed(self, patch_externals, mock_graph): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="reachability", reason="test" + ) + patch_externals["build_critical_context"].return_value = ( + ["ctx_line"], [{"name": "pkg"}], ["vuln_func"] + ) + agents = {"reachability": mock_graph} + + await _process_steps(agents, MagicMock(), ["q"], None, vuln_id="CVE-test") + + call_args = mock_graph.ainvoke.call_args[0][0] + intel = call_args["precomputed_intel"] + assert intel == (["ctx_line"], [{"name": "pkg"}], ["vuln_func"]) + + @pytest.mark.asyncio + async def test_custom_max_iterations(self, patch_externals, mock_graph): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="reachability", reason="test" + ) + agents = {"reachability": mock_graph} + + await _process_steps(agents, MagicMock(), ["q"], None, max_iterations=25, vuln_id="CVE-test") + + call_args = mock_graph.ainvoke.call_args[0][0] + assert call_args["max_steps"] == 25 + + @pytest.mark.asyncio + async def test_graph_config_recursion_limit(self, patch_externals, mock_graph): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="reachability", reason="test" + ) + agents = {"reachability": mock_graph} + + await _process_steps(agents, MagicMock(), ["q"], None, vuln_id="CVE-test") + + config_arg = mock_graph.ainvoke.call_args[1]["config"] + assert config_arg == {"recursion_limit": 50} + + +class TestConcurrency: + """Test concurrent step processing and semaphore behavior.""" + + @pytest.mark.asyncio + async def test_multiple_steps_all_processed(self, patch_externals, mock_graph): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="reachability", reason="test" + ) + agents = {"reachability": mock_graph} + steps = ["q1", "q2", "q3"] + + raw_results, questions = await _process_steps(agents, MagicMock(), steps, None, vuln_id="CVE-test") + + assert len(raw_results) == 3 + assert mock_graph.ainvoke.call_count == 3 + + @pytest.mark.asyncio + async def test_semaphore_limits_concurrency(self, patch_externals): + max_concurrent = 0 + current_concurrent = 0 + + async def slow_invoke(state, config=None): + nonlocal max_concurrent, current_concurrent + current_concurrent += 1 + max_concurrent = max(max_concurrent, current_concurrent) + await asyncio.sleep(0.01) + current_concurrent -= 1 + return {"input": state["input"], "output": "done"} + + mock_graph = AsyncMock() + mock_graph.ainvoke = slow_invoke + + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="reachability", reason="test" + ) + agents = {"reachability": mock_graph} + semaphore = asyncio.Semaphore(2) + + await _process_steps(agents, MagicMock(), ["q1", "q2", "q3", "q4"], semaphore, vuln_id="CVE-test") + + assert max_concurrent <= 2 + + @pytest.mark.asyncio + async def test_no_semaphore_allows_full_concurrency(self, patch_externals): + max_concurrent = 0 + current_concurrent = 0 + + async def slow_invoke(state, config=None): + nonlocal max_concurrent, current_concurrent + current_concurrent += 1 + max_concurrent = max(max_concurrent, current_concurrent) + await asyncio.sleep(0.01) + current_concurrent -= 1 + return {"input": state["input"], "output": "done"} + + mock_graph = AsyncMock() + mock_graph.ainvoke = slow_invoke + + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="reachability", reason="test" + ) + agents = {"reachability": mock_graph} + + await _process_steps(agents, MagicMock(), ["q1", "q2", "q3", "q4"], None, vuln_id="CVE-test") + + assert max_concurrent == 4 + + @pytest.mark.asyncio + async def test_exception_in_step_returned_not_raised(self, patch_externals): + mock_graph = AsyncMock() + mock_graph.ainvoke = AsyncMock(side_effect=RuntimeError("graph failed")) + + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="reachability", reason="test" + ) + agents = {"reachability": mock_graph} + + raw_results, questions = await _process_steps(agents, MagicMock(), ["q1"], None, vuln_id="CVE-test") + + assert len(raw_results) == 1 + assert isinstance(raw_results[0], RuntimeError) + + @pytest.mark.asyncio + async def test_empty_steps_returns_empty(self, patch_externals, mock_graph): + agents = {"reachability": mock_graph} + raw_results, questions = await _process_steps(agents, MagicMock(), [], None, vuln_id="CVE-test") + assert raw_results == [] + assert questions == [] + + +class TestSyntheticReachabilityQuestion: + """Synthetic reachability question is injected whenever candidate_packages is + non-empty and no checklist question routes to reachability.""" + + @pytest.mark.asyncio + async def test_synthetic_question_injected_with_vulnerable_functions(self, patch_externals, mock_graph): + """When vulnerable_functions is non-empty (advisory-sourced), synthetic question is injected.""" + patch_externals["build_critical_context"].return_value = ( + ["ctx"], [{"name": "xstream"}], ["convert"] + ) + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="code_understanding", reason="config question" + ) + agents = {"reachability": mock_graph, "code_understanding": AsyncMock( + ainvoke=AsyncMock(return_value={"input": "q", "output": "a"}) + )} + + await _process_steps(agents, MagicMock(), ["Is it configured?"], None, vuln_id="CVE-test") + + assert mock_graph.ainvoke.call_count == 1 + call_args = mock_graph.ainvoke.call_args[0][0] + assert "reachable" in call_args["input"] + + @pytest.mark.asyncio + async def test_synthetic_question_injected_without_vulnerable_functions(self, patch_externals, mock_graph): + """When vulnerable_functions is empty but candidate_packages exists, synthetic question is still injected.""" + patch_externals["build_critical_context"].return_value = ( + ["ctx"], [{"name": "moby"}], [] + ) + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="code_understanding", reason="config question" + ) + agents = {"reachability": mock_graph, "code_understanding": AsyncMock( + ainvoke=AsyncMock(return_value={"input": "q", "output": "a"}) + )} + + await _process_steps(agents, MagicMock(), ["Is it configured?"], None, vuln_id="CVE-test") + + assert mock_graph.ainvoke.call_count == 1 + call_args = mock_graph.ainvoke.call_args[0][0] + assert "moby" in call_args["input"] + assert "reachable" in call_args["input"] + + @pytest.mark.asyncio + async def test_no_synthetic_question_without_candidate_packages(self, patch_externals, mock_graph): + """When candidate_packages is empty, no synthetic question regardless of vuln functions.""" + patch_externals["build_critical_context"].return_value = ( + ["ctx"], [], ["convert"] + ) + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="code_understanding", reason="config question" + ) + cu_graph = AsyncMock() + cu_graph.ainvoke = AsyncMock(return_value={"input": "q", "output": "a"}) + agents = {"reachability": mock_graph, "code_understanding": cu_graph} + + await _process_steps(agents, MagicMock(), ["Is it configured?"], None, vuln_id="CVE-test") + + mock_graph.ainvoke.assert_not_called() + + @pytest.mark.asyncio + async def test_no_synthetic_when_reachability_already_routed(self, patch_externals, mock_graph): + """When a checklist question already routes to reachability, no synthetic injection.""" + patch_externals["build_critical_context"].return_value = ( + ["ctx"], [{"name": "xstream"}], ["convert"] + ) + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="reachability", reason="reachability check" + ) + agents = {"reachability": mock_graph} + + await _process_steps(agents, MagicMock(), ["Is convert reachable?"], None, vuln_id="CVE-test") + + assert mock_graph.ainvoke.call_count == 1 + call_args = mock_graph.ainvoke.call_args[0][0] + assert call_args["input"] == "Is convert reachable?" + + +class TestTracingSpan: + """Test that the tracing span receives actual_type, not routed type.""" + + @pytest.mark.asyncio + async def test_span_logs_actual_type_on_fallback(self, patch_externals, mock_graph): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="code_understanding", reason="config" + ) + agents = {"reachability": mock_graph} + + await _process_steps(agents, MagicMock(), ["q"], None, vuln_id="CVE-test") + + call_args = patch_externals["tracer"].push_active_function.call_args + assert "[reachability]" in call_args[1]["input_data"] + + @pytest.mark.asyncio + async def test_span_logs_routed_type_when_available(self, patch_externals): + patch_externals["dispatch_question"].return_value = QuestionRouting( + agent_type="code_understanding", reason="config" + ) + cu_graph = AsyncMock() + cu_graph.ainvoke = AsyncMock(return_value={"input": "q", "output": "a"}) + agents = {"reachability": MagicMock(), "code_understanding": cu_graph} + + await _process_steps(agents, MagicMock(), ["q"], None, vuln_id="CVE-test") + + call_args = patch_externals["tracer"].push_active_function.call_args + assert "[code_understanding]" in call_args[1]["input_data"] diff --git a/tests/test_python_segmenter.py b/tests/test_python_segmenter.py new file mode 100644 index 000000000..d2158cafa --- /dev/null +++ b/tests/test_python_segmenter.py @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from exploit_iq_commons.utils.python_segmenters_with_classes_methods import ( + is_python2_code, + PythonSegmenterWithClassesMethods, +) + +@pytest.mark.parametrize( + "code,expected,description", + [ + # Python 2 patterns (should return True) + # Simple exception with comma syntax + ("except Exception, e:", True, "py2 simple except with comma"), + ("except ValueError, err:", True, "py2 except ValueError with comma"), + (" except Exception, e:", True, "py2 indented except with comma"), + # Dotted module exception names + ("except module.Error, e:", True, "py2 dotted module exception"), + ("except urllib2.URLError, e:", True, "py2 urllib2 exception"), + ("except xml.parsers.expat.ExpatError, e:", True, "py2 deeply nested module exception"), + # Tuple of exceptions with comma syntax (Python 2) + ("except (IOError, OSError), e:", True, "py2 tuple exceptions with comma"), + ("except (IOError, OSError, KeyError), e:", True, "py2 tuple exceptions with comma"), + ("except (ValueError, TypeError), err:", True, "py2 tuple exceptions with variable"), + (" except (KeyError, IndexError), exc:", True, "py2 indented tuple exceptions"), + # Print statement (no parentheses) + ('print "hello"', True, "py2 print statement with double quotes"), + ("print 'hello'", True, "py2 print statement with single quotes"), + ('print "hello", "world"', True, "py2 print statement with multiple args"), + (" print 'indented'", True, "py2 indented print statement"), + # raw_input function + ('raw_input("prompt")', True, "py2 raw_input call"), + ("raw_input('Enter: ')", True, "py2 raw_input with single quotes"), + ("x = raw_input()", True, "py2 raw_input assignment"), + # Raise with comma syntax + ('raise Exception, "error message"', True, "py2 raise with comma and string"), + ("raise ValueError, msg", True, "py2 raise with comma and variable"), + (" raise TypeError, 'error'", True, "py2 indented raise with comma"), + # Shebang with python2 + ("#!/usr/bin/python2\nprint 'hello'", True, "py2 shebang with python2"), + ("#!/usr/bin/env python2\nx = 1", True, "py2 env shebang with python2"), + ("#!/usr/bin/python2.7\npass", True, "py2 shebang with python2.7"), + # Python 3 patterns (should return False) + # Python 3 tuple exceptions (no variable after) + ("except (KeyError, ValueError):", False, "py3 tuple exceptions"), + ("except (IOError, OSError):", False, "py3 IO tuple exceptions"), + (" except (TypeError, AttributeError):", False, "py3 indented tuple exceptions"), + # Python 3 'as' syntax + ("except Exception as e:", False, "py3 except as syntax"), + ("except ValueError as err:", False, "py3 ValueError as syntax"), + (" except KeyError as exc:", False, "py3 indented as syntax"), + # Python 3 dotted exception with 'as' + ("except module.Error as e:", False, "py3 dotted exception as"), + ("except urllib.error.URLError as e:", False, "py3 urllib exception as"), + # Print function + ('print("hello")', False, "py3 print function"), + ("print('hello', 'world')", False, "py3 print function multiple args"), + ("print()", False, "py3 empty print function"), + # Input function (Python 3) + ('input("prompt")', False, "py3 input function"), + ("x = input()", False, "py3 input assignment"), + # Raise with parentheses + ('raise Exception("error")', False, "py3 raise with parentheses"), + ("raise ValueError('message')", False, "py3 raise ValueError with parens"), + # Modern Python 3 code + ("def func() -> int:", False, "py3 type hints"), + ("x: int = 5", False, "py3 variable annotation"), + ("async def coro():", False, "py3 async function"), + ("f'hello {name}'", False, "py3 f-string"), + # Python 3 shebang + ("#!/usr/bin/python3\nprint('hello')", False, "py3 python3 shebang"), + ("#!/usr/bin/env python3\nx = 1", False, "py3 env python3 shebang"), + # Empty or minimal code + ("pass", False, "py3 pass statement"), + ("x = 1", False, "py3 simple assignment"), + ("import os", False, "py3 import statement"), + ], +) +def test_is_python2_code(code: str, expected: bool, description: str): + """Test that Python 2/3 patterns are correctly detected.""" + result = is_python2_code(code) + assert result is expected, f"Expected {expected} for {description}, got {result}" \ No newline at end of file diff --git a/tests/test_python_version_detection.py b/tests/test_python_version_detection.py new file mode 100644 index 000000000..85f7dad24 --- /dev/null +++ b/tests/test_python_version_detection.py @@ -0,0 +1,273 @@ +import textwrap +from pathlib import Path +from unittest.mock import patch + +import pytest + +from exploit_iq_commons.utils.dep_tree import PythonDependencyTreeBuilder + + +@pytest.fixture +def builder(): + return object.__new__(PythonDependencyTreeBuilder) + + +class TestExtractVersionFromSpecifier: + + @pytest.mark.parametrize("specifier, expected", [ + ("==3.9", "3.9"), + ("==3.9.0", "3.9.0"), + (">=3.8", "3.8"), + (">=3.8,<4.0", "3.8"), + (">=3.8,<3.12", "3.8"), + (">=3.8,!=3.9", "3.8"), + ("~=3.7", "3.7"), + ("~=3.7.2", "3.7.2"), + (">3.8", "3.9"), + ("<3", "2.7"), + ("<3.0", "2.7"), + (">=2.7,<3", "2.7"), + (">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,<3", "2.7"), + ("", None), + ("!=3.6", None), + ]) + def test_specifier_extraction(self, builder, specifier, expected): + assert builder.extract_version_from_specifier(specifier) == expected + + +class TestExtractVersionFromPyprojectToml: + + def test_pep621_requires_python_gte(self, builder): + content = textwrap.dedent("""\ + [project] + requires-python = ">=3.9" + """) + assert builder.extract_version_from_pyproject_toml(content) == "3.9" + + def test_pep621_requires_python_exact(self, builder): + content = textwrap.dedent("""\ + [project] + requires-python = "==3.11" + """) + assert builder.extract_version_from_pyproject_toml(content) == "3.11" + + def test_poetry_python_constraint(self, builder): + content = textwrap.dedent("""\ + [tool.poetry.dependencies] + python = "^3.8" + """) + assert builder.extract_version_from_pyproject_toml(content) == "3.8" + + def test_python2_upper_bound(self, builder): + content = textwrap.dedent("""\ + [project] + requires-python = ">=2.7,<3" + """) + assert builder.extract_version_from_pyproject_toml(content) == "2.7" + + def test_no_python_constraint_returns_none(self, builder): + content = textwrap.dedent("""\ + [project] + name = "myapp" + """) + assert builder.extract_version_from_pyproject_toml(content) is None + + def test_malformed_toml_returns_none(self, builder): + assert builder.extract_version_from_pyproject_toml("{ not valid toml") is None + + +class TestExtractVersionFromSetupPy: + + def test_python_requires_gte(self, builder): + content = textwrap.dedent("""\ + from setuptools import setup + setup( + name="myapp", + python_requires=">=3.8", + ) + """) + assert builder.extract_version_from_setup_py(content) == "3.8" + + def test_python_requires_exact(self, builder): + content = textwrap.dedent("""\ + from setuptools import setup + setup(python_requires="==3.9") + """) + assert builder.extract_version_from_setup_py(content) == "3.9" + + def test_python_requires_py2(self, builder): + content = textwrap.dedent("""\ + from setuptools import setup + setup(python_requires=">=2.7,<3") + """) + assert builder.extract_version_from_setup_py(content) == "2.7" + + def test_classifiers_fallback_highest_version(self, builder): + content = textwrap.dedent("""\ + from setuptools import setup + setup( + classifiers=[ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + ], + ) + """) + assert builder.extract_version_from_setup_py(content) == "3.9" + + def test_classifiers_python2(self, builder): + content = textwrap.dedent("""\ + from setuptools import setup + setup( + classifiers=[ + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + ], + ) + """) + assert builder.extract_version_from_setup_py(content) == "2.7" + + def test_no_version_info_returns_none(self, builder): + content = textwrap.dedent("""\ + from setuptools import setup + setup(name="myapp") + """) + assert builder.extract_version_from_setup_py(content) is None + + def test_syntax_error_returns_none(self, builder): + assert builder.extract_version_from_setup_py("def broken(") is None + + +class TestExtractVersionFromReadmeMd: + + def test_single_version_hint(self, builder): + content = "Requires Python 3.9 or above." + assert builder.extract_version_from_readme_md(content) == "3.9" + + def test_multiple_hints_returns_highest(self, builder): + content = textwrap.dedent("""\ + Supports Python 3.8 and above. + Tested on Python 3.11. + """) + assert builder.extract_version_from_readme_md(content) == "3.11" + + def test_python2_hint(self, builder): + content = "Works with Python 2.7." + assert builder.extract_version_from_readme_md(content) == "2.7" + + def test_no_hint_returns_none(self, builder): + assert builder.extract_version_from_readme_md("No version info here.") is None + + +class TestExtractVersionFromPythonVersionFile: + + @pytest.mark.parametrize("content, expected", [ + ("3.9.7\n", "3.9"), + ("3.9\n", "3.9"), + ("2.7.18\n", "2.7"), + ("3.11.0\n", "3.11"), + ("\n3.9\n", "3.9"), + ("pypy3.9\n", None), + ("", None), + ]) + def test_python_version_file(self, builder, content, expected): + assert builder.extract_version_from_python_version_file(content) == expected + + +class TestExtractVersionFromSetupCfg: + + def test_python_requires_gte(self, builder): + content = textwrap.dedent("""\ + [options] + python_requires = >=3.8 + """) + assert builder.extract_version_from_setup_cfg(content) == "3.8" + + def test_python_requires_py2(self, builder): + content = textwrap.dedent("""\ + [options] + python_requires = >=2.7,<3 + """) + assert builder.extract_version_from_setup_cfg(content) == "2.7" + + def test_no_python_requires_returns_none(self, builder): + content = "[metadata]\nname = myapp\n" + assert builder.extract_version_from_setup_cfg(content) is None + + +class TestExtractVersionFromPipfile: + + def test_python_version(self, builder): + content = textwrap.dedent("""\ + [requires] + python_version = "3.9" + """) + assert builder.extract_version_from_pipfile(content) == "3.9" + + def test_python_full_version(self, builder): + content = textwrap.dedent("""\ + [requires] + python_full_version = "3.9.7" + """) + assert builder.extract_version_from_pipfile(content) == "3.9" + + def test_no_requires_section_returns_none(self, builder): + content = "[packages]\nrequests = \"*\"\n" + assert builder.extract_version_from_pipfile(content) is None + + +class TestDeterminePythonVersion: + + def _make_repo(self, tmp_path, files: dict) -> Path: + for name, content in files.items(): + p = tmp_path / name + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + return tmp_path + + def test_python_version_file_wins_over_pyproject(self, builder, tmp_path): + repo = self._make_repo(tmp_path, { + ".python-version": "3.10\n", + "pyproject.toml": "[project]\nrequires-python = \">=3.9\"\n", + }) + assert builder.determine_python_version(str(repo)) == "3.10" + + def test_pyproject_used_when_no_python_version_file(self, builder, tmp_path): + repo = self._make_repo(tmp_path, { + "pyproject.toml": "[project]\nrequires-python = \">=3.9\"\n", + }) + assert builder.determine_python_version(str(repo)) == "3.9" + + def test_setup_cfg_used_when_no_pyproject(self, builder, tmp_path): + repo = self._make_repo(tmp_path, { + "setup.cfg": "[options]\npython_requires = >=3.8\n", + }) + assert builder.determine_python_version(str(repo)) == "3.8" + + def test_setup_py_used_as_fallback(self, builder, tmp_path): + repo = self._make_repo(tmp_path, { + "setup.py": "from setuptools import setup\nsetup(python_requires='>=3.7')\n", + }) + assert builder.determine_python_version(str(repo)) == "3.7" + + def test_pipfile_used_as_fallback(self, builder, tmp_path): + repo = self._make_repo(tmp_path, { + "Pipfile": "[requires]\npython_version = \"3.9\"\n", + }) + assert builder.determine_python_version(str(repo)) == "3.9" + + def test_returns_none_when_nothing_found(self, builder, tmp_path): + repo = self._make_repo(tmp_path, {"README.md": "No version info.\n"}) + assert builder.determine_python_version(str(repo)) is None + + def test_ignores_nested_pyproject_in_tests_dir(self, builder, tmp_path): + repo = self._make_repo(tmp_path, { + "tests/fixtures/pyproject.toml": "[project]\nrequires-python = \">=3.6\"\n", + }) + assert builder.determine_python_version(str(repo)) is None + + def test_python2_project(self, builder, tmp_path): + repo = self._make_repo(tmp_path, { + "setup.py": "from setuptools import setup\nsetup(python_requires='>=2.7,<3')\n", + }) + assert builder.determine_python_version(str(repo)) == "2.7" diff --git a/tests/test_reachability_agent.py b/tests/test_reachability_agent.py new file mode 100644 index 000000000..590a80426 --- /dev/null +++ b/tests/test_reachability_agent.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for ReachabilityAgent: get_tools, create_rules_tracker, +agent_type, should_truncate_tool_output.""" + +import pytest +from unittest.mock import MagicMock + +from agent_test_helpers import MockTool, ALL_TOOLS, make_builder, make_config, make_state +from vuln_analysis.functions.reachability_agent import ReachabilityAgent +from vuln_analysis.functions.react_internals import ReachabilityRulesTracker +from vuln_analysis.tools.tool_names import ToolNames + + +def _make_reachability_agent(tools=None): + mock_llm = MagicMock() + mock_llm.with_structured_output = MagicMock(return_value=MagicMock()) + config = MagicMock() + config.max_iterations = 10 + return ReachabilityAgent(tools=tools or [], llm=mock_llm, config=config) + + +class TestGetTools: + """ReachabilityAgent.get_tools selects reachability tools and filters by availability.""" + + def test_keeps_reachability_tools(self): + builder = make_builder() + config = make_config() + state = make_state() + result = ReachabilityAgent.get_tools(builder, config, state) + result_names = {t.name for t in result} + assert ToolNames.FUNCTION_LOCATOR in result_names + assert ToolNames.CALL_CHAIN_ANALYZER in result_names + assert ToolNames.CODE_KEYWORD_SEARCH in result_names + assert ToolNames.CVE_WEB_SEARCH in result_names + + def test_excludes_cu_only_tools(self): + builder = make_builder() + config = make_config() + state = make_state() + result = ReachabilityAgent.get_tools(builder, config, state) + result_names = {t.name for t in result} + assert ToolNames.CONFIGURATION_SCANNER not in result_names + assert ToolNames.IMPORT_USAGE_ANALYZER not in result_names + + def test_excludes_container_analysis_data(self): + builder = make_builder() + config = make_config() + state = make_state() + result = ReachabilityAgent.get_tools(builder, config, state) + result_names = {t.name for t in result} + assert ToolNames.CONTAINER_ANALYSIS_DATA not in result_names + + def test_keeps_all_8_reachability_tools(self): + builder = make_builder() + config = make_config() + state = make_state() + result = ReachabilityAgent.get_tools(builder, config, state) + assert len(result) == 8 + + def test_empty_builder_returns_empty(self): + builder = make_builder(tools=[]) + config = make_config() + state = make_state() + result = ReachabilityAgent.get_tools(builder, config, state) + assert result == [] + + def test_preserves_tool_order(self): + ordered_tools = [ + MockTool(ToolNames.CVE_WEB_SEARCH), + MockTool(ToolNames.FUNCTION_LOCATOR), + MockTool(ToolNames.CALL_CHAIN_ANALYZER), + ] + builder = make_builder(tools=ordered_tools) + config = make_config() + state = make_state() + result = ReachabilityAgent.get_tools(builder, config, state) + assert [t.name for t in result] == [ + ToolNames.CVE_WEB_SEARCH, + ToolNames.FUNCTION_LOCATOR, + ToolNames.CALL_CHAIN_ANALYZER, + ] + + def test_unknown_tools_excluded(self): + tools = [MockTool(ToolNames.FUNCTION_LOCATOR), MockTool("Some Future Tool")] + builder = make_builder(tools=tools) + config = make_config() + state = make_state() + result = ReachabilityAgent.get_tools(builder, config, state) + assert len(result) == 1 + assert result[0].name == ToolNames.FUNCTION_LOCATOR + + +class TestGetToolsAvailability: + """get_tools filters out tools whose infrastructure prerequisites are not met.""" + + def test_filters_code_semantic_search_when_no_vdb(self): + builder = make_builder() + config = make_config() + state = make_state(code_vdb_path=None) + result = ReachabilityAgent.get_tools(builder, config, state) + assert ToolNames.CODE_SEMANTIC_SEARCH not in {t.name for t in result} + + def test_filters_docs_semantic_search_when_no_vdb(self): + builder = make_builder() + config = make_config() + state = make_state(doc_vdb_path=None) + result = ReachabilityAgent.get_tools(builder, config, state) + assert ToolNames.DOCS_SEMANTIC_SEARCH not in {t.name for t in result} + + def test_filters_code_keyword_search_when_no_index(self): + builder = make_builder() + config = make_config() + state = make_state(code_index_path=None) + result = ReachabilityAgent.get_tools(builder, config, state) + assert ToolNames.CODE_KEYWORD_SEARCH not in {t.name for t in result} + + def test_filters_transitive_tools_when_no_index(self): + builder = make_builder() + config = make_config() + state = make_state(code_index_path=None) + result = ReachabilityAgent.get_tools(builder, config, state) + result_names = {t.name for t in result} + assert ToolNames.CALL_CHAIN_ANALYZER not in result_names + assert ToolNames.FUNCTION_CALLER_FINDER not in result_names + assert ToolNames.FUNCTION_LOCATOR not in result_names + + def test_filters_cve_web_search_when_disabled(self): + builder = make_builder() + config = make_config(cve_web_search_enabled=False) + state = make_state() + result = ReachabilityAgent.get_tools(builder, config, state) + assert ToolNames.CVE_WEB_SEARCH not in {t.name for t in result} + + def test_filters_transitive_tools_when_disabled(self): + builder = make_builder() + config = make_config(transitive_search_tool_enabled=False) + state = make_state() + result = ReachabilityAgent.get_tools(builder, config, state) + result_names = {t.name for t in result} + assert ToolNames.CALL_CHAIN_ANALYZER not in result_names + assert ToolNames.FUNCTION_CALLER_FINDER not in result_names + assert ToolNames.FUNCTION_LOCATOR not in result_names + + def test_version_finder_always_kept(self): + builder = make_builder() + config = make_config() + state = make_state(code_vdb_path=None, doc_vdb_path=None, code_index_path=None) + result = ReachabilityAgent.get_tools(builder, config, state) + assert ToolNames.FUNCTION_LIBRARY_VERSION_FINDER in {t.name for t in result} + + +class TestReachabilityDuplicateCall: + + def test_duplicate_call_blocked(self): + tracker = ReachabilityRulesTracker() + tracker.set_allowed_tools(["Function Locator"]) + tracker.set_target_package("commons-beanutils") + tracker.check_thought_behavior("Function Locator", "commons-beanutils,getProperty", ["result"]) + violated, msg = tracker.check_thought_behavior("Function Locator", "commons-beanutils,getProperty", ["result"]) + assert violated is True + assert "already called" in msg + + +class TestCreateRulesTracker: + + def test_returns_reachability_rules_tracker(self): + tracker = ReachabilityAgent.create_rules_tracker() + assert isinstance(tracker, ReachabilityRulesTracker) + + def test_returns_fresh_instance_each_call(self): + t1 = ReachabilityAgent.create_rules_tracker() + t2 = ReachabilityAgent.create_rules_tracker() + assert t1 is not t2 + + +class TestAgentType: + + def test_agent_type_is_reachability(self): + agent = _make_reachability_agent() + assert agent.agent_type == "reachability" + + +class TestShouldTruncateToolOutput: + + def test_true_for_java(self): + agent = _make_reachability_agent() + assert agent.should_truncate_tool_output({"ecosystem": "java"}, "any_tool") is True + + def test_true_for_java_uppercase(self): + agent = _make_reachability_agent() + assert agent.should_truncate_tool_output({"ecosystem": "Java"}, "any_tool") is True + + def test_false_for_go(self): + agent = _make_reachability_agent() + assert agent.should_truncate_tool_output({"ecosystem": "go"}, "any_tool") is False + + def test_false_for_python(self): + agent = _make_reachability_agent() + assert agent.should_truncate_tool_output({"ecosystem": "python"}, "any_tool") is False + + def test_false_for_empty_ecosystem(self): + agent = _make_reachability_agent() + assert agent.should_truncate_tool_output({"ecosystem": ""}, "any_tool") is False + + def test_false_when_ecosystem_missing(self): + agent = _make_reachability_agent() + assert agent.should_truncate_tool_output({}, "any_tool") is False + + +class TestInit: + + def test_creates_fifth_classification_llm(self): + mock_llm = MagicMock() + mock_llm.with_structured_output = MagicMock(return_value=MagicMock()) + config = MagicMock() + config.max_iterations = 10 + agent = ReachabilityAgent(tools=[], llm=mock_llm, config=config) + assert mock_llm.with_structured_output.call_count == 5 + assert hasattr(agent, "_classification_llm") diff --git a/tests/test_react_internals_rules.py b/tests/test_react_internals_rules.py new file mode 100644 index 000000000..73121eb5c --- /dev/null +++ b/tests/test_react_internals_rules.py @@ -0,0 +1,418 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from vuln_analysis.functions.react_internals import ( + BaseRulesTracker, + ReachabilityRulesTracker, + AgentState, + _find_image_matching_candidate, +) + + +class TestBaseRulesTracker: + def test_init_defaults(self): + tracker = BaseRulesTracker() + assert tracker.action_history == {} + assert tracker.target_package is None + assert tracker.allowed_tools == [] + + def test_set_allowed_tools(self): + tracker = BaseRulesTracker() + tools = ["Function Locator", "Code Keyword Search"] + tracker.set_allowed_tools(tools) + assert tracker.allowed_tools == tools + + def test_set_target_package(self): + tracker = BaseRulesTracker() + tracker.set_target_package("commons-beanutils") + assert tracker.target_package == "commons-beanutils" + + def test_is_empty_result_empty_list(self): + assert BaseRulesTracker._is_empty_result([]) is True + + def test_is_empty_result_bracket_string(self): + assert BaseRulesTracker._is_empty_result("[]") is True + + def test_is_empty_result_empty_string(self): + assert BaseRulesTracker._is_empty_result("") is True + + def test_is_empty_result_whitespace(self): + assert BaseRulesTracker._is_empty_result(" ") is True + + def test_is_empty_result_none(self): + assert BaseRulesTracker._is_empty_result(None) is False + + def test_is_empty_result_int(self): + assert BaseRulesTracker._is_empty_result(0) is False + + def test_is_empty_result_nonempty_list(self): + assert BaseRulesTracker._is_empty_result(["item"]) is False + + def test_add_action_accumulates(self): + tracker = BaseRulesTracker() + tracker.add_action("Function Locator", "pkg,fn1", ["result1"]) + tracker.add_action("Function Locator", "pkg,fn2", ["result2"]) + assert len(tracker.action_history["Function Locator"]) == 2 + assert tracker.action_history["Function Locator"][0]["input"] == "pkg,fn1" + assert tracker.action_history["Function Locator"][1]["input"] == "pkg,fn2" + + +class TestDuplicateCallRule: + def test_duplicate_call_blocked(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools(["Function Locator"]) + tracker.check_thought_behavior("Function Locator", "pkg,fn", ["result"]) + violated, msg = tracker.check_thought_behavior("Function Locator", "pkg,fn", ["result"]) + assert violated is True + assert "already called" in msg + + def test_different_input_allowed(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools(["Function Locator"]) + tracker.check_thought_behavior("Function Locator", "pkg,fn1", ["result"]) + violated, msg = tracker.check_thought_behavior("Function Locator", "pkg,fn2", ["result"]) + assert violated is False + assert msg == "" + + def test_different_tool_same_input_allowed(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools(["Function Locator", "Code Keyword Search"]) + tracker.check_thought_behavior("Function Locator", "pkg,fn", ["result"]) + violated, msg = tracker.check_thought_behavior("Code Keyword Search", "pkg,fn", ["result"]) + assert violated is False + assert msg == "" + + def test_first_call_always_passes(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools(["Function Locator"]) + violated, msg = tracker.check_thought_behavior("Function Locator", "pkg,fn", ["result"]) + assert violated is False + assert msg == "" + + +class TestBaseRule7: + def test_rule7_non_cks_tool_ignored(self): + tracker = BaseRulesTracker() + tracker.add_action("Code Keyword Search", "org.Class", []) + result = tracker._rule_number_7("Function Locator", "org.Class", []) + assert result is False + + def test_rule7_no_dot_in_query_ignored(self): + tracker = BaseRulesTracker() + tracker.add_action("Code Keyword Search", "ClassName", []) + result = tracker._rule_number_7("Code Keyword Search", "ClassName", []) + assert result is False + + def test_rule7_nonempty_output_ignored(self): + tracker = BaseRulesTracker() + tracker.add_action("Code Keyword Search", "org.Class", []) + result = tracker._rule_number_7("Code Keyword Search", "org.Class", ["match"]) + assert result is False + + def test_rule7_no_prior_history_ignored(self): + tracker = BaseRulesTracker() + result = tracker._rule_number_7("Code Keyword Search", "org.Class", []) + assert result is False + + def test_rule7_prior_non_dotted_ignored(self): + tracker = BaseRulesTracker() + tracker.add_action("Code Keyword Search", "ClassName", []) + result = tracker._rule_number_7("Code Keyword Search", "org.Class", []) + assert result is False + + def test_rule7_prior_had_results_ignored(self): + tracker = BaseRulesTracker() + tracker.add_action("Code Keyword Search", "org.Class", ["match"]) + result = tracker._rule_number_7("Code Keyword Search", "org.Class", []) + assert result is False + + def test_rule7_consecutive_dotted_empty_fires(self): + tracker = BaseRulesTracker() + tracker.add_action("Code Keyword Search", "org.Class", []) + result = tracker._rule_number_7("Code Keyword Search", "org.Another", []) + assert result is True + + +class TestBaseRuleAllowedTools: + def test_allowed_tools_in_list_passes(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools(["Function Locator", "Code Keyword Search"]) + result = tracker._rule_use_allowed_tools("Function Locator") + assert result is False + + def test_allowed_tools_not_in_list_fails(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools(["Function Locator"]) + result = tracker._rule_use_allowed_tools("CVE Web Search") + assert result is True + + def test_allowed_tools_empty_list_rejects_all(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools([]) + result = tracker._rule_use_allowed_tools("Function Locator") + assert result is True + + +class TestBaseCheckThoughtBehavior: + def test_happy_path_returns_false_and_adds_to_history(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools(["Function Locator"]) + violated, msg = tracker.check_thought_behavior("Function Locator", "pkg,fn", ["result"]) + assert violated is False + assert msg == "" + assert "Function Locator" in tracker.action_history + assert tracker.action_history["Function Locator"][0]["input"] == "pkg,fn" + + def test_duplicate_priority_over_rule7(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools(["Code Keyword Search"]) + tracker.add_action("Code Keyword Search", "org.Class", []) + violated, msg = tracker.check_thought_behavior("Code Keyword Search", "org.Class", []) + assert violated is True + assert "already called" in msg + assert "Rule 5" not in msg + + def test_rule7_priority_over_allowed_tools(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools(["Other Tool"]) + tracker.add_action("Code Keyword Search", "org.Class", []) + violated, msg = tracker.check_thought_behavior("Code Keyword Search", "org.Another", []) + assert violated is True + assert "Rule 5" in msg + assert "AVAILABLE_TOOLS" not in msg + + def test_allowed_tools_error_message_format(self): + tracker = BaseRulesTracker() + tracker.set_allowed_tools(["Function Locator", "Code Keyword Search"]) + violated, msg = tracker.check_thought_behavior("CVE Web Search", "query", []) + assert violated is True + assert "AVAILABLE_TOOLS" in msg + assert "['Function Locator', 'Code Keyword Search']" in msg + + +class TestReachabilityRulesTracker: + def test_extends_base(self): + tracker = ReachabilityRulesTracker() + assert isinstance(tracker, BaseRulesTracker) + + def test_set_target_functions_creates_dict(self): + tracker = ReachabilityRulesTracker() + tracker.set_target_functions(["fn1", "fn2"]) + assert tracker.target_functions == {"fn1": False, "fn2": False} + + def test_rule8_target_package_enforcement(self): + tracker = ReachabilityRulesTracker() + tracker.set_target_package("commons-beanutils") + result = tracker._rule_number_8("Function Locator", "wrong-package,fn", []) + assert result is True + + def test_rule8_prefix_matching_for_java_gav(self): + tracker = ReachabilityRulesTracker() + tracker.set_target_package("commons-beanutils:commons-beanutils") + result = tracker._rule_number_8("Function Locator", "commons-beanutils:commons-beanutils:1.9.4,fn", []) + assert result is False + + def test_rule8_skipped_after_first_call(self): + tracker = ReachabilityRulesTracker() + tracker.set_target_package("commons-beanutils") + tracker.set_allowed_tools(["Function Locator"]) + tracker.add_action("Function Locator", "commons-beanutils,fn", ["result"]) + result = tracker._rule_number_8("Function Locator", "wrong-package,fn", []) + assert result is False + + def test_rule8_no_target_package_skipped(self): + tracker = ReachabilityRulesTracker() + result = tracker._rule_number_8("Function Locator", "any-package,fn", []) + assert result is False + + def test_rule9_vulnerable_functions_first(self): + tracker = ReachabilityRulesTracker() + tracker.set_target_functions(["getProperty", "setProperty"]) + violated, msg = tracker._rule_number_9("Call Chain Analyzer", "pkg,someOtherFunction") + assert violated is True + assert "Rule 7" in msg + assert "getProperty" in msg or "setProperty" in msg + + def test_rule9_passes_after_checking_vulnerable(self): + tracker = ReachabilityRulesTracker() + tracker.set_target_functions(["getProperty"]) + violated, msg = tracker._rule_number_9("Call Chain Analyzer", "pkg,PropertyUtilsBean.getProperty") + assert violated is False + assert msg == "" + violated, msg = tracker._rule_number_9("Call Chain Analyzer", "pkg,someOtherFunction") + assert violated is False + assert msg == "" + + def test_reachability_check_order(self): + tracker = ReachabilityRulesTracker() + tracker.set_allowed_tools(["Other Tool"]) + tracker.set_target_package("commons-beanutils") + tracker.set_target_functions(["getProperty"]) + + tracker.add_action("Code Keyword Search", "org.Class", []) + violated, msg = tracker.check_thought_behavior("Code Keyword Search", "org.Another", []) + assert violated is True + assert "Rule 5" in msg + + tracker2 = ReachabilityRulesTracker() + tracker2.set_allowed_tools(["Call Chain Analyzer"]) + tracker2.set_target_package("commons-beanutils") + violated, msg = tracker2.check_thought_behavior("Call Chain Analyzer", "wrong-package,fn", []) + assert violated is True + assert "Rule 6" in msg + + tracker3 = ReachabilityRulesTracker() + tracker3.set_allowed_tools(["Other Tool"]) + violated, msg = tracker3.check_thought_behavior("Call Chain Analyzer", "pkg,fn", []) + assert violated is True + assert "AVAILABLE_TOOLS" in msg + + tracker4 = ReachabilityRulesTracker() + tracker4.set_allowed_tools(["Call Chain Analyzer"]) + tracker4.set_target_package("pkg") + tracker4.set_target_functions(["getProperty"]) + violated, msg = tracker4.check_thought_behavior("Call Chain Analyzer", "pkg,otherFunction", []) + assert violated is True + assert "Rule 7" in msg + + +class TestCheckFinishAllowed: + """Tests for ReachabilityRulesTracker.check_finish_allowed.""" + + def test_blocks_finish_when_cca_never_called(self): + """Reachability agent must call CCA before finishing.""" + tracker = ReachabilityRulesTracker() + allowed, msg = tracker.check_finish_allowed(cca_results=[]) + assert allowed is False + assert "Function Locator" in msg + assert "Call Chain Analyzer" in msg + + def test_allows_finish_when_cca_returned_false(self): + """CCA was called and returned not-reachable — finish is allowed.""" + tracker = ReachabilityRulesTracker() + tracker.add_action("Call Chain Analyzer", "pkg,fn", "(False, [])") + allowed, msg = tracker.check_finish_allowed(cca_results=[False]) + assert allowed is True + + def test_allows_finish_when_cca_returned_true_non_java(self): + """Non-Java: CCA found reachable — finish allowed (no FLVF requirement).""" + tracker = ReachabilityRulesTracker() + tracker.set_ecosystem("go") + tracker.add_action("Call Chain Analyzer", "pkg,fn", "(True, [path])") + allowed, msg = tracker.check_finish_allowed(cca_results=[True]) + assert allowed is True + + def test_blocks_finish_java_cca_true_no_flvf(self): + """Java: CCA found reachable but FLVF not called — block finish.""" + tracker = ReachabilityRulesTracker() + tracker.set_ecosystem("java") + tracker.add_action("Call Chain Analyzer", "pkg,fn", "(True, [path])") + allowed, msg = tracker.check_finish_allowed(cca_results=[True]) + assert allowed is False + assert "VERSION CHECK" in msg + assert "Function Library Version Finder" in msg + + def test_allows_finish_java_cca_true_with_flvf(self): + """Java: CCA found reachable and FLVF was called — finish allowed.""" + tracker = ReachabilityRulesTracker() + tracker.set_ecosystem("java") + tracker.add_action("Call Chain Analyzer", "pkg,fn", "(True, [path])") + tracker.add_action("Function Library Version Finder", "pkg", "1.9.4") + allowed, msg = tracker.check_finish_allowed(cca_results=[True]) + assert allowed is True + + def test_allows_finish_java_cca_false(self): + """Java: CCA returned not-reachable — no FLVF requirement.""" + tracker = ReachabilityRulesTracker() + tracker.set_ecosystem("java") + tracker.add_action("Call Chain Analyzer", "pkg,fn", "(False, [])") + allowed, msg = tracker.check_finish_allowed(cca_results=[False]) + assert allowed is True + + def test_allows_finish_when_cca_in_history_but_results_empty(self): + """Edge case: CCA was called (in action_history) but cca_results is + empty — e.g., output parsing failed. Don't block since CCA was attempted.""" + tracker = ReachabilityRulesTracker() + tracker.add_action("Call Chain Analyzer", "pkg,fn", "some unparseable output") + allowed, msg = tracker.check_finish_allowed(cca_results=[]) + assert allowed is True + + +class TestAgentState: + def test_default_tracker_type_annotation(self): + assert "rules_tracker" in AgentState.__annotations__ + + def test_default_is_reachability_field(self): + assert "is_reachability" in AgentState.__annotations__ + + +class TestFindImageMatchingCandidate: + """Tests for _find_image_matching_candidate used by package filter fast path.""" + + def test_match_in_image_name(self): + candidates = [ + {"name": "builder", "source": "rhsa"}, + {"name": "kernel", "source": "rhsa"}, + ] + result = _find_image_matching_candidate( + candidates, "registry.redhat.io/openshift4/ose-docker-builder", None, + ) + assert result == "builder" + + def test_match_in_repo(self): + candidates = [ + {"name": "kernel", "source": "rhsa"}, + {"name": "infinispan", "source": "rhsa"}, + ] + result = _find_image_matching_candidate( + candidates, "registry.redhat.io/some-image", "https://github.com/infinispan/infinispan", + ) + assert result == "infinispan" + + def test_no_match(self): + candidates = [ + {"name": "kernel", "source": "rhsa"}, + {"name": "glibc", "source": "rhsa"}, + ] + result = _find_image_matching_candidate( + candidates, "registry.redhat.io/openshift4/ose-docker-builder", + "https://github.com/openshift/builder", + ) + assert result is None + + def test_short_name_skipped(self): + """Candidate names shorter than 3 chars are not matched.""" + candidates = [{"name": "go", "source": "rhsa"}] + result = _find_image_matching_candidate( + candidates, "registry.redhat.io/golang-builder", None, + ) + assert result is None + + def test_no_image_no_repo(self): + candidates = [{"name": "builder", "source": "rhsa"}] + result = _find_image_matching_candidate(candidates, None, None) + assert result is None + + def test_first_match_wins(self): + """When multiple candidates could match, the first one wins.""" + candidates = [ + {"name": "openshift", "source": "rhsa"}, + {"name": "builder", "source": "rhsa"}, + ] + result = _find_image_matching_candidate( + candidates, "registry.redhat.io/openshift4/ose-docker-builder", + "https://github.com/openshift/builder", + ) + assert result == "openshift" diff --git a/tests/test_repo_resolver.py b/tests/test_repo_resolver.py new file mode 100644 index 000000000..bb9311380 --- /dev/null +++ b/tests/test_repo_resolver.py @@ -0,0 +1,331 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for repo_resolver: package name to git repository URL resolution.""" + +import pytest +from unittest.mock import MagicMock + +from vuln_analysis.utils.repo_resolver import ( + normalize_package_name, + purl_to_repo_url, + detect_platform, + collect_all_purls, + load_package_repo_mapping, + clear_mapping_cache, + RepoResolver, + CONFIDENCE_OSIDB_PURL, + CONFIDENCE_CURATED_DIRECT, + CONFIDENCE_CURATED_ALIAS, + CONFIDENCE_PURL_PARSE, + RESOLUTION_METHOD_OSIDB_PURL, + RESOLUTION_METHOD_CURATED_MAPPING, + RESOLUTION_METHOD_PURL_PARSE, + RESOLUTION_METHOD_NOT_FOUND, + PLATFORM_GITHUB, + PLATFORM_GITLAB, + PLATFORM_BITBUCKET, + PLATFORM_KERNEL_ORG, +) + + +class TestNormalizePackageName: + """Tests for package name normalization.""" + + def test_simple_name_unchanged(self): + """Simple package names return as-is.""" + variants = normalize_package_name("curl") + assert "curl" in variants + + def test_python3_prefix_stripped(self): + """python3- prefix is stripped.""" + variants = normalize_package_name("python3-urllib3") + assert "python3-urllib3" in variants + assert "urllib3" in variants + + def test_lib_prefix_stripped(self): + """lib prefix is stripped.""" + variants = normalize_package_name("libcurl") + assert "libcurl" in variants + assert "curl" in variants + + def test_version_suffix_stripped(self): + """Version suffixes are stripped.""" + variants = normalize_package_name("libcurl4") + assert "libcurl4" in variants + assert "libcurl" in variants + assert "curl" in variants + + def test_golang_prefix_stripped(self): + """golang- prefix is stripped.""" + variants = normalize_package_name("golang-github-foo") + assert "golang-github-foo" in variants + assert "github-foo" in variants + + def test_empty_string_returns_empty_list(self): + """Empty string returns empty list.""" + variants = normalize_package_name("") + assert variants == [] + + def test_no_duplicates(self): + """No duplicate variants returned.""" + variants = normalize_package_name("curl") + assert len(variants) == len(set(variants)) + + +class TestPurlToRepoUrl: + """Tests for PURL to repository URL conversion.""" + + def test_github_purl(self): + """GitHub PURL converts to GitHub URL.""" + url, platform = purl_to_repo_url("pkg:github/curl/curl") + assert url == "https://github.com/curl/curl" + assert platform == PLATFORM_GITHUB + + def test_gitlab_purl(self): + """GitLab PURL converts to GitLab URL.""" + url, platform = purl_to_repo_url("pkg:gitlab/gnome/libxml2") + assert url == "https://gitlab.com/gnome/libxml2" + assert platform == PLATFORM_GITLAB + + def test_bitbucket_purl(self): + """Bitbucket PURL converts to Bitbucket URL.""" + url, platform = purl_to_repo_url("pkg:bitbucket/owner/repo") + assert url == "https://bitbucket.org/owner/repo" + assert platform == PLATFORM_BITBUCKET + + def test_purl_with_version(self): + """PURL with version is handled correctly.""" + url, platform = purl_to_repo_url("pkg:github/curl/curl@7.88.0") + assert url == "https://github.com/curl/curl" + assert platform == PLATFORM_GITHUB + + def test_purl_with_qualifiers(self): + """PURL with qualifiers is handled correctly.""" + url, _ = purl_to_repo_url("pkg:github/curl/curl?foo=bar") + assert url == "https://github.com/curl/curl" + + def test_unsupported_purl_type(self): + """Unsupported PURL type returns None.""" + url, platform = purl_to_repo_url("pkg:pypi/requests") + assert url is None + assert platform is None + + def test_invalid_purl(self): + """Invalid PURL returns None.""" + url, platform = purl_to_repo_url("not-a-purl") + assert url is None + assert platform is None + + def test_empty_purl(self): + """Empty PURL returns None.""" + url, platform = purl_to_repo_url("") + assert url is None + assert platform is None + + def test_none_purl(self): + """None PURL returns None.""" + url, platform = purl_to_repo_url(None) + assert url is None + assert platform is None + + +class TestDetectPlatform: + """Tests for platform detection from URLs.""" + + def test_github_url(self): + """GitHub URLs detected.""" + assert detect_platform("https://github.com/curl/curl") == PLATFORM_GITHUB + + def test_gitlab_url(self): + """GitLab URLs detected.""" + assert detect_platform("https://gitlab.com/gnome/libxml2") == PLATFORM_GITLAB + + def test_gitlab_instance_url(self): + """Self-hosted GitLab URLs detected.""" + assert detect_platform("https://gitlab.gnome.org/GNOME/libxml2") == PLATFORM_GITLAB + + def test_bitbucket_url(self): + """Bitbucket URLs detected.""" + assert detect_platform("https://bitbucket.org/owner/repo") == PLATFORM_BITBUCKET + + def test_kernel_org_url(self): + """kernel.org URLs detected.""" + kernel_url = "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" + assert detect_platform(kernel_url) == PLATFORM_KERNEL_ORG + + def test_empty_url(self): + """Empty URL returns unknown.""" + assert detect_platform("") == "unknown" + + +class TestCollectAllPurls: + """Tests for collecting PURLs from CveIntel.""" + + def _make_osidb_with_purls(self, purls): + """Create mock OSIDB with upstream_purls.""" + mock_intel = MagicMock() + mock_intel.osidb = MagicMock() + mock_intel.osidb.upstream_purls = [ + MagicMock(purl=purl) for purl in purls + ] + mock_intel.osidb.affects = [] + return mock_intel + + def test_collects_osidb_upstream_purls(self): + """PURLs from OSIDB upstream_purls are collected.""" + intel = self._make_osidb_with_purls(["pkg:github/curl/curl"]) + purls = collect_all_purls(intel) + assert "pkg:github/curl/curl" in purls + + def test_deduplicates_purls(self): + """Duplicate PURLs are removed.""" + intel = self._make_osidb_with_purls([ + "pkg:github/curl/curl", + "pkg:github/curl/curl", + ]) + purls = collect_all_purls(intel) + assert len(purls) == 1 + + def test_none_intel_returns_empty(self): + """None intel returns empty list.""" + purls = collect_all_purls(None) + assert purls == [] + + def test_no_osidb_returns_empty(self): + """Intel without OSIDB returns empty list.""" + mock_intel = MagicMock() + mock_intel.osidb = None + purls = collect_all_purls(mock_intel) + assert purls == [] + + +class TestLoadPackageRepoMapping: + """Tests for mapping file loading.""" + + def test_mapping_loads_successfully(self): + """Mapping file loads and contains expected structure.""" + clear_mapping_cache() + mapping = load_package_repo_mapping() + assert "packages" in mapping + assert "aliases" in mapping + assert len(mapping["packages"]) > 50 + + def test_mapping_contains_curl(self): + """Mapping contains common package 'curl'.""" + mapping = load_package_repo_mapping() + assert "curl" in mapping["packages"] + assert "github.com/curl/curl" in mapping["packages"]["curl"] + + def test_mapping_contains_aliases(self): + """Mapping contains aliases like apache -> httpd.""" + mapping = load_package_repo_mapping() + assert "apache" in mapping["aliases"] + assert mapping["aliases"]["apache"] == "httpd" + + def test_cache_works(self): + """Second call returns cached mapping.""" + clear_mapping_cache() + mapping1 = load_package_repo_mapping() + mapping2 = load_package_repo_mapping() + assert mapping1 is mapping2 + + +class TestRepoResolver: + """Tests for the RepoResolver class.""" + + @pytest.fixture + def resolver(self): + """Create a resolver instance.""" + clear_mapping_cache() + return RepoResolver() + + def test_resolve_curated_direct(self, resolver): + """Direct curated mapping returns correct result.""" + result = resolver.resolve("curl") + assert result.repo_url == "https://github.com/curl/curl" + assert result.resolution_method == RESOLUTION_METHOD_CURATED_MAPPING + assert result.confidence == CONFIDENCE_CURATED_DIRECT + assert result.is_alias is False + + def test_resolve_curated_alias(self, resolver): + """Alias curated mapping returns correct result.""" + result = resolver.resolve("apache") + assert result.repo_url == "https://github.com/apache/httpd" + assert result.resolution_method == RESOLUTION_METHOD_CURATED_MAPPING + assert result.confidence == CONFIDENCE_CURATED_ALIAS + assert result.is_alias is True + + def test_resolve_with_prefix_stripping(self, resolver): + """Package with prefix resolves via normalization.""" + result = resolver.resolve("libcurl") + assert result.repo_url == "https://github.com/curl/curl" + assert result.resolution_method == RESOLUTION_METHOD_CURATED_MAPPING + + def test_resolve_not_found(self, resolver): + """Unknown package returns not_found.""" + result = resolver.resolve("nonexistent-package-xyz-123") + assert result.repo_url is None + assert result.resolution_method == RESOLUTION_METHOD_NOT_FOUND + assert result.confidence == 0.0 + + def test_resolve_osidb_purl_priority(self, resolver): + """OSIDB PURL takes priority over curated mapping.""" + mock_intel = MagicMock() + mock_intel.osidb = MagicMock() + mock_intel.osidb.upstream_purls = [ + MagicMock(purl="pkg:github/custom/custom-curl") + ] + + result = resolver.resolve("curl", intel=mock_intel) + assert result.repo_url == "https://github.com/custom/custom-curl" + assert result.resolution_method == RESOLUTION_METHOD_OSIDB_PURL + assert result.confidence == CONFIDENCE_OSIDB_PURL + + def test_resolve_strategies_tried_tracked(self, resolver): + """Strategies tried are tracked in result.""" + result = resolver.resolve("nonexistent-xyz") + assert RESOLUTION_METHOD_OSIDB_PURL in result.strategies_tried + assert RESOLUTION_METHOD_CURATED_MAPPING in result.strategies_tried + assert RESOLUTION_METHOD_PURL_PARSE in result.strategies_tried + + def test_resolve_httpd(self, resolver): + """httpd resolves correctly.""" + result = resolver.resolve("httpd") + assert result.repo_url == "https://github.com/apache/httpd" + assert result.resolution_method == RESOLUTION_METHOD_CURATED_MAPPING + + def test_resolve_openssl(self, resolver): + """openssl resolves correctly.""" + result = resolver.resolve("openssl") + assert result.repo_url == "https://github.com/openssl/openssl" + + def test_resolve_linux_kernel(self, resolver): + """linux kernel resolves to kernel.org.""" + result = resolver.resolve("linux") + assert "kernel.org" in result.repo_url + assert result.repo_platform == PLATFORM_KERNEL_ORG + + +class TestRepoResolverPurlParseFallback: + """Tests for PURL parse fallback strategy.""" + + @pytest.fixture + def resolver(self): + """Create a resolver instance.""" + clear_mapping_cache() + return RepoResolver() + + def test_purl_parse_fallback_for_unknown_package(self, resolver): + """PURL parse works as fallback when package not in mapping.""" + mock_intel = MagicMock() + mock_intel.osidb = MagicMock() + mock_intel.osidb.upstream_purls = [] + mock_intel.osidb.affects = [ + MagicMock(purl="pkg:github/unknown/unknown-package") + ] + + result = resolver.resolve("unknown-package-not-in-mapping", intel=mock_intel) + assert result.repo_url == "https://github.com/unknown/unknown-package" + assert result.resolution_method == RESOLUTION_METHOD_PURL_PARSE + assert result.confidence == CONFIDENCE_PURL_PARSE diff --git a/tests/test_serp_api_key_rotation.py b/tests/test_serp_api_key_rotation.py new file mode 100644 index 000000000..acc8a1108 --- /dev/null +++ b/tests/test_serp_api_key_rotation.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Unit tests for SerpAPI key rotation in serp_api_wrapper.py. +""" + +import re +import threading + +import pytest +from aioresponses import aioresponses +from aiohttp import ClientResponseError +from vuln_analysis.utils.serp_api_wrapper import MorpheusSerpAPIWrapper + +SERPAPI_SEARCH_URL_PATTERN = re.compile(r'https://serpapi\.com/search\?.*') +TEST_PAYLOAD = {"results": ["test"]} +TEST_QUERY = "test" +SINGLE_KEY = "key1" +TWO_KEYS = "key1,key2" + + +@pytest.fixture +def serpapi_wrapper_single_key(): + """Create a wrapper with a single API key.""" + # Reset class-level state before each test + MorpheusSerpAPIWrapper._serp_api_keys = [] + MorpheusSerpAPIWrapper._serp_api_key_index = 0 + return MorpheusSerpAPIWrapper(serpapi_api_key=SINGLE_KEY) + + +@pytest.fixture +def serpapi_wrapper_two_keys(): + """Create a wrapper with two API keys.""" + # Reset class-level state before each test + MorpheusSerpAPIWrapper._serp_api_keys = [] + MorpheusSerpAPIWrapper._serp_api_key_index = 0 + return MorpheusSerpAPIWrapper(serpapi_api_key=TWO_KEYS) + + +@pytest.mark.asyncio +async def test_single_key_success(serpapi_wrapper_single_key): + """Test that a single key is used successfully.""" + with aioresponses() as mock: + # Mock successful SerpAPI response + mock.get( + SERPAPI_SEARCH_URL_PATTERN, + status=200, + payload=TEST_PAYLOAD, + repeat=True, + ) + result = await serpapi_wrapper_single_key.aresults(TEST_QUERY) + assert result == TEST_PAYLOAD + + +@pytest.mark.asyncio +async def test_single_key_failure(serpapi_wrapper_single_key): + """Test that single failed key raises exception.""" + with aioresponses() as mock: + # Mock 402 (payment required) response + mock.get( + SERPAPI_SEARCH_URL_PATTERN, + status=402, + repeat=True, + ) + with pytest.raises(Exception, match="All API keys exhausted"): + await serpapi_wrapper_single_key.aresults(TEST_QUERY) + + +@pytest.mark.asyncio +async def test_all_keys_exhausted(serpapi_wrapper_two_keys): + """Test that exception is raised when all keys are exhausted.""" + with aioresponses() as mock: + # Mock 402 for all requests (both keys will fail) + mock.get( + SERPAPI_SEARCH_URL_PATTERN, + status=402, + repeat=True, + ) + with pytest.raises(Exception, match="All API keys exhausted"): + await serpapi_wrapper_two_keys.aresults(TEST_QUERY) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("error_code", [402, 429]) +async def test_key_rotation(error_code, serpapi_wrapper_two_keys): + """Test that key rotation works for rate limit (429) and payment (402) errors.""" + with aioresponses() as mock: + # First request returns error (triggers rotation) + mock.get( + SERPAPI_SEARCH_URL_PATTERN, + status=error_code, + ) + # Second request succeeds with rotated key + mock.get( + SERPAPI_SEARCH_URL_PATTERN, + status=200, + payload=TEST_PAYLOAD, + repeat=True, + ) + result = await serpapi_wrapper_two_keys.aresults(TEST_QUERY) + + # Verify successful response + assert result == TEST_PAYLOAD + + # Verify rotation occurred: index advanced from 0 to 1 + assert serpapi_wrapper_two_keys.serp_api_key_index == 1 + + # Verify active key changed from "key1" to "key2" + assert serpapi_wrapper_two_keys.serpapi_api_key == "key2" + +def test_concurrent_rotation(): + """Test that concurrent key rotation is thread-safe.""" + # Reset class-level state before test + MorpheusSerpAPIWrapper._serp_api_keys = [] + MorpheusSerpAPIWrapper._serp_api_key_index = 0 + wrapper = MorpheusSerpAPIWrapper(serpapi_api_key="key1,key2,key3") + num_threads = 10 + iterations_per_thread = 5 + + def rotate_many_times(): + for _ in range(iterations_per_thread): + key = wrapper._rotate_next_key() + # Key must always be from the pool + assert key in wrapper.serp_api_keys + print(f"{threading.current_thread().name} rotated to {key}", flush=True) + + threads = [threading.Thread(target=rotate_many_times) for _ in range(num_threads)] + for t in threads: + t.start() + for t in threads: + t.join() + + # After all rotations, index must be valid + assert 0 <= wrapper.serp_api_key_index < len(wrapper.serp_api_keys) diff --git a/tests/test_source_classification.py b/tests/test_source_classification.py new file mode 100644 index 000000000..53f072125 --- /dev/null +++ b/tests/test_source_classification.py @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from vuln_analysis.utils.source_classification import ( + is_dependency_path, + filter_by_source_scope, + format_app_dep_output, +) + + +class TestIsDependencyPath: + @pytest.mark.parametrize("path", [ + "dependencies-sources/commons-beanutils/PropertyUtilsBean.java", + "vendor/github.com/foo/bar/bar.go", + "transitive_env/lib/requests/models.py", + "node_modules/express/index.js", + "rpm_libs/libxml2/parser.c", + ]) + def test_dependency_paths(self, path): + assert is_dependency_path(path) is True + + @pytest.mark.parametrize("path", [ + "src/main/java/com/example/App.java", + "cmd/main.go", + "app.py", + "pom.xml", + "src/main/resources/application.yml", + ]) + def test_app_paths(self, path): + assert is_dependency_path(path) is False + + def test_empty_path(self): + assert is_dependency_path("") is False + + +class TestFilterBySourceScope: + def test_filters_by_scope(self): + items = [ + ("dependencies-sources/xstream/XStream.java", "entry1"), + ("dependencies-sources/commons-io/IOUtils.java", "entry2"), + ("dependencies-sources/xstream/Converter.java", "entry3"), + ] + result = filter_by_source_scope(items, ["xstream"], lambda x: x[0]) + assert len(result) == 2 + assert result[0][1] == "entry1" + assert result[1][1] == "entry3" + + def test_none_scope_returns_all(self): + items = [("a", "x"), ("b", "y")] + result = filter_by_source_scope(items, None, lambda x: x[0]) + assert result == items + + def test_empty_scope_returns_all(self): + items = [("a", "x"), ("b", "y")] + result = filter_by_source_scope(items, [], lambda x: x[0]) + assert result == items + + def test_multiple_scope_terms(self): + items = [ + ("vendor/foo/main.go", "e1"), + ("vendor/bar/main.go", "e2"), + ("vendor/baz/main.go", "e3"), + ] + result = filter_by_source_scope(items, ["foo", "baz"], lambda x: x[0]) + assert len(result) == 2 + assert result[0][1] == "e1" + assert result[1][1] == "e3" + + def test_no_matches_returns_empty(self): + items = [("vendor/foo/main.go", "e1")] + result = filter_by_source_scope(items, ["nonexistent"], lambda x: x[0]) + assert result == [] + + +class TestFormatAppDepOutput: + def test_both_sections(self): + result = format_app_dep_output( + ["app_match_1", "app_match_2"], + ["dep_match_1"], + total_app=2, total_dep=1, + no_results_msg="No results", + ) + assert "Main application (2 of 2 results)" in result + assert "Application library dependencies (1 of 1 results)" in result + assert "app_match_1" in result + assert "dep_match_1" in result + + def test_empty_returns_no_results_msg(self): + result = format_app_dep_output([], [], 0, 0, "Nothing found") + assert result == "Nothing found" + + def test_app_only(self): + result = format_app_dep_output( + ["app1"], [], total_app=1, total_dep=0, + no_results_msg="No results", + ) + assert "Main application (1 of 1 results)" in result + assert "Application library dependencies (0 of 0 results)" in result + assert "app1" in result + + def test_dep_only(self): + result = format_app_dep_output( + [], ["dep1"], total_app=0, total_dep=1, + no_results_msg="No results", + ) + assert "Main application (0 of 0 results)" in result + assert "Application library dependencies (1 of 1 results)" in result + assert "dep1" in result + + def test_trimmed_counts(self): + result = format_app_dep_output( + ["a1", "a2"], ["d1"], + total_app=5, total_dep=3, + no_results_msg="No results", + ) + assert "Main application (2 of 5 results)" in result + assert "Application library dependencies (1 of 3 results)" in result + + def test_app_before_dep(self): + result = format_app_dep_output( + ["APP_SECTION"], ["DEP_SECTION"], + total_app=1, total_dep=1, + no_results_msg="No results", + ) + app_pos = result.index("APP_SECTION") + dep_pos = result.index("DEP_SECTION") + assert app_pos < dep_pos diff --git a/tests/test_tool_filtering.py b/tests/test_tool_filtering.py new file mode 100644 index 000000000..2a8442484 --- /dev/null +++ b/tests/test_tool_filtering.py @@ -0,0 +1,149 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Unit tests for tool filtering logic in CVE agent functions. +""" + +import pytest + +from vuln_analysis.tools.tool_names import ToolNames + + +class MockTool: + """Mock tool for testing.""" + def __init__(self, name: str): + self.name = name + + +class MockState: + """Mock state for testing.""" + def __init__( + self, + code_vdb_path: str | None = "/path/to/code", + doc_vdb_path: str | None = "/path/to/docs", + code_index_path: str | None = "/path/to/index" + ): + self.code_vdb_path = code_vdb_path + self.doc_vdb_path = doc_vdb_path + self.code_index_path = code_index_path + + +class TestToolFiltering: + """Test tool filtering logic matches constants correctly.""" + + def create_mock_state( + self, + code_vdb_path: str | None = "/path/to/code", + doc_vdb_path: str | None = "/path/to/docs", + code_index_path: str | None = "/path/to/index" + ) -> MockState: + """Create a mock state for testing.""" + return MockState( + code_vdb_path=code_vdb_path, + doc_vdb_path=doc_vdb_path, + code_index_path=code_index_path + ) + + def test_filter_code_qa_when_path_missing(self): + """Test that Code Semantic Search tool is filtered when code_vdb_path is None.""" + state = self.create_mock_state(code_vdb_path=None) + tools = [ + MockTool(ToolNames.CODE_SEMANTIC_SEARCH), + MockTool(ToolNames.DOCS_SEMANTIC_SEARCH), + ] + + # Apply filtering logic + filtered_tools = [ + tool for tool in tools + if not ((tool.name == ToolNames.CODE_SEMANTIC_SEARCH and state.code_vdb_path is None) or + (tool.name == ToolNames.DOCS_SEMANTIC_SEARCH and state.doc_vdb_path is None)) + ] + + assert len(filtered_tools) == 1 + assert filtered_tools[0].name == ToolNames.DOCS_SEMANTIC_SEARCH + + def test_filter_doc_qa_when_path_missing(self): + """Test that Docs Semantic Search tool is filtered when doc_vdb_path is None.""" + state = self.create_mock_state(doc_vdb_path=None) + tools = [ + MockTool(ToolNames.CODE_SEMANTIC_SEARCH), + MockTool(ToolNames.DOCS_SEMANTIC_SEARCH), + ] + + filtered_tools = [ + tool for tool in tools + if not ((tool.name == ToolNames.CODE_SEMANTIC_SEARCH and state.code_vdb_path is None) or + (tool.name == ToolNames.DOCS_SEMANTIC_SEARCH and state.doc_vdb_path is None)) + ] + + assert len(filtered_tools) == 1 + assert filtered_tools[0].name == ToolNames.CODE_SEMANTIC_SEARCH + + def test_filter_lexical_search_when_path_missing(self): + """Test that Code Keyword Search tool is filtered when code_index_path is None.""" + state = self.create_mock_state(code_index_path=None) + tools = [ + MockTool(ToolNames.CODE_KEYWORD_SEARCH), + MockTool(ToolNames.CVE_WEB_SEARCH), + ] + + filtered_tools = [ + tool for tool in tools + if not (tool.name == ToolNames.CODE_KEYWORD_SEARCH and state.code_index_path is None) + ] + + assert len(filtered_tools) == 1 + assert filtered_tools[0].name == ToolNames.CVE_WEB_SEARCH + + def test_no_filtering_when_all_paths_present(self): + """Test that no tools are filtered when all paths are available.""" + state = self.create_mock_state() + tools = [ + MockTool(ToolNames.CODE_SEMANTIC_SEARCH), + MockTool(ToolNames.DOCS_SEMANTIC_SEARCH), + MockTool(ToolNames.CODE_KEYWORD_SEARCH), + ] + + filtered_tools = [ + tool for tool in tools + if not ((tool.name == ToolNames.CODE_SEMANTIC_SEARCH and state.code_vdb_path is None) or + (tool.name == ToolNames.DOCS_SEMANTIC_SEARCH and state.doc_vdb_path is None) or + (tool.name == ToolNames.CODE_KEYWORD_SEARCH and state.code_index_path is None)) + ] + + assert len(filtered_tools) == 3 + + def test_filter_transitive_search_when_disabled(self): + """Test that call chain analysis tools are filtered when disabled.""" + state = self.create_mock_state() + transitive_enabled = False + + tools = [ + MockTool(ToolNames.CALL_CHAIN_ANALYZER), + MockTool(ToolNames.FUNCTION_CALLER_FINDER), + MockTool(ToolNames.CODE_SEMANTIC_SEARCH), + ] + + filtered_tools = [ + tool for tool in tools + if not ((tool.name == ToolNames.CALL_CHAIN_ANALYZER and + (not transitive_enabled or state.code_index_path is None)) or + (tool.name == ToolNames.FUNCTION_CALLER_FINDER and + (not transitive_enabled or state.code_index_path is None))) + ] + + assert len(filtered_tools) == 1 + assert filtered_tools[0].name == ToolNames.CODE_SEMANTIC_SEARCH diff --git a/tests/test_tool_names.py b/tests/test_tool_names.py new file mode 100644 index 000000000..8aa289bb7 --- /dev/null +++ b/tests/test_tool_names.py @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Unit tests for tool name constants. +""" + +import pytest + +from vuln_analysis.tools.tool_names import ToolNames + + +class TestToolNameConstants: + """Test the tool name constants are properly defined.""" + + def test_all_constants_defined(self): + """Verify all expected tool name constants exist.""" + assert hasattr(ToolNames, 'CODE_SEMANTIC_SEARCH') + assert hasattr(ToolNames, 'DOCS_SEMANTIC_SEARCH') + assert hasattr(ToolNames, 'CODE_KEYWORD_SEARCH') + assert hasattr(ToolNames, 'CALL_CHAIN_ANALYZER') + assert hasattr(ToolNames, 'FUNCTION_CALLER_FINDER') + assert hasattr(ToolNames, 'CVE_WEB_SEARCH') + assert hasattr(ToolNames, 'CONTAINER_ANALYSIS_DATA') + + def test_constants_are_unique(self): + """Verify all tool names are unique.""" + constants = [ + ToolNames.CODE_SEMANTIC_SEARCH, + ToolNames.DOCS_SEMANTIC_SEARCH, + ToolNames.CODE_KEYWORD_SEARCH, + ToolNames.CALL_CHAIN_ANALYZER, + ToolNames.FUNCTION_CALLER_FINDER, + ToolNames.CVE_WEB_SEARCH, + ToolNames.CONTAINER_ANALYSIS_DATA, + ] + + assert len(constants) == len(set(constants)), "Tool names must be unique" + diff --git a/tests/test_transitive_detection.py b/tests/test_transitive_detection.py new file mode 100644 index 000000000..cd65acb9f --- /dev/null +++ b/tests/test_transitive_detection.py @@ -0,0 +1,114 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for detect_ecosystem in dep_tree.py.""" + +import pytest +from exploit_iq_commons.utils.dep_tree import Ecosystem, detect_ecosystem + + +# --- C/C++ detection: requires manifest + source files --- + +C_CPP_POSITIVE_SCENARIOS = [ + # manifest + source file combinations + ("CMakeLists.txt", ["src/main.c"]), + ("CMakeLists.txt", ["src/utils.cpp"]), + ("Makefile", ["include/lib.h"]), + ("meson.build", ["src/deep/nested/logic.cxx"]), + ("CMakeLists.txt", ["lib.cc"]), + ("CMakeLists.txt", ["api.hpp"]), +] + +C_CPP_NEGATIVE_SCENARIOS = [ + # manifest present but no C/C++ source files + ("Makefile", ["app.py"]), + ("CMakeLists.txt", ["README.md", "LICENSE"]), + ("Makefile", ["main.cs"]), + ("CMakeLists.txt", ["style.css"]), + ("Makefile", []), + # C/C++ sources but NO manifest — should not detect + (None, ["src/main.c"]), + (None, ["src/utils.cpp", "include/lib.h"]), + # manifest + source files in ignored dotfile directories + ("CMakeLists.txt", [".git/objects/obj.c"]), + ("CMakeLists.txt", [".venv/lib/site/test.c"]), + ("CMakeLists.txt", [".config/settings.h"]), +] + + +@pytest.mark.parametrize("manifest, file_paths", C_CPP_POSITIVE_SCENARIOS) +def test_c_cpp_detected(tmp_path, manifest, file_paths): + (tmp_path / manifest).touch() + for path in file_paths: + full_path = tmp_path / path + full_path.parent.mkdir(parents=True, exist_ok=True) + full_path.touch() + assert detect_ecosystem(tmp_path) == Ecosystem.C_CPP + + +@pytest.mark.parametrize("manifest, file_paths", C_CPP_NEGATIVE_SCENARIOS) +def test_c_cpp_not_detected(tmp_path, manifest, file_paths): + if manifest: + (tmp_path / manifest).touch() + for path in file_paths: + full_path = tmp_path / path + full_path.parent.mkdir(parents=True, exist_ok=True) + full_path.touch() + assert detect_ecosystem(tmp_path) != Ecosystem.C_CPP + + +# --- Other ecosystem detection --- + +OTHER_ECOSYSTEM_SCENARIOS = [ + ("go.mod", Ecosystem.GO), + ("requirements.txt", Ecosystem.PYTHON), + ("pyproject.toml", Ecosystem.PYTHON), + ("setup.py", Ecosystem.PYTHON), + ("package.json", Ecosystem.JAVASCRIPT), + ("pom.xml", Ecosystem.JAVA), +] + + +@pytest.mark.parametrize("manifest, expected", OTHER_ECOSYSTEM_SCENARIOS) +def test_ecosystem_detected_by_manifest(tmp_path, manifest, expected): + (tmp_path / manifest).touch() + assert detect_ecosystem(tmp_path) == expected + + +def test_no_manifest_returns_none(tmp_path): + (tmp_path / "README.md").touch() + assert detect_ecosystem(tmp_path) is None + + +def test_empty_dir_returns_none(tmp_path): + assert detect_ecosystem(tmp_path) is None + + +# --- Priority order: Go > Python > JS > Java > C/C++ --- + +def test_go_takes_priority_over_python(tmp_path): + (tmp_path / "go.mod").touch() + (tmp_path / "requirements.txt").touch() + assert detect_ecosystem(tmp_path) == Ecosystem.GO + + +def test_python_takes_priority_over_java(tmp_path): + (tmp_path / "requirements.txt").touch() + (tmp_path / "pom.xml").touch() + assert detect_ecosystem(tmp_path) == Ecosystem.PYTHON + + +def test_java_takes_priority_over_c_cpp(tmp_path): + (tmp_path / "pom.xml").touch() + (tmp_path / "CMakeLists.txt").touch() + (tmp_path / "main.c").touch() + assert detect_ecosystem(tmp_path) == Ecosystem.JAVA + + +def test_configure_in_auto_subdir(tmp_path): + """detect_ecosystem checks for auto/configure as a C/C++ manifest.""" + (tmp_path / "auto").mkdir() + (tmp_path / "auto" / "configure").touch() + (tmp_path / "src").mkdir() + (tmp_path / "src" / "main.c").touch() + assert detect_ecosystem(tmp_path) == Ecosystem.C_CPP diff --git a/tests/test_version_check.py b/tests/test_version_check.py new file mode 100644 index 000000000..2bfd431ee --- /dev/null +++ b/tests/test_version_check.py @@ -0,0 +1,560 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for version_check gate and deterministic comparison.""" + +import pytest + +from exploit_iq_commons.data_models.cve_intel import CveIntel, CveIntelGhsa, CveIntelNvd, CveIntelRhsa + +from vuln_analysis.utils.version_check import ( + HardPathReason, + StdlibVulnerabilityResult, + VersionCheckPath, + check_stdlib_vulnerability, + classify_version_check, + deterministic_version_check, + get_cve_description, +) + + +POSTGRES_MULTI_BRANCH_DESCRIPTION = ( + "Versions before PostgreSQL 17.3, 16.7, 15.11, 14.16, and 13.19 are affected." +) + + +class TestClassifyVersionCheck: + def test_lodash_ghsa_first_patched_is_easy(self): + version_info = { + "first_patched": "4.17.21", + "vulnerable_range": None, + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + } + result = classify_version_check( + installed_version="4.17.20", + version_info=version_info, + description=None, + ecosystem="npm", + package_name="lodash", + ) + assert result.path == VersionCheckPath.EASY + + def test_postgresql_module_nevra_is_hard(self): + version_info = { + "first_patched": "9.2.24-9.el7_9", + "vulnerable_range": "< 9.2.24-9.el7_9", + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": "9.2.24-9.el7_9", + } + result = classify_version_check( + installed_version="13.14-1.module+el8.9.0+21288+3d364c44", + version_info=version_info, + description=POSTGRES_MULTI_BRANCH_DESCRIPTION, + ecosystem="rpm", + package_name="postgresql", + ) + assert result.path == VersionCheckPath.HARD + assert result.hard_reason in { + HardPathReason.MODULE_NEVRA, + HardPathReason.MULTI_BRANCH_DESCRIPTION, + } + + def test_openssl_letter_suffix_is_hard(self): + version_info = { + "first_patched": "1.1.1k", + "vulnerable_range": None, + "version_start_incl": "1.0.0", + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": "1.1.1k", + } + result = classify_version_check( + installed_version="1.1.1j-2.el8", + version_info=version_info, + description=None, + ecosystem="rpm", + package_name="openssl-libs", + ) + assert result.path == VersionCheckPath.HARD + assert result.hard_reason == HardPathReason.LETTER_SUFFIX_VERSION + + def test_cross_dist_rhsa_is_hard(self): + version_info = { + "first_patched": "9.2.24-9.el7_9", + "vulnerable_range": None, + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + } + result = classify_version_check( + installed_version="13.14-1.el8", + version_info=version_info, + description=None, + ecosystem="rpm", + package_name="postgresql", + ) + assert result.path == VersionCheckPath.HARD + assert result.hard_reason == HardPathReason.CROSS_DIST_INTEL + + def test_rhsa_not_affected_is_vendor_shortcut(self): + intel = CveIntel( + vuln_id="CVE-2016-8687", + rhsa=CveIntelRhsa( + package_state=[ + CveIntelRhsa.PackageState( + product_name="Red Hat Enterprise Linux 7", + fix_state="Will not fix", + package_name="libarchive", + cpe="cpe:/o:redhat:enterprise_linux:7", + ), + ], + ), + ) + version_info = { + "fix_state": None, + "first_patched": None, + "vulnerable_range": None, + "version_start_incl": "3.2.1", + "version_start_excl": None, + "version_end_incl": "3.2.1", + "version_end_excl": None, + } + result = classify_version_check( + installed_version="3.1.2-14.el7_9.1", + version_info=version_info, + description=None, + ecosystem="rpm", + package_name="libarchive", + intel=intel, + ) + assert result.path == VersionCheckPath.VENDOR_NOT_AFFECTED + + def test_nvd_range_clearly_outside_is_easy(self): + version_info = { + "first_patched": None, + "vulnerable_range": None, + "version_start_incl": "1.0.0", + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": "1.5.0", + } + result = classify_version_check( + installed_version="2.0.0", + version_info=version_info, + description=None, + ecosystem="pypi", + package_name="somelib", + ) + assert result.path == VersionCheckPath.EASY + + def test_same_dist_rpm_nvr_is_easy(self): + version_info = { + "first_patched": "7.76.1-31.el9", + "vulnerable_range": None, + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + } + result = classify_version_check( + installed_version="7.76.1-26.el9", + version_info=version_info, + description=None, + ecosystem="rpm", + package_name="curl", + ) + assert result.path == VersionCheckPath.EASY + + def test_pypi_prerelease_version_is_easy(self): + """PyPI pre-release versions (PEP 440) should use EASY path with PypiVersion.""" + version_info = { + "first_patched": "1.0.0", + "vulnerable_range": None, + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + } + result = classify_version_check( + installed_version="1.0.0a1", + version_info=version_info, + description=None, + ecosystem="pypi", + package_name="somelib", + ) + assert result.path == VersionCheckPath.EASY + + def test_pypi_post_release_version_is_easy(self): + """PyPI post-release versions (PEP 440) should use EASY path.""" + version_info = { + "first_patched": "2.0.0", + "vulnerable_range": None, + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + } + result = classify_version_check( + installed_version="1.0.post1", + version_info=version_info, + description=None, + ecosystem="pypi", + package_name="somelib", + ) + assert result.path == VersionCheckPath.EASY + + def test_npm_prerelease_version_is_easy(self): + """npm semver pre-release versions should use EASY path with SemverVersion.""" + version_info = { + "first_patched": "1.0.0", + "vulnerable_range": None, + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + } + result = classify_version_check( + installed_version="1.0.0-beta.1", + version_info=version_info, + description=None, + ecosystem="npm", + package_name="somelib", + ) + assert result.path == VersionCheckPath.EASY + + def test_go_version_is_easy(self): + """Go versions should use EASY path with GolangVersion.""" + version_info = { + "first_patched": "v1.21.5", + "vulnerable_range": None, + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + } + result = classify_version_check( + installed_version="v1.21.0", + version_info=version_info, + description=None, + ecosystem="go", + package_name="somemodule", + ) + assert result.path == VersionCheckPath.EASY + + def test_go_pseudo_version_is_easy(self): + """Go pseudo versions (pre-release commits) should use EASY path with GolangVersion.""" + version_info = { + "first_patched": "v1.5.0", + "vulnerable_range": None, + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + } + result = classify_version_check( + installed_version="v1.4.1-0.20260512091500-f9e8d7c6b5a4", + version_info=version_info, + description=None, + ecosystem="go", + package_name="somemodule", + ) + assert result.path == VersionCheckPath.EASY + + def test_maven_version_is_easy(self): + """Maven versions should use EASY path with MavenVersion.""" + version_info = { + "first_patched": "2.0.0", + "vulnerable_range": None, + "version_start_incl": None, + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": None, + } + result = classify_version_check( + installed_version="1.0.0-SNAPSHOT", + version_info=version_info, + description=None, + ecosystem="maven", + package_name="com.example:artifact", + ) + assert result.path == VersionCheckPath.EASY + + +class TestDeterministicVersionCheck: + def test_lodash_is_vulnerable(self): + version_info = {"first_patched": "4.17.21"} + is_vulnerable, reason = deterministic_version_check( + installed_version="4.17.20", + version_info=version_info, + ecosystem="npm", + ) + assert is_vulnerable is True + assert "4.17.21" in reason + + def test_lodash_is_not_vulnerable_at_fix(self): + version_info = {"first_patched": "4.17.21"} + is_vulnerable, _reason = deterministic_version_check( + installed_version="4.17.21", + version_info=version_info, + ecosystem="npm", + ) + assert is_vulnerable is False + + def test_nvd_range_outside_is_not_vulnerable(self): + version_info = { + "version_start_incl": "1.0.0", + "version_start_excl": None, + "version_end_incl": None, + "version_end_excl": "1.5.0", + } + is_vulnerable, reason = deterministic_version_check( + installed_version="2.0.0", + version_info=version_info, + ecosystem="pypi", + ) + assert is_vulnerable is False + assert "outside" in reason + + +class TestGetCveDescription: + def test_prefers_nvd_description(self): + intel = CveIntel( + vuln_id="CVE-2025-1094", + nvd=CveIntelNvd( + cve_id="CVE-2025-1094", + cve_description=POSTGRES_MULTI_BRANCH_DESCRIPTION, + ), + ghsa=CveIntelGhsa( + ghsa_id="GHSA-xxxx-yyyy-zzzz", + summary="summary only", + ), + ) + assert get_cve_description(intel) == POSTGRES_MULTI_BRANCH_DESCRIPTION + + def test_appends_rhsa_statement_to_nvd_description(self): + """RHSA statement should be appended to NVD description when available.""" + rhsa_statement = "This vulnerability affects only the Go 1.24 release candidates." + intel = CveIntel( + vuln_id="CVE-2025-22865", + nvd=CveIntelNvd( + cve_id="CVE-2025-22865", + cve_description="Go crypto/x509 vulnerability description.", + ), + rhsa=CveIntelRhsa( + statement=rhsa_statement, + ), + ) + result = get_cve_description(intel) + assert "Go crypto/x509 vulnerability description." in result + assert f"RHSA Statement: {rhsa_statement}" in result + assert "\n\n" in result + + def test_appends_rhsa_statement_to_ghsa_description(self): + """RHSA statement should be appended to GHSA description when NVD is absent.""" + rhsa_statement = "Only affects specific configurations." + intel = CveIntel( + vuln_id="CVE-2025-12345", + ghsa=CveIntelGhsa( + ghsa_id="GHSA-abcd-efgh-ijkl", + description="GHSA description text.", + ), + rhsa=CveIntelRhsa( + statement=rhsa_statement, + ), + ) + result = get_cve_description(intel) + assert "GHSA description text." in result + assert f"RHSA Statement: {rhsa_statement}" in result + + def test_rhsa_statement_only_when_no_other_description(self): + """RHSA statement alone should be returned when no other description exists.""" + rhsa_statement = "Package not affected in RHEL 9." + intel = CveIntel( + vuln_id="CVE-2025-99999", + rhsa=CveIntelRhsa( + statement=rhsa_statement, + ), + ) + result = get_cve_description(intel) + assert result == f"RHSA Statement: {rhsa_statement}" + + def test_no_rhsa_statement_preserves_existing_behavior(self): + """When RHSA statement is None, existing behavior is preserved.""" + intel = CveIntel( + vuln_id="CVE-2025-1094", + nvd=CveIntelNvd( + cve_id="CVE-2025-1094", + cve_description=POSTGRES_MULTI_BRANCH_DESCRIPTION, + ), + rhsa=CveIntelRhsa( + details=["Some RHSA details."], + statement=None, + ), + ) + result = get_cve_description(intel) + assert result == POSTGRES_MULTI_BRANCH_DESCRIPTION + assert "RHSA Statement" not in result + + def test_rhsa_details_with_statement(self): + """RHSA details as primary description with statement appended.""" + rhsa_statement = "Upstream fix available in version 2.0." + intel = CveIntel( + vuln_id="CVE-2025-11111", + rhsa=CveIntelRhsa( + details=["First detail.", "Second detail."], + statement=rhsa_statement, + ), + ) + result = get_cve_description(intel) + assert "First detail. Second detail." in result + assert f"RHSA Statement: {rhsa_statement}" in result + + +class TestCheckStdlibVulnerability: + """Tests for check_stdlib_vulnerability() LLM fallback function. + + This function is used when no packages matched from structured intel (GHSA/NVD empty), + suggesting a stdlib/toolchain vulnerability (e.g., Go crypto/x509, Python ssl). + """ + + @pytest.mark.asyncio + async def test_returns_vulnerable_when_llm_says_vulnerable(self): + """LLM determines project's toolchain is vulnerable.""" + intel = CveIntel( + vuln_id="CVE-2025-22865", + nvd=CveIntelNvd( + cve_id="CVE-2025-22865", + cve_description="Go crypto/x509 vulnerability in parsing.", + ), + rhsa=CveIntelRhsa( + statement="This vulnerability affects Go 1.24 release candidates and Go 1.23.x before 1.23.5.", + ), + ) + + async def mock_llm_checker(prompt: str) -> StdlibVulnerabilityResult: + return StdlibVulnerabilityResult( + is_vulnerable=True, + affected_component="crypto/x509", + reason="Project uses Go 1.23.3 which is before the fix in 1.23.5", + ) + + is_vulnerable, component, reason = await check_stdlib_vulnerability( + cve_intel=intel, + ecosystem="go", + toolchain_version="1.23.3", + llm_checker=mock_llm_checker, + ) + + assert is_vulnerable is True + assert component == "crypto/x509" + assert "1.23.5" in reason + + @pytest.mark.asyncio + async def test_returns_not_vulnerable_when_llm_says_not_vulnerable(self): + """LLM determines project's toolchain is NOT vulnerable.""" + intel = CveIntel( + vuln_id="CVE-2025-22865", + nvd=CveIntelNvd( + cve_id="CVE-2025-22865", + cve_description="Go crypto/x509 vulnerability.", + ), + rhsa=CveIntelRhsa( + statement="This vulnerability affects only the Go 1.24 release candidates.", + ), + ) + + async def mock_llm_checker(prompt: str) -> StdlibVulnerabilityResult: + return StdlibVulnerabilityResult( + is_vulnerable=False, + affected_component="crypto/x509", + reason="CVE only affects Go 1.24 RC, project uses stable Go 1.21", + ) + + is_vulnerable, component, reason = await check_stdlib_vulnerability( + cve_intel=intel, + ecosystem="go", + toolchain_version="1.21", + llm_checker=mock_llm_checker, + ) + + assert is_vulnerable is False + assert component == "crypto/x509" + assert "1.24 RC" in reason + + @pytest.mark.asyncio + async def test_returns_fail_safe_when_llm_fails(self): + """LLM failure returns fail-safe: assume vulnerable.""" + intel = CveIntel( + vuln_id="CVE-2025-22865", + nvd=CveIntelNvd( + cve_id="CVE-2025-22865", + cve_description="Go crypto/x509 vulnerability.", + ), + ) + + async def mock_llm_checker_raises(prompt: str) -> StdlibVulnerabilityResult: + raise RuntimeError("LLM service unavailable") + + is_vulnerable, component, reason = await check_stdlib_vulnerability( + cve_intel=intel, + ecosystem="go", + toolchain_version="1.21", + llm_checker=mock_llm_checker_raises, + ) + + assert is_vulnerable is True + assert component == "unknown" + assert "LLM check failed" in reason + assert "LLM service unavailable" in reason + + @pytest.mark.asyncio + async def test_prompt_includes_cve_id_and_toolchain_version(self): + """Verify prompt contains key context for LLM analysis.""" + intel = CveIntel( + vuln_id="CVE-2025-22865", + nvd=CveIntelNvd( + cve_id="CVE-2025-22865", + cve_description="Test description.", + ), + ) + + captured_prompt: str = "" + + async def mock_llm_checker_capture(prompt: str) -> StdlibVulnerabilityResult: + nonlocal captured_prompt + captured_prompt = prompt + return StdlibVulnerabilityResult( + is_vulnerable=False, + affected_component="test", + reason="test reason", + ) + + await check_stdlib_vulnerability( + cve_intel=intel, + ecosystem="pypi", + toolchain_version="3.11.4", + llm_checker=mock_llm_checker_capture, + ) + + assert "CVE-2025-22865" in captured_prompt + assert "pypi" in captured_prompt + assert "3.11.4" in captured_prompt + assert "Test description." in captured_prompt diff --git a/tests/test_vex_csaf_helpers.py b/tests/test_vex_csaf_helpers.py new file mode 100644 index 000000000..687e52473 --- /dev/null +++ b/tests/test_vex_csaf_helpers.py @@ -0,0 +1,216 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Unit tests for CSAF VEX generator helper functions. +""" + +import pytest + +from exploit_iq_commons.data_models.cve_intel import CveIntel, CveIntelGhsa, CveIntelRhsa +from vuln_analysis.utils.vex.implementations.csaf_generator import ( + _enrich_vulnerabilities_with_notes, +) + + +# --- Fixtures for TestEnrichVulnerabilitiesWithNotes --- + +@pytest.fixture +def base_csaf_json(): + """Returns a fresh base CSAF JSON structure with one vulnerability.""" + return { + "vulnerabilities": [ + {"cve": "CVE-2024-1234", "notes": []} + ] + } + + +@pytest.fixture +def base_intel_map(): + """Returns a basic intel map with one CVE.""" + return {"CVE-2024-1234": CveIntel(vuln_id="CVE-2024-1234")} + + +@pytest.fixture +def base_final_summaries(): + """Returns basic final summaries dict.""" + return {"CVE-2024-1234": "Analysis summary"} + + +@pytest.fixture +def base_justifications(): + """Returns basic justifications dict.""" + return {"CVE-2024-1234": {"justification": "reasoning", "justification_label": "vulnerable"}} + + +class TestEnrichVulnerabilitiesWithNotes: + """Unit tests for _enrich_vulnerabilities_with_notes() function.""" + + def test_adds_ghsa_summary_note(self, base_csaf_json, base_final_summaries, base_justifications): + """Test that GHSA summary is added as a note.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + summary="Test vulnerability summary" + ) + intel_map = {"CVE-2024-1234": CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa)} + + _enrich_vulnerabilities_with_notes(base_csaf_json, intel_map, base_final_summaries, base_justifications) + + notes = base_csaf_json["vulnerabilities"][0]["notes"] + summary_notes = [n for n in notes if n.get("category") == "summary"] + assert len(summary_notes) == 1 + assert summary_notes[0]["text"] == "Test vulnerability summary" + assert summary_notes[0]["title"] == "Vulnerability summary" + + def test_adds_rhsa_statement_note(self, base_csaf_json, base_final_summaries, base_justifications): + """Test that RHSA statement is added as a note.""" + rhsa = CveIntelRhsa(statement="Red Hat security statement") + intel_map = {"CVE-2024-1234": CveIntel(vuln_id="CVE-2024-1234", rhsa=rhsa)} + + _enrich_vulnerabilities_with_notes(base_csaf_json, intel_map, base_final_summaries, base_justifications) + + notes = base_csaf_json["vulnerabilities"][0]["notes"] + general_notes = [n for n in notes if n.get("category") == "general"] + assert len(general_notes) == 1 + assert general_notes[0]["text"] == "Red Hat security statement" + assert general_notes[0]["title"] == "Red Hat Security Advisory Statement" + + def test_adds_analysis_summary_note(self, base_csaf_json, base_intel_map, base_justifications): + """Test that analysis summary is added as a note.""" + final_summaries = {"CVE-2024-1234": "This is the analysis summary"} + + _enrich_vulnerabilities_with_notes(base_csaf_json, base_intel_map, final_summaries, base_justifications) + + notes = base_csaf_json["vulnerabilities"][0]["notes"] + analysis_notes = [n for n in notes if n.get("title") == "ExploitIQ Analysis Summary"] + assert len(analysis_notes) == 1 + assert analysis_notes[0]["text"] == "This is the analysis summary" + assert analysis_notes[0]["category"] == "other" + + def test_adds_justification_notes(self, base_csaf_json, base_intel_map, base_final_summaries): + """Test that justification reasoning and label are added as notes.""" + justifications = { + "CVE-2024-1234": { + "justification": "The vulnerable code path is reachable", + "justification_label": "vulnerable" + } + } + + _enrich_vulnerabilities_with_notes(base_csaf_json, base_intel_map, base_final_summaries, justifications) + + notes = base_csaf_json["vulnerabilities"][0]["notes"] + + reasoning_notes = [n for n in notes if n.get("title") == "ExploitIQ Analysis Justification Reasoning"] + assert len(reasoning_notes) == 1 + assert reasoning_notes[0]["text"] == "The vulnerable code path is reachable" + + label_notes = [n for n in notes if n.get("title") == "ExploitIQ Analysis Justification Label"] + assert len(label_notes) == 1 + assert label_notes[0]["text"] == "vulnerable" + + def test_updates_existing_description_note(self, base_final_summaries, base_justifications): + """Test that existing description note is updated with GHSA description.""" + csaf_json = { + "vulnerabilities": [ + { + "cve": "CVE-2024-1234", + "notes": [ + {"category": "description", "text": "Original description", "title": ""} + ] + } + ] + } + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + description="GHSA detailed description" + ) + intel_map = {"CVE-2024-1234": CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa)} + + _enrich_vulnerabilities_with_notes(csaf_json, intel_map, base_final_summaries, base_justifications) + + notes = csaf_json["vulnerabilities"][0]["notes"] + desc_notes = [n for n in notes if n.get("category") == "description"] + assert len(desc_notes) == 1 + assert desc_notes[0]["text"] == "GHSA detailed description" + assert desc_notes[0]["title"] == "Vulnerability description" + + def test_removes_description_note_when_no_ghsa_description(self, base_intel_map, base_final_summaries, base_justifications): + """Test that description note is removed when no GHSA description is available.""" + csaf_json = { + "vulnerabilities": [ + { + "cve": "CVE-2024-1234", + "notes": [ + {"category": "description", "text": "Original description", "title": "Original"}, + {"category": "summary", "text": "Some summary", "title": "Summary"} + ] + } + ] + } + + _enrich_vulnerabilities_with_notes(csaf_json, base_intel_map, base_final_summaries, base_justifications) + + notes = csaf_json["vulnerabilities"][0]["notes"] + desc_notes = [n for n in notes if n.get("category") == "description"] + # Description note should be removed + assert len(desc_notes) == 0 + # Other notes should still be present + summary_notes = [n for n in notes if n.get("category") == "summary"] + assert len(summary_notes) >= 1 + + def test_handles_multiple_vulnerabilities(self): + """Test that multiple vulnerabilities are all enriched.""" + csaf_json = { + "vulnerabilities": [ + {"cve": "CVE-2024-1234", "notes": []}, + {"cve": "CVE-2024-5678", "notes": []}, + ] + } + intel_map = { + "CVE-2024-1234": CveIntel(vuln_id="CVE-2024-1234"), + "CVE-2024-5678": CveIntel(vuln_id="CVE-2024-5678"), + } + final_summaries = { + "CVE-2024-1234": "Summary 1", + "CVE-2024-5678": "Summary 2", + } + justifications = { + "CVE-2024-1234": {"justification": "reason1", "justification_label": "vulnerable"}, + "CVE-2024-5678": {"justification": "reason2", "justification_label": "not_vulnerable"}, + } + + _enrich_vulnerabilities_with_notes(csaf_json, intel_map, final_summaries, justifications) + + # Both vulnerabilities should have notes + for vuln in csaf_json["vulnerabilities"]: + assert len(vuln["notes"]) >= 3 # At least analysis summary + 2 justification notes + + def test_handles_missing_intel_for_vulnerability(self, base_csaf_json, base_final_summaries, base_justifications): + """Test that missing intel for a vulnerability is handled gracefully.""" + # Should not raise an exception + _enrich_vulnerabilities_with_notes(base_csaf_json, {}, base_final_summaries, base_justifications) + + notes = base_csaf_json["vulnerabilities"][0]["notes"] + # Should still have analysis notes + assert len(notes) >= 3 + + def test_handles_empty_vulnerabilities_list(self): + """Test that empty vulnerabilities list is handled.""" + csaf_json = {"vulnerabilities": []} + + # Should not raise an exception + _enrich_vulnerabilities_with_notes(csaf_json, {}, {}, {}) + + assert csaf_json["vulnerabilities"] == [] diff --git a/tests/test_vex_loader.py b/tests/test_vex_loader.py new file mode 100644 index 000000000..76987d787 --- /dev/null +++ b/tests/test_vex_loader.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Unit tests for VEX generator loader/factory. + +Tests the load_vex_generator() function in loader.py. +""" + +import pytest + +from vuln_analysis.utils.vex.vex_generator_loader import load_vex_generator +from vuln_analysis.utils.vex.vex_generator_base import VexGenerator +from vuln_analysis.utils.vex.implementations.csaf_generator import CsafVexGenerator + + +class TestLoadVexGenerator: + """Unit tests for load_vex_generator() factory function.""" + + def test_csaf_format_returns_csaf_generator(self): + """Test that 'csaf' format returns CsafVexGenerator instance.""" + generator = load_vex_generator("csaf") + assert isinstance(generator, CsafVexGenerator) + + def test_csaf_format_returns_vex_generator_subclass(self): + """Test that returned generator is a VexGenerator subclass.""" + generator = load_vex_generator("csaf") + assert isinstance(generator, VexGenerator) + + def test_csaf_uppercase_is_case_insensitive(self): + """Test that format matching is case insensitive (uppercase).""" + generator = load_vex_generator("CSAF") + assert isinstance(generator, CsafVexGenerator) + + def test_invalid_format_raises_value_error(self): + """Test that unsupported format raises ValueError.""" + with pytest.raises(ValueError, match="Unsupported VEX format"): + load_vex_generator("WrongFormat") + + def test_empty_format_raises_value_error(self): + """Test that empty format string raises ValueError.""" + with pytest.raises(ValueError, match="Unsupported VEX format"): + load_vex_generator("") \ No newline at end of file diff --git a/tests/test_vex_utils.py b/tests/test_vex_utils.py new file mode 100644 index 000000000..1e7a28ba7 --- /dev/null +++ b/tests/test_vex_utils.py @@ -0,0 +1,232 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Unit tests for VEX utility functions in vex_utils.py. +""" + +from pathlib import Path +import tempfile + +import pytest +from jsonschema import Draft202012Validator + +from exploit_iq_commons.data_models.cve_intel import CveIntel, CveIntelGhsa +from vuln_analysis.utils.vex.implementations.csaf_generator import ( + CSAF_SCHEMA_PATH as vex_schema_path_example, +) +from vuln_analysis.utils.vex.vex_utils import ( + get_vex_validator, + get_patched_package, + build_patch_recommendation, +) + + +class TestGetVexValidator: + """Unit tests for get_vex_validator() function.""" + + def test_returns_draft202012_validator(self): + """Test that get_vex_validator returns a Draft202012Validator instance.""" + validator = get_vex_validator(vex_schema_path_example) + assert isinstance(validator, Draft202012Validator) + + def test_caching_returns_same_instance(self): + """Test that calling with same path returns the same cached validator (same object in memory and not just equal in vlaue).""" + validator1 = get_vex_validator(vex_schema_path_example) + validator2 = get_vex_validator(vex_schema_path_example) + assert validator1 is validator2 + + def test_different_paths_return_different_validators(self): + """Test that different schema paths return different validator instances.""" + minimal_schema = '{"type": "object"}' + + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=True) as f: + f.write(minimal_schema) + f.flush() # Ensure all contents are written before reading + temp_path = Path(f.name) + validator1 = get_vex_validator(vex_schema_path_example) + validator2 = get_vex_validator(temp_path) + assert validator1 is not validator2 + + def test_validator_can_validate_documents(self): + """Test that returned validator can actually validate documents.""" + validator = get_vex_validator(vex_schema_path_example) + + errors = list(validator.iter_errors({})) + assert len(errors) > 0 + + def test_invalid_path_raises_file_not_found(self): + """Test that invalid schema path raises FileNotFoundError.""" + invalid_path = Path("/nonexistent/path/schema.json") + + with pytest.raises(FileNotFoundError): + get_vex_validator(invalid_path) + + +class TestGetPatchedPackage: + """Unit tests for get_patched_package() function.""" + + def test_valid_package_returns_name_and_version(self): + """Test extraction of name and version from valid vulnerability dict.""" + vuln = { + "package": {"name": "lodash"}, + "first_patched_version": "4.17.21" + } + result = get_patched_package(vuln) + assert result == ("lodash", "4.17.21") + + def test_empty_dict_returns_none_tuple(self): + """Test that empty dict returns (None, None).""" + result = get_patched_package({}) + assert result == (None, None) + + def test_missing_package_key_returns_none_name(self): + """Test that missing 'package' key returns None for name.""" + vuln = {"first_patched_version": "1.0.0"} + result = get_patched_package(vuln) + assert result == (None, "1.0.0") + + def test_missing_version_returns_none_version(self): + """Test that missing version returns None for version.""" + vuln = {"package": {"name": "express"}} + result = get_patched_package(vuln) + assert result == ("express", None) + + def test_null_package_returns_none_name(self): + """Test that null package value returns None for name.""" + vuln = {"package": None, "first_patched_version": "2.0.0"} + result = get_patched_package(vuln) + assert result == (None, "2.0.0") + + def test_empty_package_dict_returns_none_name(self): + """Test that empty package dict returns None for name.""" + vuln = {"package": {}, "first_patched_version": "3.0.0"} + result = get_patched_package(vuln) + assert result == (None, "3.0.0") + + +class TestBuildPatchRecommendation: + """Unit tests for build_patch_recommendation() function.""" + + def test_returns_empty_when_intel_is_none(self): + """Test that None intel returns empty string.""" + result = build_patch_recommendation(None, None) + assert result == "" + + def test_returns_empty_when_ghsa_is_none(self): + """Test that intel without GHSA returns empty string.""" + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=None) + result = build_patch_recommendation(ci, None) + assert result == "" + + def test_returns_empty_when_vulnerabilities_is_none(self): + """Test that GHSA without vulnerabilities returns empty string.""" + ghsa = CveIntelGhsa(ghsa_id="GHSA-1234-5678-9012", vulnerabilities=None) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + result = build_patch_recommendation(ci, None) + assert result == "" + + def test_returns_empty_when_vulnerabilities_is_empty(self): + """Test that empty vulnerabilities list returns empty string.""" + ghsa = CveIntelGhsa(ghsa_id="GHSA-1234-5678-9012", vulnerabilities=[]) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + result = build_patch_recommendation(ci, None) + assert result == "" + + def test_with_sbom_returns_matching_package(self): + """Test that with SBOM, only matching package is returned.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, + {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, + ] + ) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + + result = build_patch_recommendation(ci, {"lodash", "express", "react"}) + assert result == "lodash:4.17.21, express:4.18.0" + + def test_with_sbom_no_match_returns_empty(self): + """Test that with SBOM but no match, empty string is returned.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, + ] + ) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + + result = build_patch_recommendation(ci, {"react", "vue"}) + assert result == "" + + def test_without_sbom_returns_all_packages(self): + """Test that without SBOM, all packages are returned.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, + {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, + ] + ) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + + result = build_patch_recommendation(ci, None) + assert "lodash:4.17.21" in result + assert "express:4.18.0" in result + + def test_without_sbom_deduplicates_packages(self): + """Test that duplicate package names are deduplicated.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}, "first_patched_version": "4.17.21"}, + {"package": {"name": "lodash"}, "first_patched_version": "4.17.22"}, + ] + ) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + + result = build_patch_recommendation(ci, None) + # First version should win + assert result == "lodash:4.17.21" + + def test_skips_vulnerabilities_without_name(self): + """Test that vulnerabilities without package name are skipped.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {}, "first_patched_version": "1.0.0"}, + {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, + ] + ) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + + result = build_patch_recommendation(ci, None) + assert result == "express:4.18.0" + + def test_skips_vulnerabilities_without_version(self): + """Test that vulnerabilities without patched version are skipped.""" + ghsa = CveIntelGhsa( + ghsa_id="GHSA-1234-5678-9012", + vulnerabilities=[ + {"package": {"name": "lodash"}}, + {"package": {"name": "express"}, "first_patched_version": "4.18.0"}, + ] + ) + ci = CveIntel(vuln_id="CVE-2024-1234", ghsa=ghsa) + + result = build_patch_recommendation(ci, None) + assert result == "express:4.18.0" + diff --git a/try b/try new file mode 100644 index 000000000..e69de29bb diff --git a/uv.lock b/uv.lock index e179dc1f2..ed640a803 100644 --- a/uv.lock +++ b/uv.lock @@ -1,32 +1,43 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.11, <3.13" resolution-markers = [ "python_full_version >= '3.12' and sys_platform == 'darwin'", - "python_full_version >= '3.12' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and sys_platform == 'linux'", + "python_full_version >= '3.12' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version < '3.12' and sys_platform == 'darwin'", - "python_full_version < '3.12' and sys_platform != 'darwin'", + "python_full_version < '3.12' and sys_platform == 'linux'", + "python_full_version < '3.12' and sys_platform != 'darwin' and sys_platform != 'linux'", ] -[options] -prerelease-mode = "allow" +[[package]] +name = "abnf" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/f2/7b5fac50ee42e8b8d4a098d76743a394546f938c94125adbb93414e5ae7d/abnf-2.2.0.tar.gz", hash = "sha256:433380fd32855bbc60bc7b3d35d40616e21383a32ed1c9b8893d16d9f4a6c2f4", size = 197507, upload-time = "2023-03-17T18:26:24.577Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/95/f456ae7928a2f3a913f467d4fd9e662e295dd7349fc58b35f77f6c757a23/abnf-2.2.0-py3-none-any.whl", hash = "sha256:5dc2ae31a84ff454f7de46e08a2a21a442a0e21a092468420587a1590b490d1f", size = 39938, upload-time = "2023-03-17T18:26:22.608Z" }, +] [[package]] name = "aioboto3" -version = "15.0.0" +version = "15.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiobotocore", extra = ["boto3"] }, { name = "aiofiles" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/80/d0/ed107e16551ba1b93ddcca9a6bf79580450945268a8bc396530687b3189f/aioboto3-15.0.0.tar.gz", hash = "sha256:dce40b701d1f8e0886dc874d27cd9799b8bf6b32d63743f57e7bef7e4a562756", size = 225278, upload-time = "2025-06-26T16:30:48.967Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/b1/b0331786c50f6ef881f9a71c3441ccf7b64c7eed210297d882c37ce31713/aioboto3-15.1.0.tar.gz", hash = "sha256:37763bbc6321ceb479106dc63bc84c8fdb59dd02540034a12941aebef2057c5c", size = 234664, upload-time = "2025-08-14T19:49:15.35Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/95/d69c744f408e5e4592fe53ed98fc244dd13b83d84cf1f83b2499d98bfcc9/aioboto3-15.0.0-py3-none-any.whl", hash = "sha256:9cf54b3627c8b34bb82eaf43ab327e7027e37f92b1e10dd5cfe343cd512568d0", size = 35785, upload-time = "2025-06-26T16:30:47.444Z" }, + { url = "https://files.pythonhosted.org/packages/93/b0/28e3ac89e7119b1cb4e6830664060b96a2b5761291e92a10fb3044b5a11d/aioboto3-15.1.0-py3-none-any.whl", hash = "sha256:66006142a2ccc7d6d07aa260ba291c4922b6767d270ba42f95c59e85d8b3e645", size = 35791, upload-time = "2025-08-14T19:49:14.14Z" }, ] [[package]] name = "aiobotocore" -version = "2.23.0" +version = "2.24.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -37,9 +48,9 @@ dependencies = [ { name = "python-dateutil" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9d/25/4b06ea1214ddf020a28df27dc7136ac9dfaf87929d51e6f6044dd350ed67/aiobotocore-2.23.0.tar.gz", hash = "sha256:0333931365a6c7053aee292fe6ef50c74690c4ae06bb019afdf706cb6f2f5e32", size = 115825, upload-time = "2025-06-12T23:46:38.055Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/ca/ac82c0c699815b6d5b4017f3d8fb2c2d49537f4937f4a0bdf58b4c75d321/aiobotocore-2.24.0.tar.gz", hash = "sha256:b32c0c45d38c22a18ce395a0b5448606c5260603296a152895b5bdb40ab3139d", size = 119597, upload-time = "2025-08-08T18:26:50.373Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ea/43/ccf9b29669cdb09fd4bfc0a8effeb2973b22a0f3c3be4142d0b485975d11/aiobotocore-2.23.0-py3-none-any.whl", hash = "sha256:8202cebbf147804a083a02bc282fbfda873bfdd0065fd34b64784acb7757b66e", size = 84161, upload-time = "2025-06-12T23:46:36.305Z" }, + { url = "https://files.pythonhosted.org/packages/e2/68/b29577197aa2e54b50d6f214524790cc1cb27d289585ad7c7bdfe5125285/aiobotocore-2.24.0-py3-none-any.whl", hash = "sha256:72bb1f8eb1b962779a95e1bcc9cf35bc33196ad763b622a40ae7fa9d2e95c87c", size = 84971, upload-time = "2025-08-08T18:26:48.777Z" }, ] [package.optional-dependencies] @@ -67,7 +78,7 @@ wheels = [ [[package]] name = "aiohttp" -version = "3.12.14" +version = "3.12.15" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohappyeyeballs" }, @@ -78,42 +89,42 @@ dependencies = [ { name = "propcache" }, { name = "yarl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e6/0b/e39ad954107ebf213a2325038a3e7a506be3d98e1435e1f82086eec4cde2/aiohttp-3.12.14.tar.gz", hash = "sha256:6e06e120e34d93100de448fd941522e11dafa78ef1a893c179901b7d66aa29f2", size = 7822921, upload-time = "2025-07-10T13:05:33.968Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/53/e1/8029b29316971c5fa89cec170274582619a01b3d82dd1036872acc9bc7e8/aiohttp-3.12.14-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f4552ff7b18bcec18b60a90c6982049cdb9dac1dba48cf00b97934a06ce2e597", size = 709960, upload-time = "2025-07-10T13:03:11.936Z" }, - { url = "https://files.pythonhosted.org/packages/96/bd/4f204cf1e282041f7b7e8155f846583b19149e0872752711d0da5e9cc023/aiohttp-3.12.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8283f42181ff6ccbcf25acaae4e8ab2ff7e92b3ca4a4ced73b2c12d8cd971393", size = 482235, upload-time = "2025-07-10T13:03:14.118Z" }, - { url = "https://files.pythonhosted.org/packages/d6/0f/2a580fcdd113fe2197a3b9df30230c7e85bb10bf56f7915457c60e9addd9/aiohttp-3.12.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:040afa180ea514495aaff7ad34ec3d27826eaa5d19812730fe9e529b04bb2179", size = 470501, upload-time = "2025-07-10T13:03:16.153Z" }, - { url = "https://files.pythonhosted.org/packages/38/78/2c1089f6adca90c3dd74915bafed6d6d8a87df5e3da74200f6b3a8b8906f/aiohttp-3.12.14-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b413c12f14c1149f0ffd890f4141a7471ba4b41234fe4fd4a0ff82b1dc299dbb", size = 1740696, upload-time = "2025-07-10T13:03:18.4Z" }, - { url = "https://files.pythonhosted.org/packages/4a/c8/ce6c7a34d9c589f007cfe064da2d943b3dee5aabc64eaecd21faf927ab11/aiohttp-3.12.14-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1d6f607ce2e1a93315414e3d448b831238f1874b9968e1195b06efaa5c87e245", size = 1689365, upload-time = "2025-07-10T13:03:20.629Z" }, - { url = "https://files.pythonhosted.org/packages/18/10/431cd3d089de700756a56aa896faf3ea82bee39d22f89db7ddc957580308/aiohttp-3.12.14-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:565e70d03e924333004ed101599902bba09ebb14843c8ea39d657f037115201b", size = 1788157, upload-time = "2025-07-10T13:03:22.44Z" }, - { url = "https://files.pythonhosted.org/packages/fa/b2/26f4524184e0f7ba46671c512d4b03022633bcf7d32fa0c6f1ef49d55800/aiohttp-3.12.14-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4699979560728b168d5ab63c668a093c9570af2c7a78ea24ca5212c6cdc2b641", size = 1827203, upload-time = "2025-07-10T13:03:24.628Z" }, - { url = "https://files.pythonhosted.org/packages/e0/30/aadcdf71b510a718e3d98a7bfeaea2396ac847f218b7e8edb241b09bd99a/aiohttp-3.12.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad5fdf6af93ec6c99bf800eba3af9a43d8bfd66dce920ac905c817ef4a712afe", size = 1729664, upload-time = "2025-07-10T13:03:26.412Z" }, - { url = "https://files.pythonhosted.org/packages/67/7f/7ccf11756ae498fdedc3d689a0c36ace8fc82f9d52d3517da24adf6e9a74/aiohttp-3.12.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4ac76627c0b7ee0e80e871bde0d376a057916cb008a8f3ffc889570a838f5cc7", size = 1666741, upload-time = "2025-07-10T13:03:28.167Z" }, - { url = "https://files.pythonhosted.org/packages/6b/4d/35ebc170b1856dd020c92376dbfe4297217625ef4004d56587024dc2289c/aiohttp-3.12.14-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:798204af1180885651b77bf03adc903743a86a39c7392c472891649610844635", size = 1715013, upload-time = "2025-07-10T13:03:30.018Z" }, - { url = "https://files.pythonhosted.org/packages/7b/24/46dc0380146f33e2e4aa088b92374b598f5bdcde1718c77e8d1a0094f1a4/aiohttp-3.12.14-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:4f1205f97de92c37dd71cf2d5bcfb65fdaed3c255d246172cce729a8d849b4da", size = 1710172, upload-time = "2025-07-10T13:03:31.821Z" }, - { url = "https://files.pythonhosted.org/packages/2f/0a/46599d7d19b64f4d0fe1b57bdf96a9a40b5c125f0ae0d8899bc22e91fdce/aiohttp-3.12.14-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:76ae6f1dd041f85065d9df77c6bc9c9703da9b5c018479d20262acc3df97d419", size = 1690355, upload-time = "2025-07-10T13:03:34.754Z" }, - { url = "https://files.pythonhosted.org/packages/08/86/b21b682e33d5ca317ef96bd21294984f72379454e689d7da584df1512a19/aiohttp-3.12.14-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a194ace7bc43ce765338ca2dfb5661489317db216ea7ea700b0332878b392cab", size = 1783958, upload-time = "2025-07-10T13:03:36.53Z" }, - { url = "https://files.pythonhosted.org/packages/4f/45/f639482530b1396c365f23c5e3b1ae51c9bc02ba2b2248ca0c855a730059/aiohttp-3.12.14-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:16260e8e03744a6fe3fcb05259eeab8e08342c4c33decf96a9dad9f1187275d0", size = 1804423, upload-time = "2025-07-10T13:03:38.504Z" }, - { url = "https://files.pythonhosted.org/packages/7e/e5/39635a9e06eed1d73671bd4079a3caf9cf09a49df08490686f45a710b80e/aiohttp-3.12.14-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8c779e5ebbf0e2e15334ea404fcce54009dc069210164a244d2eac8352a44b28", size = 1717479, upload-time = "2025-07-10T13:03:40.158Z" }, - { url = "https://files.pythonhosted.org/packages/51/e1/7f1c77515d369b7419c5b501196526dad3e72800946c0099594c1f0c20b4/aiohttp-3.12.14-cp311-cp311-win32.whl", hash = "sha256:a289f50bf1bd5be227376c067927f78079a7bdeccf8daa6a9e65c38bae14324b", size = 427907, upload-time = "2025-07-10T13:03:41.801Z" }, - { url = "https://files.pythonhosted.org/packages/06/24/a6bf915c85b7a5b07beba3d42b3282936b51e4578b64a51e8e875643c276/aiohttp-3.12.14-cp311-cp311-win_amd64.whl", hash = "sha256:0b8a69acaf06b17e9c54151a6c956339cf46db4ff72b3ac28516d0f7068f4ced", size = 452334, upload-time = "2025-07-10T13:03:43.485Z" }, - { url = "https://files.pythonhosted.org/packages/c3/0d/29026524e9336e33d9767a1e593ae2b24c2b8b09af7c2bd8193762f76b3e/aiohttp-3.12.14-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a0ecbb32fc3e69bc25efcda7d28d38e987d007096cbbeed04f14a6662d0eee22", size = 701055, upload-time = "2025-07-10T13:03:45.59Z" }, - { url = "https://files.pythonhosted.org/packages/0a/b8/a5e8e583e6c8c1056f4b012b50a03c77a669c2e9bf012b7cf33d6bc4b141/aiohttp-3.12.14-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0400f0ca9bb3e0b02f6466421f253797f6384e9845820c8b05e976398ac1d81a", size = 475670, upload-time = "2025-07-10T13:03:47.249Z" }, - { url = "https://files.pythonhosted.org/packages/29/e8/5202890c9e81a4ec2c2808dd90ffe024952e72c061729e1d49917677952f/aiohttp-3.12.14-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a56809fed4c8a830b5cae18454b7464e1529dbf66f71c4772e3cfa9cbec0a1ff", size = 468513, upload-time = "2025-07-10T13:03:49.377Z" }, - { url = "https://files.pythonhosted.org/packages/23/e5/d11db8c23d8923d3484a27468a40737d50f05b05eebbb6288bafcb467356/aiohttp-3.12.14-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27f2e373276e4755691a963e5d11756d093e346119f0627c2d6518208483fb6d", size = 1715309, upload-time = "2025-07-10T13:03:51.556Z" }, - { url = "https://files.pythonhosted.org/packages/53/44/af6879ca0eff7a16b1b650b7ea4a827301737a350a464239e58aa7c387ef/aiohttp-3.12.14-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ca39e433630e9a16281125ef57ece6817afd1d54c9f1bf32e901f38f16035869", size = 1697961, upload-time = "2025-07-10T13:03:53.511Z" }, - { url = "https://files.pythonhosted.org/packages/bb/94/18457f043399e1ec0e59ad8674c0372f925363059c276a45a1459e17f423/aiohttp-3.12.14-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9c748b3f8b14c77720132b2510a7d9907a03c20ba80f469e58d5dfd90c079a1c", size = 1753055, upload-time = "2025-07-10T13:03:55.368Z" }, - { url = "https://files.pythonhosted.org/packages/26/d9/1d3744dc588fafb50ff8a6226d58f484a2242b5dd93d8038882f55474d41/aiohttp-3.12.14-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0a568abe1b15ce69d4cc37e23020720423f0728e3cb1f9bcd3f53420ec3bfe7", size = 1799211, upload-time = "2025-07-10T13:03:57.216Z" }, - { url = "https://files.pythonhosted.org/packages/73/12/2530fb2b08773f717ab2d249ca7a982ac66e32187c62d49e2c86c9bba9b4/aiohttp-3.12.14-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9888e60c2c54eaf56704b17feb558c7ed6b7439bca1e07d4818ab878f2083660", size = 1718649, upload-time = "2025-07-10T13:03:59.469Z" }, - { url = "https://files.pythonhosted.org/packages/b9/34/8d6015a729f6571341a311061b578e8b8072ea3656b3d72329fa0faa2c7c/aiohttp-3.12.14-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3006a1dc579b9156de01e7916d38c63dc1ea0679b14627a37edf6151bc530088", size = 1634452, upload-time = "2025-07-10T13:04:01.698Z" }, - { url = "https://files.pythonhosted.org/packages/ff/4b/08b83ea02595a582447aeb0c1986792d0de35fe7a22fb2125d65091cbaf3/aiohttp-3.12.14-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa8ec5c15ab80e5501a26719eb48a55f3c567da45c6ea5bb78c52c036b2655c7", size = 1695511, upload-time = "2025-07-10T13:04:04.165Z" }, - { url = "https://files.pythonhosted.org/packages/b5/66/9c7c31037a063eec13ecf1976185c65d1394ded4a5120dd5965e3473cb21/aiohttp-3.12.14-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:39b94e50959aa07844c7fe2206b9f75d63cc3ad1c648aaa755aa257f6f2498a9", size = 1716967, upload-time = "2025-07-10T13:04:06.132Z" }, - { url = "https://files.pythonhosted.org/packages/ba/02/84406e0ad1acb0fb61fd617651ab6de760b2d6a31700904bc0b33bd0894d/aiohttp-3.12.14-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:04c11907492f416dad9885d503fbfc5dcb6768d90cad8639a771922d584609d3", size = 1657620, upload-time = "2025-07-10T13:04:07.944Z" }, - { url = "https://files.pythonhosted.org/packages/07/53/da018f4013a7a179017b9a274b46b9a12cbeb387570f116964f498a6f211/aiohttp-3.12.14-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:88167bd9ab69bb46cee91bd9761db6dfd45b6e76a0438c7e884c3f8160ff21eb", size = 1737179, upload-time = "2025-07-10T13:04:10.182Z" }, - { url = "https://files.pythonhosted.org/packages/49/e8/ca01c5ccfeaafb026d85fa4f43ceb23eb80ea9c1385688db0ef322c751e9/aiohttp-3.12.14-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:791504763f25e8f9f251e4688195e8b455f8820274320204f7eafc467e609425", size = 1765156, upload-time = "2025-07-10T13:04:12.029Z" }, - { url = "https://files.pythonhosted.org/packages/22/32/5501ab525a47ba23c20613e568174d6c63aa09e2caa22cded5c6ea8e3ada/aiohttp-3.12.14-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2785b112346e435dd3a1a67f67713a3fe692d288542f1347ad255683f066d8e0", size = 1724766, upload-time = "2025-07-10T13:04:13.961Z" }, - { url = "https://files.pythonhosted.org/packages/06/af/28e24574801fcf1657945347ee10df3892311c2829b41232be6089e461e7/aiohttp-3.12.14-cp312-cp312-win32.whl", hash = "sha256:15f5f4792c9c999a31d8decf444e79fcfd98497bf98e94284bf390a7bb8c1729", size = 422641, upload-time = "2025-07-10T13:04:16.018Z" }, - { url = "https://files.pythonhosted.org/packages/98/d5/7ac2464aebd2eecac38dbe96148c9eb487679c512449ba5215d233755582/aiohttp-3.12.14-cp312-cp312-win_amd64.whl", hash = "sha256:3b66e1a182879f579b105a80d5c4bd448b91a57e8933564bf41665064796a338", size = 449316, upload-time = "2025-07-10T13:04:18.289Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/9b/e7/d92a237d8802ca88483906c388f7c201bbe96cd80a165ffd0ac2f6a8d59f/aiohttp-3.12.15.tar.gz", hash = "sha256:4fc61385e9c98d72fcdf47e6dd81833f47b2f77c114c29cd64a361be57a763a2", size = 7823716, upload-time = "2025-07-29T05:52:32.215Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/19/9e86722ec8e835959bd97ce8c1efa78cf361fa4531fca372551abcc9cdd6/aiohttp-3.12.15-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d3ce17ce0220383a0f9ea07175eeaa6aa13ae5a41f30bc61d84df17f0e9b1117", size = 711246, upload-time = "2025-07-29T05:50:15.937Z" }, + { url = "https://files.pythonhosted.org/packages/71/f9/0a31fcb1a7d4629ac9d8f01f1cb9242e2f9943f47f5d03215af91c3c1a26/aiohttp-3.12.15-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:010cc9bbd06db80fe234d9003f67e97a10fe003bfbedb40da7d71c1008eda0fe", size = 483515, upload-time = "2025-07-29T05:50:17.442Z" }, + { url = "https://files.pythonhosted.org/packages/62/6c/94846f576f1d11df0c2e41d3001000527c0fdf63fce7e69b3927a731325d/aiohttp-3.12.15-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3f9d7c55b41ed687b9d7165b17672340187f87a773c98236c987f08c858145a9", size = 471776, upload-time = "2025-07-29T05:50:19.568Z" }, + { url = "https://files.pythonhosted.org/packages/f8/6c/f766d0aaafcee0447fad0328da780d344489c042e25cd58fde566bf40aed/aiohttp-3.12.15-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc4fbc61bb3548d3b482f9ac7ddd0f18c67e4225aaa4e8552b9f1ac7e6bda9e5", size = 1741977, upload-time = "2025-07-29T05:50:21.665Z" }, + { url = "https://files.pythonhosted.org/packages/17/e5/fb779a05ba6ff44d7bc1e9d24c644e876bfff5abe5454f7b854cace1b9cc/aiohttp-3.12.15-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7fbc8a7c410bb3ad5d595bb7118147dfbb6449d862cc1125cf8867cb337e8728", size = 1690645, upload-time = "2025-07-29T05:50:23.333Z" }, + { url = "https://files.pythonhosted.org/packages/37/4e/a22e799c2035f5d6a4ad2cf8e7c1d1bd0923192871dd6e367dafb158b14c/aiohttp-3.12.15-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:74dad41b3458dbb0511e760fb355bb0b6689e0630de8a22b1b62a98777136e16", size = 1789437, upload-time = "2025-07-29T05:50:25.007Z" }, + { url = "https://files.pythonhosted.org/packages/28/e5/55a33b991f6433569babb56018b2fb8fb9146424f8b3a0c8ecca80556762/aiohttp-3.12.15-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b6f0af863cf17e6222b1735a756d664159e58855da99cfe965134a3ff63b0b0", size = 1828482, upload-time = "2025-07-29T05:50:26.693Z" }, + { url = "https://files.pythonhosted.org/packages/c6/82/1ddf0ea4f2f3afe79dffed5e8a246737cff6cbe781887a6a170299e33204/aiohttp-3.12.15-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5b7fe4972d48a4da367043b8e023fb70a04d1490aa7d68800e465d1b97e493b", size = 1730944, upload-time = "2025-07-29T05:50:28.382Z" }, + { url = "https://files.pythonhosted.org/packages/1b/96/784c785674117b4cb3877522a177ba1b5e4db9ce0fd519430b5de76eec90/aiohttp-3.12.15-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6443cca89553b7a5485331bc9bedb2342b08d073fa10b8c7d1c60579c4a7b9bd", size = 1668020, upload-time = "2025-07-29T05:50:30.032Z" }, + { url = "https://files.pythonhosted.org/packages/12/8a/8b75f203ea7e5c21c0920d84dd24a5c0e971fe1e9b9ebbf29ae7e8e39790/aiohttp-3.12.15-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6c5f40ec615e5264f44b4282ee27628cea221fcad52f27405b80abb346d9f3f8", size = 1716292, upload-time = "2025-07-29T05:50:31.983Z" }, + { url = "https://files.pythonhosted.org/packages/47/0b/a1451543475bb6b86a5cfc27861e52b14085ae232896a2654ff1231c0992/aiohttp-3.12.15-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2abbb216a1d3a2fe86dbd2edce20cdc5e9ad0be6378455b05ec7f77361b3ab50", size = 1711451, upload-time = "2025-07-29T05:50:33.989Z" }, + { url = "https://files.pythonhosted.org/packages/55/fd/793a23a197cc2f0d29188805cfc93aa613407f07e5f9da5cd1366afd9d7c/aiohttp-3.12.15-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:db71ce547012a5420a39c1b744d485cfb823564d01d5d20805977f5ea1345676", size = 1691634, upload-time = "2025-07-29T05:50:35.846Z" }, + { url = "https://files.pythonhosted.org/packages/ca/bf/23a335a6670b5f5dfc6d268328e55a22651b440fca341a64fccf1eada0c6/aiohttp-3.12.15-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:ced339d7c9b5030abad5854aa5413a77565e5b6e6248ff927d3e174baf3badf7", size = 1785238, upload-time = "2025-07-29T05:50:37.597Z" }, + { url = "https://files.pythonhosted.org/packages/57/4f/ed60a591839a9d85d40694aba5cef86dde9ee51ce6cca0bb30d6eb1581e7/aiohttp-3.12.15-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:7c7dd29c7b5bda137464dc9bfc738d7ceea46ff70309859ffde8c022e9b08ba7", size = 1805701, upload-time = "2025-07-29T05:50:39.591Z" }, + { url = "https://files.pythonhosted.org/packages/85/e0/444747a9455c5de188c0f4a0173ee701e2e325d4b2550e9af84abb20cdba/aiohttp-3.12.15-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:421da6fd326460517873274875c6c5a18ff225b40da2616083c5a34a7570b685", size = 1718758, upload-time = "2025-07-29T05:50:41.292Z" }, + { url = "https://files.pythonhosted.org/packages/36/ab/1006278d1ffd13a698e5dd4bfa01e5878f6bddefc296c8b62649753ff249/aiohttp-3.12.15-cp311-cp311-win32.whl", hash = "sha256:4420cf9d179ec8dfe4be10e7d0fe47d6d606485512ea2265b0d8c5113372771b", size = 428868, upload-time = "2025-07-29T05:50:43.063Z" }, + { url = "https://files.pythonhosted.org/packages/10/97/ad2b18700708452400278039272032170246a1bf8ec5d832772372c71f1a/aiohttp-3.12.15-cp311-cp311-win_amd64.whl", hash = "sha256:edd533a07da85baa4b423ee8839e3e91681c7bfa19b04260a469ee94b778bf6d", size = 453273, upload-time = "2025-07-29T05:50:44.613Z" }, + { url = "https://files.pythonhosted.org/packages/63/97/77cb2450d9b35f517d6cf506256bf4f5bda3f93a66b4ad64ba7fc917899c/aiohttp-3.12.15-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:802d3868f5776e28f7bf69d349c26fc0efadb81676d0afa88ed00d98a26340b7", size = 702333, upload-time = "2025-07-29T05:50:46.507Z" }, + { url = "https://files.pythonhosted.org/packages/83/6d/0544e6b08b748682c30b9f65640d006e51f90763b41d7c546693bc22900d/aiohttp-3.12.15-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f2800614cd560287be05e33a679638e586a2d7401f4ddf99e304d98878c29444", size = 476948, upload-time = "2025-07-29T05:50:48.067Z" }, + { url = "https://files.pythonhosted.org/packages/3a/1d/c8c40e611e5094330284b1aea8a4b02ca0858f8458614fa35754cab42b9c/aiohttp-3.12.15-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8466151554b593909d30a0a125d638b4e5f3836e5aecde85b66b80ded1cb5b0d", size = 469787, upload-time = "2025-07-29T05:50:49.669Z" }, + { url = "https://files.pythonhosted.org/packages/38/7d/b76438e70319796bfff717f325d97ce2e9310f752a267bfdf5192ac6082b/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e5a495cb1be69dae4b08f35a6c4579c539e9b5706f606632102c0f855bcba7c", size = 1716590, upload-time = "2025-07-29T05:50:51.368Z" }, + { url = "https://files.pythonhosted.org/packages/79/b1/60370d70cdf8b269ee1444b390cbd72ce514f0d1cd1a715821c784d272c9/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6404dfc8cdde35c69aaa489bb3542fb86ef215fc70277c892be8af540e5e21c0", size = 1699241, upload-time = "2025-07-29T05:50:53.628Z" }, + { url = "https://files.pythonhosted.org/packages/a3/2b/4968a7b8792437ebc12186db31523f541943e99bda8f30335c482bea6879/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3ead1c00f8521a5c9070fcb88f02967b1d8a0544e6d85c253f6968b785e1a2ab", size = 1754335, upload-time = "2025-07-29T05:50:55.394Z" }, + { url = "https://files.pythonhosted.org/packages/fb/c1/49524ed553f9a0bec1a11fac09e790f49ff669bcd14164f9fab608831c4d/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6990ef617f14450bc6b34941dba4f12d5613cbf4e33805932f853fbd1cf18bfb", size = 1800491, upload-time = "2025-07-29T05:50:57.202Z" }, + { url = "https://files.pythonhosted.org/packages/de/5e/3bf5acea47a96a28c121b167f5ef659cf71208b19e52a88cdfa5c37f1fcc/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd736ed420f4db2b8148b52b46b88ed038d0354255f9a73196b7bbce3ea97545", size = 1719929, upload-time = "2025-07-29T05:50:59.192Z" }, + { url = "https://files.pythonhosted.org/packages/39/94/8ae30b806835bcd1cba799ba35347dee6961a11bd507db634516210e91d8/aiohttp-3.12.15-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c5092ce14361a73086b90c6efb3948ffa5be2f5b6fbcf52e8d8c8b8848bb97c", size = 1635733, upload-time = "2025-07-29T05:51:01.394Z" }, + { url = "https://files.pythonhosted.org/packages/7a/46/06cdef71dd03acd9da7f51ab3a9107318aee12ad38d273f654e4f981583a/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aaa2234bb60c4dbf82893e934d8ee8dea30446f0647e024074237a56a08c01bd", size = 1696790, upload-time = "2025-07-29T05:51:03.657Z" }, + { url = "https://files.pythonhosted.org/packages/02/90/6b4cfaaf92ed98d0ec4d173e78b99b4b1a7551250be8937d9d67ecb356b4/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6d86a2fbdd14192e2f234a92d3b494dd4457e683ba07e5905a0b3ee25389ac9f", size = 1718245, upload-time = "2025-07-29T05:51:05.911Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e6/2593751670fa06f080a846f37f112cbe6f873ba510d070136a6ed46117c6/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a041e7e2612041a6ddf1c6a33b883be6a421247c7afd47e885969ee4cc58bd8d", size = 1658899, upload-time = "2025-07-29T05:51:07.753Z" }, + { url = "https://files.pythonhosted.org/packages/8f/28/c15bacbdb8b8eb5bf39b10680d129ea7410b859e379b03190f02fa104ffd/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5015082477abeafad7203757ae44299a610e89ee82a1503e3d4184e6bafdd519", size = 1738459, upload-time = "2025-07-29T05:51:09.56Z" }, + { url = "https://files.pythonhosted.org/packages/00/de/c269cbc4faa01fb10f143b1670633a8ddd5b2e1ffd0548f7aa49cb5c70e2/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:56822ff5ddfd1b745534e658faba944012346184fbfe732e0d6134b744516eea", size = 1766434, upload-time = "2025-07-29T05:51:11.423Z" }, + { url = "https://files.pythonhosted.org/packages/52/b0/4ff3abd81aa7d929b27d2e1403722a65fc87b763e3a97b3a2a494bfc63bc/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b2acbbfff69019d9014508c4ba0401822e8bae5a5fdc3b6814285b71231b60f3", size = 1726045, upload-time = "2025-07-29T05:51:13.689Z" }, + { url = "https://files.pythonhosted.org/packages/71/16/949225a6a2dd6efcbd855fbd90cf476052e648fb011aa538e3b15b89a57a/aiohttp-3.12.15-cp312-cp312-win32.whl", hash = "sha256:d849b0901b50f2185874b9a232f38e26b9b3d4810095a7572eacea939132d4e1", size = 423591, upload-time = "2025-07-29T05:51:15.452Z" }, + { url = "https://files.pythonhosted.org/packages/2b/d8/fa65d2a349fe938b76d309db1a56a75c4fb8cc7b17a398b698488a939903/aiohttp-3.12.15-cp312-cp312-win_amd64.whl", hash = "sha256:b390ef5f62bb508a9d67cb3bba9b8356e23b3996da7062f1a57ce1a79d2b3d34", size = 450266, upload-time = "2025-07-29T05:51:17.239Z" }, ] [[package]] @@ -177,70 +188,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f5/10/6c25ed6de94c49f88a91fa5018cb4c0f3625f31d5be9f771ebe5cc7cd506/aiosqlite-0.21.0-py3-none-any.whl", hash = "sha256:2549cf4057f95f53dcba16f2b64e8e2791d7e1adedb13197dd8ed77bb226d7d0", size = 15792, upload-time = "2025-02-03T07:30:13.6Z" }, ] -[[package]] -name = "aiqtoolkit" -version = "1.2.0a20250707" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aioboto3" }, - { name = "click" }, - { name = "colorama" }, - { name = "expandvars" }, - { name = "fastapi" }, - { name = "httpx" }, - { name = "jinja2" }, - { name = "jsonpath-ng" }, - { name = "mcp" }, - { name = "networkx" }, - { name = "numpy" }, - { name = "openinference-semantic-conventions" }, - { name = "openpyxl" }, - { name = "pkginfo" }, - { name = "platformdirs" }, - { name = "pydantic" }, - { name = "pymilvus" }, - { name = "pyyaml" }, - { name = "ragas" }, - { name = "rich" }, - { name = "uvicorn", extra = ["standard"] }, - { name = "wikipedia" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/7d/25/e3621910b51233a5e64a7361f620105791092af12e462bde2a37b2dff5f6/aiqtoolkit-1.2.0a20250707-py3-none-any.whl", hash = "sha256:3dfadc2d1eab45511f01c2b7b919ebaf132b1cf4f4cc14a20fc193204de9a892", size = 511867, upload-time = "2025-07-07T09:51:26.332Z" }, -] - -[package.optional-dependencies] -langchain = [ - { name = "aiqtoolkit-langchain" }, -] -profiling = [ - { name = "matplotlib" }, - { name = "prefixspan" }, - { name = "scikit-learn" }, -] -telemetry = [ - { name = "arize-phoenix" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-sdk" }, -] - -[[package]] -name = "aiqtoolkit-langchain" -version = "1.2.0a20250707" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiqtoolkit" }, - { name = "langchain-aws" }, - { name = "langchain-core" }, - { name = "langchain-milvus" }, - { name = "langchain-nvidia-ai-endpoints" }, - { name = "langchain-openai" }, - { name = "langgraph" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/65/31/5754bd1db8be023b9cfd0a8bd8afb3846e42145445ab2ace7410b0a9188d/aiqtoolkit_langchain-1.2.0a20250707-py3-none-any.whl", hash = "sha256:d8ac542d4e82d971eef991444284d172e36f44f5a83a20f5dac0588f4931566c", size = 13095, upload-time = "2025-07-07T09:52:01.593Z" }, -] - [[package]] name = "alembic" version = "1.16.4" @@ -266,16 +213,16 @@ wheels = [ [[package]] name = "anyio" -version = "4.9.0" +version = "4.10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "idna" }, { name = "sniffio" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/95/7d/4c1bd541d4dffa1b52bd83fb8527089e097a106fc90b467a7313b105f840/anyio-4.9.0.tar.gz", hash = "sha256:673c0c244e15788651a4ff38710fea9675823028a6f08a5eda409e0c9840a028", size = 190949, upload-time = "2025-03-17T00:02:54.77Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/b4/636b3b65173d3ce9a38ef5f0522789614e590dab6a8d505340a4efe4c567/anyio-4.10.0.tar.gz", hash = "sha256:3f3fae35c96039744587aa5b8371e7e8e603c0702999535961dd336026973ba6", size = 213252, upload-time = "2025-08-04T08:54:26.451Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a1/ee/48ca1a7c89ffec8b6a0c5d02b89c305671d5ffd8d3c94acf8b8c408575bb/anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c", size = 100916, upload-time = "2025-03-17T00:02:52.713Z" }, + { url = "https://files.pythonhosted.org/packages/6f/12/e5e0282d673bb9746bacfb6e2dba8719989d3660cdb2ea79aee9a9651afb/anyio-4.10.0-py3-none-any.whl", hash = "sha256:60e474ac86736bbfd6f210f7a61218939c318f43f9972497381f1c5e930ed3d1", size = 107213, upload-time = "2025-08-04T08:54:24.882Z" }, ] [[package]] @@ -287,6 +234,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/00/2344469e2084fb287c2e0b57b72910309874c3245463acd6cf5e3db69324/appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128", size = 9566, upload-time = "2020-05-11T07:59:49.499Z" }, ] +[[package]] +name = "appnope" +version = "0.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/35/5d/752690df9ef5b76e169e68d6a129fa6d08a7100ca7f754c89495db3c6019/appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee", size = 4170, upload-time = "2024-02-06T09:43:11.258Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/29/5ecc3a15d5a33e31b26c11426c45c501e439cb865d0bff96315d86443b78/appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c", size = 4321, upload-time = "2024-02-06T09:43:09.663Z" }, +] + [[package]] name = "arize-phoenix" version = "6.2.0" @@ -336,21 +292,22 @@ wheels = [ [[package]] name = "arize-phoenix-evals" -version = "0.22.0" +version = "0.27.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pandas" }, + { name = "pystache" }, { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e0/bb/2bac66f6471f87a765cacda532e5200a94adfc3dbce7ec8c0a6aafdf1f18/arize_phoenix_evals-0.22.0.tar.gz", hash = "sha256:bc84fafc53ffc01f7832c3313ee22418d397543d0b20520b29f175d45968ef23", size = 50110, upload-time = "2025-07-02T18:01:53.93Z" } +sdist = { url = "https://files.pythonhosted.org/packages/30/8f/3096dbb74ea796e9d868aa11269b688162f445afb56d670c13bd9c7d4e6d/arize_phoenix_evals-0.27.0.tar.gz", hash = "sha256:e2838e45b620a316293df46502d9d7b7b7ab4353939e48baa18cfe678be7962a", size = 77293, upload-time = "2025-08-13T15:20:35.738Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/ca/838c91b093bb4f56458eb79c3482d104a5554575bef8a70b97aa11cf0c36/arize_phoenix_evals-0.22.0-py3-none-any.whl", hash = "sha256:30bde46b3c79b15cc8bce42dc2fb213242a30f820f9d7e4717c3192ebb3158a8", size = 64341, upload-time = "2025-07-02T18:01:52.901Z" }, + { url = "https://files.pythonhosted.org/packages/fd/b5/4549cb087cfa0865f30a5680a6a12e4f509f24e1ad739a00a1a47c1783bf/arize_phoenix_evals-0.27.0-py3-none-any.whl", hash = "sha256:b5110a9688c8c9993c8427ecfadf1dead06aeb03b56060a19e42e52c7f5212c5", size = 105443, upload-time = "2025-08-13T15:20:34.269Z" }, ] [[package]] name = "arize-phoenix-otel" -version = "0.12.1" +version = "0.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "openinference-instrumentation" }, @@ -362,9 +319,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a4/b2/e990deaad2eba163c2d6f6af967495263efd4cefdf8545f17ddf3846e221/arize_phoenix_otel-0.12.1.tar.gz", hash = "sha256:0150cdc586abbd3cca49168352f94501d59c2690cab5d283a5e0689bb3dc1234", size = 16402, upload-time = "2025-06-24T15:58:34.914Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/13/bbfaaeac32a304505b7d229b3ef7326f631a1599bce58ce7bf80db9431f0/arize_phoenix_otel-0.13.0.tar.gz", hash = "sha256:1c2061f146528bf39ec7185632a904cf0642b63949b4f6e74ca3833f2ade4f78", size = 18630, upload-time = "2025-08-14T05:07:28.576Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/46/27/be2ead7273ecf3be0c5b3ebf9e4bccedcd49ab45a7909e82588f56189cd2/arize_phoenix_otel-0.12.1-py3-none-any.whl", hash = "sha256:85167dd061d7d4e14c98edd5733afe151da8027d69f2ecfe29a6aa2484906fa3", size = 13871, upload-time = "2025-06-24T15:58:33.702Z" }, + { url = "https://files.pythonhosted.org/packages/48/72/f17886ca0692854cfd721ea2828dd303bf8361d5b8ac8092f2af7ef94a27/arize_phoenix_otel-0.13.0-py3-none-any.whl", hash = "sha256:7a4d86a9e8bb16f55d3bb3f240891ac663c72c7527bd7094b3dfea58cd1daa7b", size = 16303, upload-time = "2025-08-14T05:07:27.443Z" }, ] [[package]] @@ -382,6 +339,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/0f/3b8fdc946b4d9cc8cc1e8af42c4e409468c84441b933d037e101b3d72d86/astroid-3.3.11-py3-none-any.whl", hash = "sha256:54c760ae8322ece1abd213057c4b5bba7c49818853fc901ef09719a60dbf9dec", size = 275612, upload-time = "2025-07-13T18:04:21.07Z" }, ] +[[package]] +name = "asttokens" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4a/e7/82da0a03e7ba5141f05cce0d302e6eed121ae055e0456ca228bf693984bc/asttokens-3.0.0.tar.gz", hash = "sha256:0dcd8baa8d62b0c1d118b399b2ddba3c4aff271d0d7a9e0d4c1681c79035bbc7", size = 61978, upload-time = "2024-11-30T04:30:14.439Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/8a/c46dcc25341b5bce5472c718902eb3d38600a903b14fa6aeecef3f21a46f/asttokens-3.0.0-py3-none-any.whl", hash = "sha256:e3078351a059199dd5138cb1c706e6430c05eff2ff136af5eb4790f9d28932e2", size = 26918, upload-time = "2024-11-30T04:30:10.946Z" }, +] + [[package]] name = "attrs" version = "25.3.0" @@ -393,14 +359,53 @@ wheels = [ [[package]] name = "authlib" -version = "1.6.0" +version = "1.3.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a2/9d/b1e08d36899c12c8b894a44a5583ee157789f26fc4b176f8e4b6217b56e1/authlib-1.6.0.tar.gz", hash = "sha256:4367d32031b7af175ad3a323d571dc7257b7099d55978087ceae4a0d88cd3210", size = 158371, upload-time = "2025-05-23T00:21:45.011Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/75/47dbab150ef6f9298e227a40c93c7fed5f3ffb67c9fb62cd49f66285e46e/authlib-1.3.2.tar.gz", hash = "sha256:4b16130117f9eb82aa6eec97f6dd4673c3f960ac0283ccdae2897ee4bc030ba2", size = 147313, upload-time = "2024-08-26T07:10:04.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/4c/9aa0416a403d5cc80292cb030bcd2c918cce2755e314d8c1aa18656e1e12/Authlib-1.3.2-py2.py3-none-any.whl", hash = "sha256:ede026a95e9f5cdc2d4364a52103f5405e75aa156357e831ef2bfd0bc5094dfc", size = 225111, upload-time = "2024-08-26T07:10:02.811Z" }, +] + +[[package]] +name = "azure-core" +version = "1.35.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, + { name = "six" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ce/89/f53968635b1b2e53e4aad2dd641488929fef4ca9dfb0b97927fa7697ddf3/azure_core-1.35.0.tar.gz", hash = "sha256:c0be528489485e9ede59b6971eb63c1eaacf83ef53001bfe3904e475e972be5c", size = 339689, upload-time = "2025-07-03T00:55:23.496Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/78/bf94897361fdd650850f0f2e405b2293e2f12808239046232bdedf554301/azure_core-1.35.0-py3-none-any.whl", hash = "sha256:8db78c72868a58f3de8991eb4d22c4d368fae226dac1002998d6c50437e7dad1", size = 210708, upload-time = "2025-07-03T00:55:25.238Z" }, +] + +[[package]] +name = "backoff" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/d7/5bbeb12c44d7c4f2fb5b56abce497eb5ed9f34d85701de869acedd602619/backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", size = 17001, upload-time = "2022-10-05T19:19:32.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148, upload-time = "2022-10-05T19:19:30.546Z" }, +] + +[[package]] +name = "banks" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecated" }, + { name = "griffe" }, + { name = "jinja2" }, + { name = "platformdirs" }, + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/f8/25ef24814f77f3fd7f0fd3bd1ef3749e38a9dbd23502fbb53034de49900c/banks-2.2.0.tar.gz", hash = "sha256:d1446280ce6e00301e3e952dd754fd8cee23ff277d29ed160994a84d0d7ffe62", size = 179052, upload-time = "2025-07-18T16:28:26.892Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/84/29/587c189bbab1ccc8c86a03a5d0e13873df916380ef1be461ebe6acebf48d/authlib-1.6.0-py2.py3-none-any.whl", hash = "sha256:91685589498f79e8655e8a8947431ad6288831d643f11c55c2143ffcc738048d", size = 239981, upload-time = "2025-05-23T00:21:43.075Z" }, + { url = "https://files.pythonhosted.org/packages/b4/d6/f9168956276934162ec8d48232f9920f2985ee45aa7602e3c6b4bc203613/banks-2.2.0-py3-none-any.whl", hash = "sha256:963cd5c85a587b122abde4f4064078def35c50c688c1b9d36f43c92503854e7d", size = 29244, upload-time = "2025-07-18T16:28:27.835Z" }, ] [[package]] @@ -416,32 +421,61 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/50/cd/30110dc0ffcf3b131156077b90e9f60ed75711223f306da4db08eff8403b/beautifulsoup4-4.13.4-py3-none-any.whl", hash = "sha256:9bbbb14bfde9d79f38b8cd5f8c7c85f4b8f2523190ebed90e950a8dea4cb1c4b", size = 187285, upload-time = "2025-04-15T17:05:12.221Z" }, ] +[[package]] +name = "blis" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/aa/0743c994884de83472c854bb534c9edab8d711e1880d4fa194e6d876bb60/blis-1.2.1.tar.gz", hash = "sha256:1066beedbedc2143c22bd28742658de05694afebacde8d8c2d14dd4b5a96765a", size = 2510297, upload-time = "2025-04-01T12:01:56.849Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/57/ae6596b1e27859886e0b81fb99497bcfff139895585a9e2284681c8a8846/blis-1.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:778c4f72b71f97187e3304acfbd30eab98c9ba1a5b03b65128bc3875400ae604", size = 6976808, upload-time = "2025-04-01T12:01:21.175Z" }, + { url = "https://files.pythonhosted.org/packages/ce/35/6225e6ad2bccf23ac124448d59112c098d63a8917462e9f73967bc217168/blis-1.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5c5f2ffb0ae9c1f5aaa95b9681bcdd9a777d007c501fa220796329b939ca2790", size = 1281913, upload-time = "2025-04-01T12:01:23.202Z" }, + { url = "https://files.pythonhosted.org/packages/7a/84/c6a6d1c0a8a00799d2ec5db05d676bd9a9b0472cac4d3eff2e2fd1953521/blis-1.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db4dc5d2d57106bb411633603a5c7d178a0845267c3efc7e5ea4fa7a44772976", size = 3104139, upload-time = "2025-04-01T12:01:24.781Z" }, + { url = "https://files.pythonhosted.org/packages/a5/6c/c5fab7ed1fe6e8bdcda732017400d1adc53db5b6dd2c2a6046acab91f4fa/blis-1.2.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c621271c2843101927407e052b35a67f853da59d5c74e9e070e982c7f82e2e04", size = 3304143, upload-time = "2025-04-01T12:01:27.363Z" }, + { url = "https://files.pythonhosted.org/packages/22/d1/85f03269886253758546fcfdbeddee7e717d843ea134596b60db9c2648c4/blis-1.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43f65f882250b817566d7543abd1f6da297f1662e5dd9936e14c04b88285a497", size = 11660080, upload-time = "2025-04-01T12:01:29.478Z" }, + { url = "https://files.pythonhosted.org/packages/78/c8/c81ed3036e8ce0d6ce0d19a032c7f3d69247f221c5357e18548dea9380d3/blis-1.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:78a0613d559ccc426c101c67e8f84e1f93491e29d722c370872c538ee652bd07", size = 3133133, upload-time = "2025-04-01T12:01:31.537Z" }, + { url = "https://files.pythonhosted.org/packages/b8/42/7c296e04b979204777ecae2fe9287ac7b0255d8c4c2111d2a735c439b9d7/blis-1.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:2f5e32e5e5635fc7087b724b53120dbcd86201f56c0405882ce254bc0e493392", size = 4360695, upload-time = "2025-04-01T12:01:33.449Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ab/aa5c8dfd0068d2cc976830797dd092779259860f964286db05739154e3a7/blis-1.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d339c97cc83f53e39c1013d0dcd7d5278c853dc102d931132eeb05b226e28429", size = 14828081, upload-time = "2025-04-01T12:01:35.129Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c0/047fef3ac4a531903c52ba7c108fd608556627723bfef7554f040b10e556/blis-1.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:8d284323cc994e9b818c32046f1aa3e57bcc41c74e02daebdf0d3bc3e14355cb", size = 6232639, upload-time = "2025-04-01T12:01:37.268Z" }, + { url = "https://files.pythonhosted.org/packages/2f/f1/2aecd2447de0eb5deea3a13e471ab43e42e8561afe56a13d830f95c58909/blis-1.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:1cd35e94a1a97b37b31b11f097f998a3a0e75ac06d57e6edf7d9597200f55756", size = 6989811, upload-time = "2025-04-01T12:01:39.013Z" }, + { url = "https://files.pythonhosted.org/packages/cf/39/4c097508f6b9ef7df27dd5ada0a175e8169f58cbe33d40a303a844abdaea/blis-1.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7b6394d27f2259c580df8d13ebe9c0a188a6ace0a689e93d6e49cb15018d4d9c", size = 1282669, upload-time = "2025-04-01T12:01:41.418Z" }, + { url = "https://files.pythonhosted.org/packages/7a/8e/b8a5eafa9824fcc7f3339a283e910f7af110d749fd09f52e83f432124543/blis-1.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9c127159415dc772f345abc3575e1e2d02bb1ae7cb7f532267d67705be04c66", size = 3063750, upload-time = "2025-04-01T12:01:43.277Z" }, + { url = "https://files.pythonhosted.org/packages/f7/7a/f88e935f2cd3ad52ef363beeddf9a537d5038e519aa7b09dc18c762fbb66/blis-1.2.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5f9fa589aa72448009fd5001afb05e69f3bc953fe778b44580fd7d79ee8201a1", size = 3260903, upload-time = "2025-04-01T12:01:44.815Z" }, + { url = "https://files.pythonhosted.org/packages/4a/26/283f1392974e5c597228f8485f45f89de33f2c85becebc25e846d0485e44/blis-1.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1aa6150259caf4fa0b527bfc8c1e858542f9ca88a386aa90b93e1ca4c2add6df", size = 11616588, upload-time = "2025-04-01T12:01:46.356Z" }, + { url = "https://files.pythonhosted.org/packages/fa/86/57047b688e42c92e35d0581ef9db15ee3bdf14deff4d9a2481ce331f2dae/blis-1.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3ba67c09883cae52da3d9e9d3f4305464efedd336032c4d5c6c429b27b16f4c1", size = 3072892, upload-time = "2025-04-01T12:01:48.314Z" }, + { url = "https://files.pythonhosted.org/packages/c7/db/85b6f5fa2a2515470cc5a2cbeaedd25aa465fa572801f18d14c24c9e5102/blis-1.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:7d9c5fca21b01c4b2f3cb95b71ce7ef95e58b3b62f0d79d1f699178c72c1e03e", size = 4310005, upload-time = "2025-04-01T12:01:49.815Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/6e610e950476ebc9868a0207a827d67433ef65e2b14b837d317e60248e5a/blis-1.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6952a4a1f15e0d1f73cc1206bd71368b32551f2e94852dae288b50c4ea0daf31", size = 14790198, upload-time = "2025-04-01T12:01:52.601Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0e/353e29e8dd3d31bba25a3eabbbfb798d82bd19ca2d24fd00583b6d3992f3/blis-1.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:bd0360427b1669684cd35a8355be126d7a33992ccac6dcb1fbef5e100f4e3026", size = 6260640, upload-time = "2025-04-01T12:01:54.849Z" }, +] + [[package]] name = "boto3" -version = "1.38.27" +version = "1.39.11" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "jmespath" }, { name = "s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e7/96/fc74d8521d2369dd8c412438401ff12e1350a1cd3eab5c758ed3dd5e5f82/boto3-1.38.27.tar.gz", hash = "sha256:94bd7fdd92d5701b362d4df100d21e28f8307a67ff56b6a8b0398119cf22f859", size = 111875, upload-time = "2025-05-30T19:32:41.352Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/2e/ed75ea3ee0fd1afacc3379bc2b7457c67a6b0f0e554e1f7ccbdbaed2351b/boto3-1.39.11.tar.gz", hash = "sha256:3027edf20642fe1d5f9dc50a420d0fe2733073ed6a9f0f047b60fe08c3682132", size = 111869, upload-time = "2025-07-22T19:26:50.867Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/43/8b/b2361188bd1e293eede1bc165e2461d390394f71ec0c8c21211c8dabf62c/boto3-1.38.27-py3-none-any.whl", hash = "sha256:95f5fe688795303a8a15e8b7e7f255cadab35eae459d00cc281a4fd77252ea80", size = 139938, upload-time = "2025-05-30T19:32:38.006Z" }, + { url = "https://files.pythonhosted.org/packages/72/66/88566a6484e746c0b075f7c9bb248e8548eda0a486de4460d150a41e2d57/boto3-1.39.11-py3-none-any.whl", hash = "sha256:af8f1dad35eceff7658fab43b39b0f55892b6e3dd12308733521cc24dd2c9a02", size = 139900, upload-time = "2025-07-22T19:26:48.706Z" }, ] [[package]] name = "botocore" -version = "1.38.27" +version = "1.39.11" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/36/5e/67899214ad57f7f26af5bd776ac5eb583dc4ecf5c1e52e2cbfdc200e487a/botocore-1.38.27.tar.gz", hash = "sha256:9788f7efe974328a38cbade64cc0b1e67d27944b899f88cb786ae362973133b6", size = 13919963, upload-time = "2025-05-30T19:32:29.657Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/d0/9d64261186cff650fe63168441edb4f4cd33f085a74c0c54455630a71f91/botocore-1.39.11.tar.gz", hash = "sha256:953b12909d6799350e346ab038e55b6efe622c616f80aef74d7a6683ffdd972c", size = 14217749, upload-time = "2025-07-22T19:26:40.723Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/83/a753562020b69fa90cebc39e8af2c753b24dcdc74bee8355ee3f6cefdf34/botocore-1.38.27-py3-none-any.whl", hash = "sha256:a785d5e9a5eda88ad6ab9ed8b87d1f2ac409d0226bba6ff801c55359e94d91a8", size = 13580545, upload-time = "2025-05-30T19:32:26.712Z" }, + { url = "https://files.pythonhosted.org/packages/1c/2c/8a0b02d60a1dbbae7faa5af30484b016aa3023f9833dfc0d19b0b770dd6a/botocore-1.39.11-py3-none-any.whl", hash = "sha256:1545352931a8a186f3e977b1e1a4542d7d434796e274c3c62efd0210b5ea76dc", size = 13876276, upload-time = "2025-07-22T19:26:35.164Z" }, ] [[package]] @@ -488,89 +522,114 @@ wheels = [ [[package]] name = "cachetools" -version = "6.1.0" +version = "5.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/81/3747dad6b14fa2cf53fcf10548cf5aea6913e96fab41a3c198676f8948a5/cachetools-5.5.2.tar.gz", hash = "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4", size = 28380, upload-time = "2025-02-20T21:01:19.524Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/76/20fa66124dbe6be5cafeb312ece67de6b61dd91a0247d1ea13db4ebb33c2/cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a", size = 10080, upload-time = "2025-02-20T21:01:16.647Z" }, +] + +[[package]] +name = "catalogue" +version = "2.0.10" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8a/89/817ad5d0411f136c484d535952aef74af9b25e0d99e90cdffbe121e6d628/cachetools-6.1.0.tar.gz", hash = "sha256:b4c4f404392848db3ce7aac34950d17be4d864da4b8b66911008e430bc544587", size = 30714, upload-time = "2025-06-16T18:51:03.07Z" } +sdist = { url = "https://files.pythonhosted.org/packages/38/b4/244d58127e1cdf04cf2dc7d9566f0d24ef01d5ce21811bab088ecc62b5ea/catalogue-2.0.10.tar.gz", hash = "sha256:4f56daa940913d3f09d589c191c74e5a6d51762b3a9e37dd53b7437afd6cda15", size = 19561, upload-time = "2023-09-25T06:29:24.962Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/00/f0/2ef431fe4141f5e334759d73e81120492b23b2824336883a91ac04ba710b/cachetools-6.1.0-py3-none-any.whl", hash = "sha256:1c7bb3cf9193deaf3508b7c5f2a79986c13ea38965c5adcff1f84519cf39163e", size = 11189, upload-time = "2025-06-16T18:51:01.514Z" }, + { url = "https://files.pythonhosted.org/packages/9e/96/d32b941a501ab566a16358d68b6eb4e4acc373fab3c3c4d7d9e649f7b4bb/catalogue-2.0.10-py3-none-any.whl", hash = "sha256:58c2de0020aa90f4a2da7dfad161bf7b3b054c86a5f09fcedc0b2b740c109a9f", size = 17325, upload-time = "2023-09-25T06:29:23.337Z" }, ] [[package]] name = "certifi" -version = "2025.7.14" +version = "2025.8.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b3/76/52c535bcebe74590f296d6c77c86dabf761c41980e1347a2422e4aa2ae41/certifi-2025.7.14.tar.gz", hash = "sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995", size = 163981, upload-time = "2025-07-14T03:29:28.449Z" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386, upload-time = "2025-08-03T03:07:47.08Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/52/34c6cf5bb9285074dc3531c437b3919e825d976fde097a7a73f79e726d03/certifi-2025.7.14-py3-none-any.whl", hash = "sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2", size = 162722, upload-time = "2025-07-14T03:29:26.863Z" }, + { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" }, ] [[package]] name = "cffi" -version = "1.17.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pycparser" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621, upload-time = "2024-09-04T20:45:21.852Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/f4/927e3a8899e52a27fa57a48607ff7dc91a9ebe97399b357b85a0c7892e00/cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401", size = 182264, upload-time = "2024-09-04T20:43:51.124Z" }, - { url = "https://files.pythonhosted.org/packages/6c/f5/6c3a8efe5f503175aaddcbea6ad0d2c96dad6f5abb205750d1b3df44ef29/cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf", size = 178651, upload-time = "2024-09-04T20:43:52.872Z" }, - { url = "https://files.pythonhosted.org/packages/94/dd/a3f0118e688d1b1a57553da23b16bdade96d2f9bcda4d32e7d2838047ff7/cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4", size = 445259, upload-time = "2024-09-04T20:43:56.123Z" }, - { url = "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", size = 469200, upload-time = "2024-09-04T20:43:57.891Z" }, - { url = "https://files.pythonhosted.org/packages/1c/a0/a4fa9f4f781bda074c3ddd57a572b060fa0df7655d2a4247bbe277200146/cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1", size = 477235, upload-time = "2024-09-04T20:44:00.18Z" }, - { url = "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", size = 459721, upload-time = "2024-09-04T20:44:01.585Z" }, - { url = "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", size = 467242, upload-time = "2024-09-04T20:44:03.467Z" }, - { url = "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", size = 477999, upload-time = "2024-09-04T20:44:05.023Z" }, - { url = "https://files.pythonhosted.org/packages/44/74/f2a2460684a1a2d00ca799ad880d54652841a780c4c97b87754f660c7603/cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f", size = 454242, upload-time = "2024-09-04T20:44:06.444Z" }, - { url = "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", size = 478604, upload-time = "2024-09-04T20:44:08.206Z" }, - { url = "https://files.pythonhosted.org/packages/34/33/e1b8a1ba29025adbdcda5fb3a36f94c03d771c1b7b12f726ff7fef2ebe36/cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655", size = 171727, upload-time = "2024-09-04T20:44:09.481Z" }, - { url = "https://files.pythonhosted.org/packages/3d/97/50228be003bb2802627d28ec0627837ac0bf35c90cf769812056f235b2d1/cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0", size = 181400, upload-time = "2024-09-04T20:44:10.873Z" }, - { url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178, upload-time = "2024-09-04T20:44:12.232Z" }, - { url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840, upload-time = "2024-09-04T20:44:13.739Z" }, - { url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803, upload-time = "2024-09-04T20:44:15.231Z" }, - { url = "https://files.pythonhosted.org/packages/1a/df/f8d151540d8c200eb1c6fba8cd0dfd40904f1b0682ea705c36e6c2e97ab3/cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", size = 478850, upload-time = "2024-09-04T20:44:17.188Z" }, - { url = "https://files.pythonhosted.org/packages/28/c0/b31116332a547fd2677ae5b78a2ef662dfc8023d67f41b2a83f7c2aa78b1/cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", size = 485729, upload-time = "2024-09-04T20:44:18.688Z" }, - { url = "https://files.pythonhosted.org/packages/91/2b/9a1ddfa5c7f13cab007a2c9cc295b70fbbda7cb10a286aa6810338e60ea1/cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", size = 471256, upload-time = "2024-09-04T20:44:20.248Z" }, - { url = "https://files.pythonhosted.org/packages/b2/d5/da47df7004cb17e4955df6a43d14b3b4ae77737dff8bf7f8f333196717bf/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", size = 479424, upload-time = "2024-09-04T20:44:21.673Z" }, - { url = "https://files.pythonhosted.org/packages/0b/ac/2a28bcf513e93a219c8a4e8e125534f4f6db03e3179ba1c45e949b76212c/cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", size = 484568, upload-time = "2024-09-04T20:44:23.245Z" }, - { url = "https://files.pythonhosted.org/packages/d4/38/ca8a4f639065f14ae0f1d9751e70447a261f1a30fa7547a828ae08142465/cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", size = 488736, upload-time = "2024-09-04T20:44:24.757Z" }, - { url = "https://files.pythonhosted.org/packages/86/c5/28b2d6f799ec0bdecf44dced2ec5ed43e0eb63097b0f58c293583b406582/cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", size = 172448, upload-time = "2024-09-04T20:44:26.208Z" }, - { url = "https://files.pythonhosted.org/packages/50/b9/db34c4755a7bd1cb2d1603ac3863f22bcecbd1ba29e5ee841a4bc510b294/cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", size = 181976, upload-time = "2024-09-04T20:44:27.578Z" }, +version = "2.0.0b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/75/84/7930c3586ca7c66a63b2d7a30d9df649ce8c3660f8da241b0661bba4e566/cffi-2.0.0b1.tar.gz", hash = "sha256:4440de58d19c0bebe6a2f3b721253d67b27aabb34e00ab35756d8699876191ea", size = 521625, upload-time = "2025-07-29T01:11:50.959Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/90/14deaf13603dfff56bb872a4d53e1043486178ae7a2ce8cc17ea5677d97e/cffi-2.0.0b1-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:5f373f9bdc3569acd8aaebb6b521080eeb5a298533a58715537caf74e9e27f6b", size = 184383, upload-time = "2025-07-29T01:10:17.675Z" }, + { url = "https://files.pythonhosted.org/packages/f7/36/0a125a1ab354a95aae2165ce4c2b8fcd057706a85380670e3991052dcfcd/cffi-2.0.0b1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a898f76bac81f9a371df6c8664228a85cdea6b283a721f2493f0df6f80afd208", size = 180599, upload-time = "2025-07-29T01:10:18.753Z" }, + { url = "https://files.pythonhosted.org/packages/bc/cb/27237bcd6c4e883104db737929f02838a7405caed422aeeb76ee5ffa14d9/cffi-2.0.0b1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:314afab228f7b45de7bae55059b4e706296e7d3984d53e643cc0389757216221", size = 203212, upload-time = "2025-07-29T01:10:20.057Z" }, + { url = "https://files.pythonhosted.org/packages/12/94/bbeddca63090c5335ad597310bd6f2011f1c8733bc71e88f53c38ac4ff4c/cffi-2.0.0b1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6de033c73dc89f80139c5a7d135fbd6c1d7b28ebb0d2df98cd1f4ef76991b15c", size = 202714, upload-time = "2025-07-29T01:10:21.401Z" }, + { url = "https://files.pythonhosted.org/packages/f4/9b/b7587a1f3f7f52795a7d125d6c6b844f7a8355cbb54ae8fdef2a03488914/cffi-2.0.0b1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ffbbeedd6bac26c0373b71831d3c73181a1c100dc6fc7aadbfcca54cace417db", size = 217093, upload-time = "2025-07-29T01:10:22.481Z" }, + { url = "https://files.pythonhosted.org/packages/a3/b2/af4e0ed2c2aded25ed54107f96d424407839bdfa7e90858f8e0f6fed6ee9/cffi-2.0.0b1-cp311-cp311-manylinux_2_27_i686.manylinux_2_28_i686.whl", hash = "sha256:c5713cac21b2351a53958c765d8e9eda45184bb757c3ccab139608e708788796", size = 209019, upload-time = "2025-07-29T01:10:23.584Z" }, + { url = "https://files.pythonhosted.org/packages/7b/6e/899c5473c3d7cc89815db894abcd81cd976a1f314c142e708aef3c0982a3/cffi-2.0.0b1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:71ab35c6cc375da1e2c06af65bf0b5049199ad9b264f9ed7c90c0fe9450900e3", size = 215662, upload-time = "2025-07-29T01:10:24.997Z" }, + { url = "https://files.pythonhosted.org/packages/1c/8e/953a07806f307bf1089239858013cc81c6d5cc8ca23593704b0530429302/cffi-2.0.0b1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:53c780c2ec8ce0e5db9b74e9b0b55ff5d5f70071202740cef073a2771fa1d2ce", size = 219015, upload-time = "2025-07-29T01:10:27.077Z" }, + { url = "https://files.pythonhosted.org/packages/ea/0a/ffd99099d96a911236decff459cb330a1c046483008456b23554f62c81c6/cffi-2.0.0b1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:be957dd266facf8e4925643073159b05021a990b46620b06ca27eaf9d900dbc2", size = 212021, upload-time = "2025-07-29T01:10:28.527Z" }, + { url = "https://files.pythonhosted.org/packages/2f/00/c68c1a1665a28dfb8c848668f128d0f1919dc8e843f2e20ce90bce7b60d8/cffi-2.0.0b1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:16dc303af3630f54186b86aadf1121badf3cba6de17dfeacb84c5091e059a690", size = 217124, upload-time = "2025-07-29T01:10:29.877Z" }, + { url = "https://files.pythonhosted.org/packages/de/a7/194d80668bebc5a6a8d95ec5f3a1f186e8d87c864882c96a9ec2ecbd06a8/cffi-2.0.0b1-cp311-cp311-win32.whl", hash = "sha256:504d264944d0934d7b02164af5c62b175255ef0d39c5142d95968b710c58a8f6", size = 172111, upload-time = "2025-07-29T01:10:30.973Z" }, + { url = "https://files.pythonhosted.org/packages/e2/b6/0002211aab83b6bfbdba09dc8cd354e44c49216e6207999b9f0d1d0053cb/cffi-2.0.0b1-cp311-cp311-win_amd64.whl", hash = "sha256:e2920fa42cf0616c21ea6d3948ad207cf0e420d2d2ef449d86ccad6ef9c13393", size = 182858, upload-time = "2025-07-29T01:10:32.021Z" }, + { url = "https://files.pythonhosted.org/packages/52/9e/c6773b5b91b20c5642166c57503a9c67c6948ae4009aa4d2ce233a6b570f/cffi-2.0.0b1-cp311-cp311-win_arm64.whl", hash = "sha256:142c9c0c75fbc95ce23836e538681bd89e483de37b7cdf251dbdf0975995f8ac", size = 177421, upload-time = "2025-07-29T01:10:33.191Z" }, + { url = "https://files.pythonhosted.org/packages/50/20/432dc366952574ea190bce0a2970f92e676e972c78ef501d58406b459883/cffi-2.0.0b1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9d04b5fc06ba0ce45d7e51dfd8a14dc20708ef301fcf5a215c507f4e084b00c8", size = 185303, upload-time = "2025-07-29T01:10:34.291Z" }, + { url = "https://files.pythonhosted.org/packages/54/2d/e89016a2019212d54be2523756faa5b2c3ab8cb6f520a82e0d6bcacd527d/cffi-2.0.0b1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7b17e92900eb61bce62ea07ea8dd0dc33aa476ee8f977918050e52f90f5b645c", size = 181101, upload-time = "2025-07-29T01:10:35.641Z" }, + { url = "https://files.pythonhosted.org/packages/89/4f/6978a38ee0d8976f3087c09e779f9306ed51b9fb68ce5e3606244f6e2469/cffi-2.0.0b1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2155d2a0819c3fdcaa37832fb69e698d455627c23f83bc9c7adbef699fe4be19", size = 208122, upload-time = "2025-07-29T01:10:36.757Z" }, + { url = "https://files.pythonhosted.org/packages/20/2f/568d19b010aa304f6f55aaf160834e0db9677943b0c268462876c4e1c0ef/cffi-2.0.0b1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4210ddc2b41c20739c64dede1304fb81415220ea671885623063fab44066e376", size = 206747, upload-time = "2025-07-29T01:10:37.837Z" }, + { url = "https://files.pythonhosted.org/packages/bf/7b/171907beef5622bc6164ae9db94eaaa8e56bfb986f375742a9669ecc18f7/cffi-2.0.0b1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:31b8e3204cdef043e59a296383e6a43461d17c5c3d73fa9cebf4716a561291b0", size = 220804, upload-time = "2025-07-29T01:10:39.299Z" }, + { url = "https://files.pythonhosted.org/packages/49/2a/539d6021b1570308159745e775d0bd4164e43957e515bffd33cb6e57cf06/cffi-2.0.0b1-cp312-cp312-manylinux_2_27_i686.manylinux_2_28_i686.whl", hash = "sha256:cbde39be02aa7d8fbcd6bf1a9241cb1d84f2e2f0614970c51a707a9a176b85c6", size = 211912, upload-time = "2025-07-29T01:10:40.767Z" }, + { url = "https://files.pythonhosted.org/packages/87/a9/2cddc8eeabd7b32d494de5bb9db95e3816b47ad00e05269b33e2bb8be9f3/cffi-2.0.0b1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ea57043b545f346b081877737cb0320960012107d0250fa5183a4306f9365d6", size = 219528, upload-time = "2025-07-29T01:10:42.419Z" }, + { url = "https://files.pythonhosted.org/packages/a8/18/49ff9cbe89eae3fff54a7af79474dd897bac44325073a6a7dc9b7ae4b64e/cffi-2.0.0b1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d31ba9f54739dcf98edb87e4881e326fad79e4866137c24afb0da531c1a965ca", size = 223011, upload-time = "2025-07-29T01:10:43.906Z" }, + { url = "https://files.pythonhosted.org/packages/a1/1e/4f10dd0fd9cb8d921620663beb497af0a6175c96cecd87e5baf613d0c947/cffi-2.0.0b1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:27309de8cebf48e056550db6607e2fb2c50109b54fc72c02b3b34811233483be", size = 221408, upload-time = "2025-07-29T01:10:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/00/82/cbbb23951d9890475f151c1137d067a712e7f1e59509def619c5d9a645aa/cffi-2.0.0b1-cp312-cp312-win32.whl", hash = "sha256:f4b5acb4cddcaf0ebb82a226f9fa1d5063505e0c206031ee1f4d173750b592fd", size = 172972, upload-time = "2025-07-29T01:10:46.458Z" }, + { url = "https://files.pythonhosted.org/packages/ea/6b/e52b88ee438acd26fd84963f357a90ce8f4494cc7d94cbde1b26e199bd22/cffi-2.0.0b1-cp312-cp312-win_amd64.whl", hash = "sha256:cf1b2510f1a91c4d7e8f83df6a13404332421e6e4a067059174d455653ae5314", size = 183592, upload-time = "2025-07-29T01:10:47.916Z" }, + { url = "https://files.pythonhosted.org/packages/73/ac/3a5a182637b9a02c16335743b14485cb916ca984dcdc18737851732bff16/cffi-2.0.0b1-cp312-cp312-win_arm64.whl", hash = "sha256:bd7ce5d8224fb5a57bd7f1d9843aa4ecb870ec3f4a2101e1ba8314e91177e184", size = 177583, upload-time = "2025-07-29T01:10:49.091Z" }, +] + +[[package]] +name = "chardet" +version = "5.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/0d/f7b6ab21ec75897ed80c17d79b15951a719226b9fababf1e40ea74d69079/chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7", size = 2069618, upload-time = "2023-08-01T19:23:02.662Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970", size = 199385, upload-time = "2023-08-01T19:23:00.661Z" }, ] [[package]] name = "charset-normalizer" -version = "3.4.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e4/33/89c2ced2b67d1c2a61c19c6751aa8902d46ce3dacb23600a283619f5a12d/charset_normalizer-3.4.2.tar.gz", hash = "sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63", size = 126367, upload-time = "2025-05-02T08:34:42.01Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/85/4c40d00dcc6284a1c1ad5de5e0996b06f39d8232f1031cd23c2f5c07ee86/charset_normalizer-3.4.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2", size = 198794, upload-time = "2025-05-02T08:32:11.945Z" }, - { url = "https://files.pythonhosted.org/packages/41/d9/7a6c0b9db952598e97e93cbdfcb91bacd89b9b88c7c983250a77c008703c/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645", size = 142846, upload-time = "2025-05-02T08:32:13.946Z" }, - { url = "https://files.pythonhosted.org/packages/66/82/a37989cda2ace7e37f36c1a8ed16c58cf48965a79c2142713244bf945c89/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd", size = 153350, upload-time = "2025-05-02T08:32:15.873Z" }, - { url = "https://files.pythonhosted.org/packages/df/68/a576b31b694d07b53807269d05ec3f6f1093e9545e8607121995ba7a8313/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8", size = 145657, upload-time = "2025-05-02T08:32:17.283Z" }, - { url = "https://files.pythonhosted.org/packages/92/9b/ad67f03d74554bed3aefd56fe836e1623a50780f7c998d00ca128924a499/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f", size = 147260, upload-time = "2025-05-02T08:32:18.807Z" }, - { url = "https://files.pythonhosted.org/packages/a6/e6/8aebae25e328160b20e31a7e9929b1578bbdc7f42e66f46595a432f8539e/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7", size = 149164, upload-time = "2025-05-02T08:32:20.333Z" }, - { url = "https://files.pythonhosted.org/packages/8b/f2/b3c2f07dbcc248805f10e67a0262c93308cfa149a4cd3d1fe01f593e5fd2/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9", size = 144571, upload-time = "2025-05-02T08:32:21.86Z" }, - { url = "https://files.pythonhosted.org/packages/60/5b/c3f3a94bc345bc211622ea59b4bed9ae63c00920e2e8f11824aa5708e8b7/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544", size = 151952, upload-time = "2025-05-02T08:32:23.434Z" }, - { url = "https://files.pythonhosted.org/packages/e2/4d/ff460c8b474122334c2fa394a3f99a04cf11c646da895f81402ae54f5c42/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82", size = 155959, upload-time = "2025-05-02T08:32:24.993Z" }, - { url = "https://files.pythonhosted.org/packages/a2/2b/b964c6a2fda88611a1fe3d4c400d39c66a42d6c169c924818c848f922415/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0", size = 153030, upload-time = "2025-05-02T08:32:26.435Z" }, - { url = "https://files.pythonhosted.org/packages/59/2e/d3b9811db26a5ebf444bc0fa4f4be5aa6d76fc6e1c0fd537b16c14e849b6/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5", size = 148015, upload-time = "2025-05-02T08:32:28.376Z" }, - { url = "https://files.pythonhosted.org/packages/90/07/c5fd7c11eafd561bb51220d600a788f1c8d77c5eef37ee49454cc5c35575/charset_normalizer-3.4.2-cp311-cp311-win32.whl", hash = "sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a", size = 98106, upload-time = "2025-05-02T08:32:30.281Z" }, - { url = "https://files.pythonhosted.org/packages/a8/05/5e33dbef7e2f773d672b6d79f10ec633d4a71cd96db6673625838a4fd532/charset_normalizer-3.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28", size = 105402, upload-time = "2025-05-02T08:32:32.191Z" }, - { url = "https://files.pythonhosted.org/packages/d7/a4/37f4d6035c89cac7930395a35cc0f1b872e652eaafb76a6075943754f095/charset_normalizer-3.4.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7", size = 199936, upload-time = "2025-05-02T08:32:33.712Z" }, - { url = "https://files.pythonhosted.org/packages/ee/8a/1a5e33b73e0d9287274f899d967907cd0bf9c343e651755d9307e0dbf2b3/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3", size = 143790, upload-time = "2025-05-02T08:32:35.768Z" }, - { url = "https://files.pythonhosted.org/packages/66/52/59521f1d8e6ab1482164fa21409c5ef44da3e9f653c13ba71becdd98dec3/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a", size = 153924, upload-time = "2025-05-02T08:32:37.284Z" }, - { url = "https://files.pythonhosted.org/packages/86/2d/fb55fdf41964ec782febbf33cb64be480a6b8f16ded2dbe8db27a405c09f/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214", size = 146626, upload-time = "2025-05-02T08:32:38.803Z" }, - { url = "https://files.pythonhosted.org/packages/8c/73/6ede2ec59bce19b3edf4209d70004253ec5f4e319f9a2e3f2f15601ed5f7/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a", size = 148567, upload-time = "2025-05-02T08:32:40.251Z" }, - { url = "https://files.pythonhosted.org/packages/09/14/957d03c6dc343c04904530b6bef4e5efae5ec7d7990a7cbb868e4595ee30/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd", size = 150957, upload-time = "2025-05-02T08:32:41.705Z" }, - { url = "https://files.pythonhosted.org/packages/0d/c8/8174d0e5c10ccebdcb1b53cc959591c4c722a3ad92461a273e86b9f5a302/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981", size = 145408, upload-time = "2025-05-02T08:32:43.709Z" }, - { url = "https://files.pythonhosted.org/packages/58/aa/8904b84bc8084ac19dc52feb4f5952c6df03ffb460a887b42615ee1382e8/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c", size = 153399, upload-time = "2025-05-02T08:32:46.197Z" }, - { url = "https://files.pythonhosted.org/packages/c2/26/89ee1f0e264d201cb65cf054aca6038c03b1a0c6b4ae998070392a3ce605/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b", size = 156815, upload-time = "2025-05-02T08:32:48.105Z" }, - { url = "https://files.pythonhosted.org/packages/fd/07/68e95b4b345bad3dbbd3a8681737b4338ff2c9df29856a6d6d23ac4c73cb/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d", size = 154537, upload-time = "2025-05-02T08:32:49.719Z" }, - { url = "https://files.pythonhosted.org/packages/77/1a/5eefc0ce04affb98af07bc05f3bac9094513c0e23b0562d64af46a06aae4/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f", size = 149565, upload-time = "2025-05-02T08:32:51.404Z" }, - { url = "https://files.pythonhosted.org/packages/37/a0/2410e5e6032a174c95e0806b1a6585eb21e12f445ebe239fac441995226a/charset_normalizer-3.4.2-cp312-cp312-win32.whl", hash = "sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c", size = 98357, upload-time = "2025-05-02T08:32:53.079Z" }, - { url = "https://files.pythonhosted.org/packages/6c/4f/c02d5c493967af3eda9c771ad4d2bbc8df6f99ddbeb37ceea6e8716a32bc/charset_normalizer-3.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e", size = 105776, upload-time = "2025-05-02T08:32:54.573Z" }, - { url = "https://files.pythonhosted.org/packages/20/94/c5790835a017658cbfabd07f3bfb549140c3ac458cfc196323996b10095a/charset_normalizer-3.4.2-py3-none-any.whl", hash = "sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0", size = 52626, upload-time = "2025-05-02T08:34:40.053Z" }, +version = "3.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/2d/5fd176ceb9b2fc619e63405525573493ca23441330fcdaee6bef9460e924/charset_normalizer-3.4.3.tar.gz", hash = "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", size = 122371, upload-time = "2025-08-09T07:57:28.46Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/b5/991245018615474a60965a7c9cd2b4efbaabd16d582a5547c47ee1c7730b/charset_normalizer-3.4.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b256ee2e749283ef3ddcff51a675ff43798d92d746d1a6e4631bf8c707d22d0b", size = 204483, upload-time = "2025-08-09T07:55:53.12Z" }, + { url = "https://files.pythonhosted.org/packages/c7/2a/ae245c41c06299ec18262825c1569c5d3298fc920e4ddf56ab011b417efd/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13faeacfe61784e2559e690fc53fa4c5ae97c6fcedb8eb6fb8d0a15b475d2c64", size = 145520, upload-time = "2025-08-09T07:55:54.712Z" }, + { url = "https://files.pythonhosted.org/packages/3a/a4/b3b6c76e7a635748c4421d2b92c7b8f90a432f98bda5082049af37ffc8e3/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:00237675befef519d9af72169d8604a067d92755e84fe76492fef5441db05b91", size = 158876, upload-time = "2025-08-09T07:55:56.024Z" }, + { url = "https://files.pythonhosted.org/packages/e2/e6/63bb0e10f90a8243c5def74b5b105b3bbbfb3e7bb753915fe333fb0c11ea/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:585f3b2a80fbd26b048a0be90c5aae8f06605d3c92615911c3a2b03a8a3b796f", size = 156083, upload-time = "2025-08-09T07:55:57.582Z" }, + { url = "https://files.pythonhosted.org/packages/87/df/b7737ff046c974b183ea9aa111b74185ac8c3a326c6262d413bd5a1b8c69/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e78314bdc32fa80696f72fa16dc61168fda4d6a0c014e0380f9d02f0e5d8a07", size = 150295, upload-time = "2025-08-09T07:55:59.147Z" }, + { url = "https://files.pythonhosted.org/packages/61/f1/190d9977e0084d3f1dc169acd060d479bbbc71b90bf3e7bf7b9927dec3eb/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:96b2b3d1a83ad55310de8c7b4a2d04d9277d5591f40761274856635acc5fcb30", size = 148379, upload-time = "2025-08-09T07:56:00.364Z" }, + { url = "https://files.pythonhosted.org/packages/4c/92/27dbe365d34c68cfe0ca76f1edd70e8705d82b378cb54ebbaeabc2e3029d/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:939578d9d8fd4299220161fdd76e86c6a251987476f5243e8864a7844476ba14", size = 160018, upload-time = "2025-08-09T07:56:01.678Z" }, + { url = "https://files.pythonhosted.org/packages/99/04/baae2a1ea1893a01635d475b9261c889a18fd48393634b6270827869fa34/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:fd10de089bcdcd1be95a2f73dbe6254798ec1bda9f450d5828c96f93e2536b9c", size = 157430, upload-time = "2025-08-09T07:56:02.87Z" }, + { url = "https://files.pythonhosted.org/packages/2f/36/77da9c6a328c54d17b960c89eccacfab8271fdaaa228305330915b88afa9/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1e8ac75d72fa3775e0b7cb7e4629cec13b7514d928d15ef8ea06bca03ef01cae", size = 151600, upload-time = "2025-08-09T07:56:04.089Z" }, + { url = "https://files.pythonhosted.org/packages/64/d4/9eb4ff2c167edbbf08cdd28e19078bf195762e9bd63371689cab5ecd3d0d/charset_normalizer-3.4.3-cp311-cp311-win32.whl", hash = "sha256:6cf8fd4c04756b6b60146d98cd8a77d0cdae0e1ca20329da2ac85eed779b6849", size = 99616, upload-time = "2025-08-09T07:56:05.658Z" }, + { url = "https://files.pythonhosted.org/packages/f4/9c/996a4a028222e7761a96634d1820de8a744ff4327a00ada9c8942033089b/charset_normalizer-3.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:31a9a6f775f9bcd865d88ee350f0ffb0e25936a7f930ca98995c05abf1faf21c", size = 107108, upload-time = "2025-08-09T07:56:07.176Z" }, + { url = "https://files.pythonhosted.org/packages/e9/5e/14c94999e418d9b87682734589404a25854d5f5d0408df68bc15b6ff54bb/charset_normalizer-3.4.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", size = 205655, upload-time = "2025-08-09T07:56:08.475Z" }, + { url = "https://files.pythonhosted.org/packages/7d/a8/c6ec5d389672521f644505a257f50544c074cf5fc292d5390331cd6fc9c3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", size = 146223, upload-time = "2025-08-09T07:56:09.708Z" }, + { url = "https://files.pythonhosted.org/packages/fc/eb/a2ffb08547f4e1e5415fb69eb7db25932c52a52bed371429648db4d84fb1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", size = 159366, upload-time = "2025-08-09T07:56:11.326Z" }, + { url = "https://files.pythonhosted.org/packages/82/10/0fd19f20c624b278dddaf83b8464dcddc2456cb4b02bb902a6da126b87a1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392", size = 157104, upload-time = "2025-08-09T07:56:13.014Z" }, + { url = "https://files.pythonhosted.org/packages/16/ab/0233c3231af734f5dfcf0844aa9582d5a1466c985bbed6cedab85af9bfe3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f", size = 151830, upload-time = "2025-08-09T07:56:14.428Z" }, + { url = "https://files.pythonhosted.org/packages/ae/02/e29e22b4e02839a0e4a06557b1999d0a47db3567e82989b5bb21f3fbbd9f/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154", size = 148854, upload-time = "2025-08-09T07:56:16.051Z" }, + { url = "https://files.pythonhosted.org/packages/05/6b/e2539a0a4be302b481e8cafb5af8792da8093b486885a1ae4d15d452bcec/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491", size = 160670, upload-time = "2025-08-09T07:56:17.314Z" }, + { url = "https://files.pythonhosted.org/packages/31/e7/883ee5676a2ef217a40ce0bffcc3d0dfbf9e64cbcfbdf822c52981c3304b/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93", size = 158501, upload-time = "2025-08-09T07:56:18.641Z" }, + { url = "https://files.pythonhosted.org/packages/c1/35/6525b21aa0db614cf8b5792d232021dca3df7f90a1944db934efa5d20bb1/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f", size = 153173, upload-time = "2025-08-09T07:56:20.289Z" }, + { url = "https://files.pythonhosted.org/packages/50/ee/f4704bad8201de513fdc8aac1cabc87e38c5818c93857140e06e772b5892/charset_normalizer-3.4.3-cp312-cp312-win32.whl", hash = "sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37", size = 99822, upload-time = "2025-08-09T07:56:21.551Z" }, + { url = "https://files.pythonhosted.org/packages/39/f5/3b3836ca6064d0992c58c7561c6b6eee1b3892e9665d650c803bd5614522/charset_normalizer-3.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc", size = 107543, upload-time = "2025-08-09T07:56:23.115Z" }, + { url = "https://files.pythonhosted.org/packages/8a/1f/f041989e93b001bc4e44bb1669ccdcf54d3f00e628229a85b08d330615c5/charset_normalizer-3.4.3-py3-none-any.whl", hash = "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", size = 53175, upload-time = "2025-08-09T07:57:26.864Z" }, +] + +[[package]] +name = "cint" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/c8/3ae22fa142be0bf9eee856e90c314f4144dfae376cc5e3e55b9a169670fb/cint-1.0.0.tar.gz", hash = "sha256:66f026d28c46ef9ea9635be5cb342506c6a1af80d11cb1c881a8898ca429fc91", size = 4641, upload-time = "2019-03-19T01:07:48.723Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/c2/898e59963084e1e2cbd4aad1dee92c5bd7a79d121dcff1e659c2a0c2174e/cint-1.0.0-py3-none-any.whl", hash = "sha256:8aa33028e04015711c0305f918cb278f1dc8c5c9997acdc45efad2c7cb1abf50", size = 5573, upload-time = "2019-03-19T01:07:46.496Z" }, ] [[package]] @@ -585,6 +644,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/85/32/10bb5764d90a8eee674e9dc6f4db6a0ab47c8c4d0d83c27f7c39ac415a4d/click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b", size = 102215, upload-time = "2025-05-20T23:19:47.796Z" }, ] +[[package]] +name = "cloudpathlib" +version = "0.21.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/15/ae3256348834b92b9594d73eb7230538bae2bf726c2d721b920a668017c5/cloudpathlib-0.21.1.tar.gz", hash = "sha256:f26a855abf34d98f267aafd15efdb2db3c9665913dbabe5fad079df92837a431", size = 45295, upload-time = "2025-05-15T02:32:05.42Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/e7/6fea57b887f8e367c1e4a496ba03bfaf57824b766f777723ce1faf28834b/cloudpathlib-0.21.1-py3-none-any.whl", hash = "sha256:bfe580ad72ec030472ec233cd7380701b2d3227da7b2898387bd170aa70c803c", size = 52776, upload-time = "2025-05-15T02:32:03.99Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -594,70 +662,95 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "comm" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/13/7d740c5849255756bc17888787313b61fd38a0a8304fc4f073dfc46122aa/comm-0.2.3.tar.gz", hash = "sha256:2dc8048c10962d55d7ad693be1e7045d891b7ce8d999c97963a5e3e99c055971", size = 6319, upload-time = "2025-07-25T14:02:04.452Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl", hash = "sha256:c615d91d75f7f04f095b30d1c1711babd43bdc6419c1be9886a85f2f4e489417", size = 7294, upload-time = "2025-07-25T14:02:02.896Z" }, +] + +[[package]] +name = "confection" +version = "0.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "srsly" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/51/d3/57c6631159a1b48d273b40865c315cf51f89df7a9d1101094ef12e3a37c2/confection-0.1.5.tar.gz", hash = "sha256:8e72dd3ca6bd4f48913cd220f10b8275978e740411654b6e8ca6d7008c590f0e", size = 38924, upload-time = "2024-05-31T16:17:01.559Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/00/3106b1854b45bd0474ced037dfe6b73b90fe68a68968cef47c23de3d43d2/confection-0.1.5-py3-none-any.whl", hash = "sha256:e29d3c3f8eac06b3f77eb9dfb4bf2fc6bcc9622a98ca00a698e3d019c6430b14", size = 35451, upload-time = "2024-05-31T16:16:59.075Z" }, +] + [[package]] name = "contourpy" -version = "1.3.2" +version = "1.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/66/54/eb9bfc647b19f2009dd5c7f5ec51c4e6ca831725f1aea7a993034f483147/contourpy-1.3.2.tar.gz", hash = "sha256:b6945942715a034c671b7fc54f9588126b0b8bf23db2696e3ca8328f3ff0ab54", size = 13466130, upload-time = "2025-04-15T17:47:53.79Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/b9/ede788a0b56fc5b071639d06c33cb893f68b1178938f3425debebe2dab78/contourpy-1.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6a37a2fb93d4df3fc4c0e363ea4d16f83195fc09c891bc8ce072b9d084853445", size = 269636, upload-time = "2025-04-15T17:35:54.473Z" }, - { url = "https://files.pythonhosted.org/packages/e6/75/3469f011d64b8bbfa04f709bfc23e1dd71be54d05b1b083be9f5b22750d1/contourpy-1.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b7cd50c38f500bbcc9b6a46643a40e0913673f869315d8e70de0438817cb7773", size = 254636, upload-time = "2025-04-15T17:35:58.283Z" }, - { url = "https://files.pythonhosted.org/packages/8d/2f/95adb8dae08ce0ebca4fd8e7ad653159565d9739128b2d5977806656fcd2/contourpy-1.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6658ccc7251a4433eebd89ed2672c2ed96fba367fd25ca9512aa92a4b46c4f1", size = 313053, upload-time = "2025-04-15T17:36:03.235Z" }, - { url = "https://files.pythonhosted.org/packages/c3/a6/8ccf97a50f31adfa36917707fe39c9a0cbc24b3bbb58185577f119736cc9/contourpy-1.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:70771a461aaeb335df14deb6c97439973d253ae70660ca085eec25241137ef43", size = 352985, upload-time = "2025-04-15T17:36:08.275Z" }, - { url = "https://files.pythonhosted.org/packages/1d/b6/7925ab9b77386143f39d9c3243fdd101621b4532eb126743201160ffa7e6/contourpy-1.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65a887a6e8c4cd0897507d814b14c54a8c2e2aa4ac9f7686292f9769fcf9a6ab", size = 323750, upload-time = "2025-04-15T17:36:13.29Z" }, - { url = "https://files.pythonhosted.org/packages/c2/f3/20c5d1ef4f4748e52d60771b8560cf00b69d5c6368b5c2e9311bcfa2a08b/contourpy-1.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3859783aefa2b8355697f16642695a5b9792e7a46ab86da1118a4a23a51a33d7", size = 326246, upload-time = "2025-04-15T17:36:18.329Z" }, - { url = "https://files.pythonhosted.org/packages/8c/e5/9dae809e7e0b2d9d70c52b3d24cba134dd3dad979eb3e5e71f5df22ed1f5/contourpy-1.3.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:eab0f6db315fa4d70f1d8ab514e527f0366ec021ff853d7ed6a2d33605cf4b83", size = 1308728, upload-time = "2025-04-15T17:36:33.878Z" }, - { url = "https://files.pythonhosted.org/packages/e2/4a/0058ba34aeea35c0b442ae61a4f4d4ca84d6df8f91309bc2d43bb8dd248f/contourpy-1.3.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d91a3ccc7fea94ca0acab82ceb77f396d50a1f67412efe4c526f5d20264e6ecd", size = 1375762, upload-time = "2025-04-15T17:36:51.295Z" }, - { url = "https://files.pythonhosted.org/packages/09/33/7174bdfc8b7767ef2c08ed81244762d93d5c579336fc0b51ca57b33d1b80/contourpy-1.3.2-cp311-cp311-win32.whl", hash = "sha256:1c48188778d4d2f3d48e4643fb15d8608b1d01e4b4d6b0548d9b336c28fc9b6f", size = 178196, upload-time = "2025-04-15T17:36:55.002Z" }, - { url = "https://files.pythonhosted.org/packages/5e/fe/4029038b4e1c4485cef18e480b0e2cd2d755448bb071eb9977caac80b77b/contourpy-1.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:5ebac872ba09cb8f2131c46b8739a7ff71de28a24c869bcad554477eb089a878", size = 222017, upload-time = "2025-04-15T17:36:58.576Z" }, - { url = "https://files.pythonhosted.org/packages/34/f7/44785876384eff370c251d58fd65f6ad7f39adce4a093c934d4a67a7c6b6/contourpy-1.3.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4caf2bcd2969402bf77edc4cb6034c7dd7c0803213b3523f111eb7460a51b8d2", size = 271580, upload-time = "2025-04-15T17:37:03.105Z" }, - { url = "https://files.pythonhosted.org/packages/93/3b/0004767622a9826ea3d95f0e9d98cd8729015768075d61f9fea8eeca42a8/contourpy-1.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:82199cb78276249796419fe36b7386bd8d2cc3f28b3bc19fe2454fe2e26c4c15", size = 255530, upload-time = "2025-04-15T17:37:07.026Z" }, - { url = "https://files.pythonhosted.org/packages/e7/bb/7bd49e1f4fa805772d9fd130e0d375554ebc771ed7172f48dfcd4ca61549/contourpy-1.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:106fab697af11456fcba3e352ad50effe493a90f893fca6c2ca5c033820cea92", size = 307688, upload-time = "2025-04-15T17:37:11.481Z" }, - { url = "https://files.pythonhosted.org/packages/fc/97/e1d5dbbfa170725ef78357a9a0edc996b09ae4af170927ba8ce977e60a5f/contourpy-1.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d14f12932a8d620e307f715857107b1d1845cc44fdb5da2bc8e850f5ceba9f87", size = 347331, upload-time = "2025-04-15T17:37:18.212Z" }, - { url = "https://files.pythonhosted.org/packages/6f/66/e69e6e904f5ecf6901be3dd16e7e54d41b6ec6ae3405a535286d4418ffb4/contourpy-1.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:532fd26e715560721bb0d5fc7610fce279b3699b018600ab999d1be895b09415", size = 318963, upload-time = "2025-04-15T17:37:22.76Z" }, - { url = "https://files.pythonhosted.org/packages/a8/32/b8a1c8965e4f72482ff2d1ac2cd670ce0b542f203c8e1d34e7c3e6925da7/contourpy-1.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26b383144cf2d2c29f01a1e8170f50dacf0eac02d64139dcd709a8ac4eb3cfe", size = 323681, upload-time = "2025-04-15T17:37:33.001Z" }, - { url = "https://files.pythonhosted.org/packages/30/c6/12a7e6811d08757c7162a541ca4c5c6a34c0f4e98ef2b338791093518e40/contourpy-1.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c49f73e61f1f774650a55d221803b101d966ca0c5a2d6d5e4320ec3997489441", size = 1308674, upload-time = "2025-04-15T17:37:48.64Z" }, - { url = "https://files.pythonhosted.org/packages/2a/8a/bebe5a3f68b484d3a2b8ffaf84704b3e343ef1addea528132ef148e22b3b/contourpy-1.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3d80b2c0300583228ac98d0a927a1ba6a2ba6b8a742463c564f1d419ee5b211e", size = 1380480, upload-time = "2025-04-15T17:38:06.7Z" }, - { url = "https://files.pythonhosted.org/packages/34/db/fcd325f19b5978fb509a7d55e06d99f5f856294c1991097534360b307cf1/contourpy-1.3.2-cp312-cp312-win32.whl", hash = "sha256:90df94c89a91b7362e1142cbee7568f86514412ab8a2c0d0fca72d7e91b62912", size = 178489, upload-time = "2025-04-15T17:38:10.338Z" }, - { url = "https://files.pythonhosted.org/packages/01/c8/fadd0b92ffa7b5eb5949bf340a63a4a496a6930a6c37a7ba0f12acb076d6/contourpy-1.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:8c942a01d9163e2e5cfb05cb66110121b8d07ad438a17f9e766317bcb62abf73", size = 223042, upload-time = "2025-04-15T17:38:14.239Z" }, - { url = "https://files.pythonhosted.org/packages/ff/c0/91f1215d0d9f9f343e4773ba6c9b89e8c0cc7a64a6263f21139da639d848/contourpy-1.3.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:5f5964cdad279256c084b69c3f412b7801e15356b16efa9d78aa974041903da0", size = 266807, upload-time = "2025-04-15T17:45:15.535Z" }, - { url = "https://files.pythonhosted.org/packages/d4/79/6be7e90c955c0487e7712660d6cead01fa17bff98e0ea275737cc2bc8e71/contourpy-1.3.2-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49b65a95d642d4efa8f64ba12558fcb83407e58a2dfba9d796d77b63ccfcaff5", size = 318729, upload-time = "2025-04-15T17:45:20.166Z" }, - { url = "https://files.pythonhosted.org/packages/87/68/7f46fb537958e87427d98a4074bcde4b67a70b04900cfc5ce29bc2f556c1/contourpy-1.3.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:8c5acb8dddb0752bf252e01a3035b21443158910ac16a3b0d20e7fed7d534ce5", size = 221791, upload-time = "2025-04-15T17:45:24.794Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/2e/c4390a31919d8a78b90e8ecf87cd4b4c4f05a5b48d05ec17db8e5404c6f4/contourpy-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:709a48ef9a690e1343202916450bc48b9e51c049b089c7f79a267b46cffcdaa1", size = 288773, upload-time = "2025-07-26T12:01:02.277Z" }, + { url = "https://files.pythonhosted.org/packages/0d/44/c4b0b6095fef4dc9c420e041799591e3b63e9619e3044f7f4f6c21c0ab24/contourpy-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:23416f38bfd74d5d28ab8429cc4d63fa67d5068bd711a85edb1c3fb0c3e2f381", size = 270149, upload-time = "2025-07-26T12:01:04.072Z" }, + { url = "https://files.pythonhosted.org/packages/30/2e/dd4ced42fefac8470661d7cb7e264808425e6c5d56d175291e93890cce09/contourpy-1.3.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:929ddf8c4c7f348e4c0a5a3a714b5c8542ffaa8c22954862a46ca1813b667ee7", size = 329222, upload-time = "2025-07-26T12:01:05.688Z" }, + { url = "https://files.pythonhosted.org/packages/f2/74/cc6ec2548e3d276c71389ea4802a774b7aa3558223b7bade3f25787fafc2/contourpy-1.3.3-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e999574eddae35f1312c2b4b717b7885d4edd6cb46700e04f7f02db454e67c1", size = 377234, upload-time = "2025-07-26T12:01:07.054Z" }, + { url = "https://files.pythonhosted.org/packages/03/b3/64ef723029f917410f75c09da54254c5f9ea90ef89b143ccadb09df14c15/contourpy-1.3.3-cp311-cp311-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0bf67e0e3f482cb69779dd3061b534eb35ac9b17f163d851e2a547d56dba0a3a", size = 380555, upload-time = "2025-07-26T12:01:08.801Z" }, + { url = "https://files.pythonhosted.org/packages/5f/4b/6157f24ca425b89fe2eb7e7be642375711ab671135be21e6faa100f7448c/contourpy-1.3.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51e79c1f7470158e838808d4a996fa9bac72c498e93d8ebe5119bc1e6becb0db", size = 355238, upload-time = "2025-07-26T12:01:10.319Z" }, + { url = "https://files.pythonhosted.org/packages/98/56/f914f0dd678480708a04cfd2206e7c382533249bc5001eb9f58aa693e200/contourpy-1.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:598c3aaece21c503615fd59c92a3598b428b2f01bfb4b8ca9c4edeecc2438620", size = 1326218, upload-time = "2025-07-26T12:01:12.659Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d7/4a972334a0c971acd5172389671113ae82aa7527073980c38d5868ff1161/contourpy-1.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:322ab1c99b008dad206d406bb61d014cf0174df491ae9d9d0fac6a6fda4f977f", size = 1392867, upload-time = "2025-07-26T12:01:15.533Z" }, + { url = "https://files.pythonhosted.org/packages/75/3e/f2cc6cd56dc8cff46b1a56232eabc6feea52720083ea71ab15523daab796/contourpy-1.3.3-cp311-cp311-win32.whl", hash = "sha256:fd907ae12cd483cd83e414b12941c632a969171bf90fc937d0c9f268a31cafff", size = 183677, upload-time = "2025-07-26T12:01:17.088Z" }, + { url = "https://files.pythonhosted.org/packages/98/4b/9bd370b004b5c9d8045c6c33cf65bae018b27aca550a3f657cdc99acdbd8/contourpy-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:3519428f6be58431c56581f1694ba8e50626f2dd550af225f82fb5f5814d2a42", size = 225234, upload-time = "2025-07-26T12:01:18.256Z" }, + { url = "https://files.pythonhosted.org/packages/d9/b6/71771e02c2e004450c12b1120a5f488cad2e4d5b590b1af8bad060360fe4/contourpy-1.3.3-cp311-cp311-win_arm64.whl", hash = "sha256:15ff10bfada4bf92ec8b31c62bf7c1834c244019b4a33095a68000d7075df470", size = 193123, upload-time = "2025-07-26T12:01:19.848Z" }, + { url = "https://files.pythonhosted.org/packages/be/45/adfee365d9ea3d853550b2e735f9d66366701c65db7855cd07621732ccfc/contourpy-1.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b08a32ea2f8e42cf1d4be3169a98dd4be32bafe4f22b6c4cb4ba810fa9e5d2cb", size = 293419, upload-time = "2025-07-26T12:01:21.16Z" }, + { url = "https://files.pythonhosted.org/packages/53/3e/405b59cfa13021a56bba395a6b3aca8cec012b45bf177b0eaf7a202cde2c/contourpy-1.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:556dba8fb6f5d8742f2923fe9457dbdd51e1049c4a43fd3986a0b14a1d815fc6", size = 273979, upload-time = "2025-07-26T12:01:22.448Z" }, + { url = "https://files.pythonhosted.org/packages/d4/1c/a12359b9b2ca3a845e8f7f9ac08bdf776114eb931392fcad91743e2ea17b/contourpy-1.3.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92d9abc807cf7d0e047b95ca5d957cf4792fcd04e920ca70d48add15c1a90ea7", size = 332653, upload-time = "2025-07-26T12:01:24.155Z" }, + { url = "https://files.pythonhosted.org/packages/63/12/897aeebfb475b7748ea67b61e045accdfcf0d971f8a588b67108ed7f5512/contourpy-1.3.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2e8faa0ed68cb29af51edd8e24798bb661eac3bd9f65420c1887b6ca89987c8", size = 379536, upload-time = "2025-07-26T12:01:25.91Z" }, + { url = "https://files.pythonhosted.org/packages/43/8a/a8c584b82deb248930ce069e71576fc09bd7174bbd35183b7943fb1064fd/contourpy-1.3.3-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:626d60935cf668e70a5ce6ff184fd713e9683fb458898e4249b63be9e28286ea", size = 384397, upload-time = "2025-07-26T12:01:27.152Z" }, + { url = "https://files.pythonhosted.org/packages/cc/8f/ec6289987824b29529d0dfda0d74a07cec60e54b9c92f3c9da4c0ac732de/contourpy-1.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d00e655fcef08aba35ec9610536bfe90267d7ab5ba944f7032549c55a146da1", size = 362601, upload-time = "2025-07-26T12:01:28.808Z" }, + { url = "https://files.pythonhosted.org/packages/05/0a/a3fe3be3ee2dceb3e615ebb4df97ae6f3828aa915d3e10549ce016302bd1/contourpy-1.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:451e71b5a7d597379ef572de31eeb909a87246974d960049a9848c3bc6c41bf7", size = 1331288, upload-time = "2025-07-26T12:01:31.198Z" }, + { url = "https://files.pythonhosted.org/packages/33/1d/acad9bd4e97f13f3e2b18a3977fe1b4a37ecf3d38d815333980c6c72e963/contourpy-1.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:459c1f020cd59fcfe6650180678a9993932d80d44ccde1fa1868977438f0b411", size = 1403386, upload-time = "2025-07-26T12:01:33.947Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8f/5847f44a7fddf859704217a99a23a4f6417b10e5ab1256a179264561540e/contourpy-1.3.3-cp312-cp312-win32.whl", hash = "sha256:023b44101dfe49d7d53932be418477dba359649246075c996866106da069af69", size = 185018, upload-time = "2025-07-26T12:01:35.64Z" }, + { url = "https://files.pythonhosted.org/packages/19/e8/6026ed58a64563186a9ee3f29f41261fd1828f527dd93d33b60feca63352/contourpy-1.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:8153b8bfc11e1e4d75bcb0bff1db232f9e10b274e0929de9d608027e0d34ff8b", size = 226567, upload-time = "2025-07-26T12:01:36.804Z" }, + { url = "https://files.pythonhosted.org/packages/d1/e2/f05240d2c39a1ed228d8328a78b6f44cd695f7ef47beb3e684cf93604f86/contourpy-1.3.3-cp312-cp312-win_arm64.whl", hash = "sha256:07ce5ed73ecdc4a03ffe3e1b3e3c1166db35ae7584be76f65dbbe28a7791b0cc", size = 193655, upload-time = "2025-07-26T12:01:37.999Z" }, + { url = "https://files.pythonhosted.org/packages/a5/29/8dcfe16f0107943fa92388c23f6e05cff0ba58058c4c95b00280d4c75a14/contourpy-1.3.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:cd5dfcaeb10f7b7f9dc8941717c6c2ade08f587be2226222c12b25f0483ed497", size = 278809, upload-time = "2025-07-26T12:02:52.74Z" }, + { url = "https://files.pythonhosted.org/packages/85/a9/8b37ef4f7dafeb335daee3c8254645ef5725be4d9c6aa70b50ec46ef2f7e/contourpy-1.3.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:0c1fc238306b35f246d61a1d416a627348b5cf0648648a031e14bb8705fcdfe8", size = 261593, upload-time = "2025-07-26T12:02:54.037Z" }, + { url = "https://files.pythonhosted.org/packages/0a/59/ebfb8c677c75605cc27f7122c90313fd2f375ff3c8d19a1694bda74aaa63/contourpy-1.3.3-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70f9aad7de812d6541d29d2bbf8feb22ff7e1c299523db288004e3157ff4674e", size = 302202, upload-time = "2025-07-26T12:02:55.947Z" }, + { url = "https://files.pythonhosted.org/packages/3c/37/21972a15834d90bfbfb009b9d004779bd5a07a0ec0234e5ba8f64d5736f4/contourpy-1.3.3-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5ed3657edf08512fc3fe81b510e35c2012fbd3081d2e26160f27ca28affec989", size = 329207, upload-time = "2025-07-26T12:02:57.468Z" }, + { url = "https://files.pythonhosted.org/packages/0c/58/bd257695f39d05594ca4ad60df5bcb7e32247f9951fd09a9b8edb82d1daa/contourpy-1.3.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:3d1a3799d62d45c18bafd41c5fa05120b96a28079f2393af559b843d1a966a77", size = 225315, upload-time = "2025-07-26T12:02:58.801Z" }, ] [[package]] name = "coverage" -version = "7.9.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/04/b7/c0465ca253df10a9e8dae0692a4ae6e9726d245390aaef92360e1d6d3832/coverage-7.9.2.tar.gz", hash = "sha256:997024fa51e3290264ffd7492ec97d0690293ccd2b45a6cd7d82d945a4a80c8b", size = 813556, upload-time = "2025-07-03T10:54:15.101Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/39/40/916786453bcfafa4c788abee4ccd6f592b5b5eca0cd61a32a4e5a7ef6e02/coverage-7.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a7a56a2964a9687b6aba5b5ced6971af308ef6f79a91043c05dd4ee3ebc3e9ba", size = 212152, upload-time = "2025-07-03T10:52:53.562Z" }, - { url = "https://files.pythonhosted.org/packages/9f/66/cc13bae303284b546a030762957322bbbff1ee6b6cb8dc70a40f8a78512f/coverage-7.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:123d589f32c11d9be7fe2e66d823a236fe759b0096f5db3fb1b75b2fa414a4fa", size = 212540, upload-time = "2025-07-03T10:52:55.196Z" }, - { url = "https://files.pythonhosted.org/packages/0f/3c/d56a764b2e5a3d43257c36af4a62c379df44636817bb5f89265de4bf8bd7/coverage-7.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:333b2e0ca576a7dbd66e85ab402e35c03b0b22f525eed82681c4b866e2e2653a", size = 245097, upload-time = "2025-07-03T10:52:56.509Z" }, - { url = "https://files.pythonhosted.org/packages/b1/46/bd064ea8b3c94eb4ca5d90e34d15b806cba091ffb2b8e89a0d7066c45791/coverage-7.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:326802760da234baf9f2f85a39e4a4b5861b94f6c8d95251f699e4f73b1835dc", size = 242812, upload-time = "2025-07-03T10:52:57.842Z" }, - { url = "https://files.pythonhosted.org/packages/43/02/d91992c2b29bc7afb729463bc918ebe5f361be7f1daae93375a5759d1e28/coverage-7.9.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19e7be4cfec248df38ce40968c95d3952fbffd57b400d4b9bb580f28179556d2", size = 244617, upload-time = "2025-07-03T10:52:59.239Z" }, - { url = "https://files.pythonhosted.org/packages/b7/4f/8fadff6bf56595a16d2d6e33415841b0163ac660873ed9a4e9046194f779/coverage-7.9.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0b4a4cb73b9f2b891c1788711408ef9707666501ba23684387277ededab1097c", size = 244263, upload-time = "2025-07-03T10:53:00.601Z" }, - { url = "https://files.pythonhosted.org/packages/9b/d2/e0be7446a2bba11739edb9f9ba4eff30b30d8257370e237418eb44a14d11/coverage-7.9.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:2c8937fa16c8c9fbbd9f118588756e7bcdc7e16a470766a9aef912dd3f117dbd", size = 242314, upload-time = "2025-07-03T10:53:01.932Z" }, - { url = "https://files.pythonhosted.org/packages/9d/7d/dcbac9345000121b8b57a3094c2dfcf1ccc52d8a14a40c1d4bc89f936f80/coverage-7.9.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:42da2280c4d30c57a9b578bafd1d4494fa6c056d4c419d9689e66d775539be74", size = 242904, upload-time = "2025-07-03T10:53:03.478Z" }, - { url = "https://files.pythonhosted.org/packages/41/58/11e8db0a0c0510cf31bbbdc8caf5d74a358b696302a45948d7c768dfd1cf/coverage-7.9.2-cp311-cp311-win32.whl", hash = "sha256:14fa8d3da147f5fdf9d298cacc18791818f3f1a9f542c8958b80c228320e90c6", size = 214553, upload-time = "2025-07-03T10:53:05.174Z" }, - { url = "https://files.pythonhosted.org/packages/3a/7d/751794ec8907a15e257136e48dc1021b1f671220ecccfd6c4eaf30802714/coverage-7.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:549cab4892fc82004f9739963163fd3aac7a7b0df430669b75b86d293d2df2a7", size = 215441, upload-time = "2025-07-03T10:53:06.472Z" }, - { url = "https://files.pythonhosted.org/packages/62/5b/34abcedf7b946c1c9e15b44f326cb5b0da852885312b30e916f674913428/coverage-7.9.2-cp311-cp311-win_arm64.whl", hash = "sha256:c2667a2b913e307f06aa4e5677f01a9746cd08e4b35e14ebcde6420a9ebb4c62", size = 213873, upload-time = "2025-07-03T10:53:07.699Z" }, - { url = "https://files.pythonhosted.org/packages/53/d7/7deefc6fd4f0f1d4c58051f4004e366afc9e7ab60217ac393f247a1de70a/coverage-7.9.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ae9eb07f1cfacd9cfe8eaee6f4ff4b8a289a668c39c165cd0c8548484920ffc0", size = 212344, upload-time = "2025-07-03T10:53:09.3Z" }, - { url = "https://files.pythonhosted.org/packages/95/0c/ee03c95d32be4d519e6a02e601267769ce2e9a91fc8faa1b540e3626c680/coverage-7.9.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9ce85551f9a1119f02adc46d3014b5ee3f765deac166acf20dbb851ceb79b6f3", size = 212580, upload-time = "2025-07-03T10:53:11.52Z" }, - { url = "https://files.pythonhosted.org/packages/8b/9f/826fa4b544b27620086211b87a52ca67592622e1f3af9e0a62c87aea153a/coverage-7.9.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8f6389ac977c5fb322e0e38885fbbf901743f79d47f50db706e7644dcdcb6e1", size = 246383, upload-time = "2025-07-03T10:53:13.134Z" }, - { url = "https://files.pythonhosted.org/packages/7f/b3/4477aafe2a546427b58b9c540665feff874f4db651f4d3cb21b308b3a6d2/coverage-7.9.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff0d9eae8cdfcd58fe7893b88993723583a6ce4dfbfd9f29e001922544f95615", size = 243400, upload-time = "2025-07-03T10:53:14.614Z" }, - { url = "https://files.pythonhosted.org/packages/f8/c2/efffa43778490c226d9d434827702f2dfbc8041d79101a795f11cbb2cf1e/coverage-7.9.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fae939811e14e53ed8a9818dad51d434a41ee09df9305663735f2e2d2d7d959b", size = 245591, upload-time = "2025-07-03T10:53:15.872Z" }, - { url = "https://files.pythonhosted.org/packages/c6/e7/a59888e882c9a5f0192d8627a30ae57910d5d449c80229b55e7643c078c4/coverage-7.9.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:31991156251ec202c798501e0a42bbdf2169dcb0f137b1f5c0f4267f3fc68ef9", size = 245402, upload-time = "2025-07-03T10:53:17.124Z" }, - { url = "https://files.pythonhosted.org/packages/92/a5/72fcd653ae3d214927edc100ce67440ed8a0a1e3576b8d5e6d066ed239db/coverage-7.9.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d0d67963f9cbfc7c7f96d4ac74ed60ecbebd2ea6eeb51887af0f8dce205e545f", size = 243583, upload-time = "2025-07-03T10:53:18.781Z" }, - { url = "https://files.pythonhosted.org/packages/5c/f5/84e70e4df28f4a131d580d7d510aa1ffd95037293da66fd20d446090a13b/coverage-7.9.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:49b752a2858b10580969ec6af6f090a9a440a64a301ac1528d7ca5f7ed497f4d", size = 244815, upload-time = "2025-07-03T10:53:20.168Z" }, - { url = "https://files.pythonhosted.org/packages/39/e7/d73d7cbdbd09fdcf4642655ae843ad403d9cbda55d725721965f3580a314/coverage-7.9.2-cp312-cp312-win32.whl", hash = "sha256:88d7598b8ee130f32f8a43198ee02edd16d7f77692fa056cb779616bbea1b355", size = 214719, upload-time = "2025-07-03T10:53:21.521Z" }, - { url = "https://files.pythonhosted.org/packages/9f/d6/7486dcc3474e2e6ad26a2af2db7e7c162ccd889c4c68fa14ea8ec189c9e9/coverage-7.9.2-cp312-cp312-win_amd64.whl", hash = "sha256:9dfb070f830739ee49d7c83e4941cc767e503e4394fdecb3b54bfdac1d7662c0", size = 215509, upload-time = "2025-07-03T10:53:22.853Z" }, - { url = "https://files.pythonhosted.org/packages/b7/34/0439f1ae2593b0346164d907cdf96a529b40b7721a45fdcf8b03c95fcd90/coverage-7.9.2-cp312-cp312-win_arm64.whl", hash = "sha256:4e2c058aef613e79df00e86b6d42a641c877211384ce5bd07585ed7ba71ab31b", size = 213910, upload-time = "2025-07-03T10:53:24.472Z" }, - { url = "https://files.pythonhosted.org/packages/d7/85/f8bbefac27d286386961c25515431482a425967e23d3698b75a250872924/coverage-7.9.2-pp39.pp310.pp311-none-any.whl", hash = "sha256:8a1166db2fb62473285bcb092f586e081e92656c7dfa8e9f62b4d39d7e6b5050", size = 204013, upload-time = "2025-07-03T10:54:12.084Z" }, - { url = "https://files.pythonhosted.org/packages/3c/38/bbe2e63902847cf79036ecc75550d0698af31c91c7575352eb25190d0fb3/coverage-7.9.2-py3-none-any.whl", hash = "sha256:e425cd5b00f6fc0ed7cdbd766c70be8baab4b7839e4d4fe5fac48581dd968ea4", size = 204005, upload-time = "2025-07-03T10:54:13.491Z" }, +version = "7.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/4e/08b493f1f1d8a5182df0044acc970799b58a8d289608e0d891a03e9d269a/coverage-7.10.4.tar.gz", hash = "sha256:25f5130af6c8e7297fd14634955ba9e1697f47143f289e2a23284177c0061d27", size = 823798, upload-time = "2025-08-17T00:26:43.314Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/ba/2c9817e62018e7d480d14f684c160b3038df9ff69c5af7d80e97d143e4d1/coverage-7.10.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:05d5f98ec893d4a2abc8bc5f046f2f4367404e7e5d5d18b83de8fde1093ebc4f", size = 216514, upload-time = "2025-08-17T00:24:34.188Z" }, + { url = "https://files.pythonhosted.org/packages/e3/5a/093412a959a6b6261446221ba9fb23bb63f661a5de70b5d130763c87f916/coverage-7.10.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9267efd28f8994b750d171e58e481e3bbd69e44baed540e4c789f8e368b24b88", size = 216914, upload-time = "2025-08-17T00:24:35.881Z" }, + { url = "https://files.pythonhosted.org/packages/2c/1f/2fdf4a71cfe93b07eae845ebf763267539a7d8b7e16b062f959d56d7e433/coverage-7.10.4-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4456a039fdc1a89ea60823d0330f1ac6f97b0dbe9e2b6fb4873e889584b085fb", size = 247308, upload-time = "2025-08-17T00:24:37.61Z" }, + { url = "https://files.pythonhosted.org/packages/ba/16/33f6cded458e84f008b9f6bc379609a6a1eda7bffe349153b9960803fc11/coverage-7.10.4-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c2bfbd2a9f7e68a21c5bd191be94bfdb2691ac40d325bac9ef3ae45ff5c753d9", size = 249241, upload-time = "2025-08-17T00:24:38.919Z" }, + { url = "https://files.pythonhosted.org/packages/84/98/9c18e47c889be58339ff2157c63b91a219272503ee32b49d926eea2337f2/coverage-7.10.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab7765f10ae1df7e7fe37de9e64b5a269b812ee22e2da3f84f97b1c7732a0d8", size = 251346, upload-time = "2025-08-17T00:24:40.507Z" }, + { url = "https://files.pythonhosted.org/packages/6d/07/00a6c0d53e9a22d36d8e95ddd049b860eef8f4b9fd299f7ce34d8e323356/coverage-7.10.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0a09b13695166236e171ec1627ff8434b9a9bae47528d0ba9d944c912d33b3d2", size = 249037, upload-time = "2025-08-17T00:24:41.904Z" }, + { url = "https://files.pythonhosted.org/packages/3e/0e/1e1b944d6a6483d07bab5ef6ce063fcf3d0cc555a16a8c05ebaab11f5607/coverage-7.10.4-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5c9e75dfdc0167d5675e9804f04a56b2cf47fb83a524654297000b578b8adcb7", size = 247090, upload-time = "2025-08-17T00:24:43.193Z" }, + { url = "https://files.pythonhosted.org/packages/62/43/2ce5ab8a728b8e25ced077111581290ffaef9efaf860a28e25435ab925cf/coverage-7.10.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c751261bfe6481caba15ec005a194cb60aad06f29235a74c24f18546d8377df0", size = 247732, upload-time = "2025-08-17T00:24:44.906Z" }, + { url = "https://files.pythonhosted.org/packages/a4/f3/706c4a24f42c1c5f3a2ca56637ab1270f84d9e75355160dc34d5e39bb5b7/coverage-7.10.4-cp311-cp311-win32.whl", hash = "sha256:051c7c9e765f003c2ff6e8c81ccea28a70fb5b0142671e4e3ede7cebd45c80af", size = 218961, upload-time = "2025-08-17T00:24:46.241Z" }, + { url = "https://files.pythonhosted.org/packages/e8/aa/6b9ea06e0290bf1cf2a2765bba89d561c5c563b4e9db8298bf83699c8b67/coverage-7.10.4-cp311-cp311-win_amd64.whl", hash = "sha256:1a647b152f10be08fb771ae4a1421dbff66141e3d8ab27d543b5eb9ea5af8e52", size = 219851, upload-time = "2025-08-17T00:24:48.795Z" }, + { url = "https://files.pythonhosted.org/packages/8b/be/f0dc9ad50ee183369e643cd7ed8f2ef5c491bc20b4c3387cbed97dd6e0d1/coverage-7.10.4-cp311-cp311-win_arm64.whl", hash = "sha256:b09b9e4e1de0d406ca9f19a371c2beefe3193b542f64a6dd40cfcf435b7d6aa0", size = 218530, upload-time = "2025-08-17T00:24:50.164Z" }, + { url = "https://files.pythonhosted.org/packages/9e/4a/781c9e4dd57cabda2a28e2ce5b00b6be416015265851060945a5ed4bd85e/coverage-7.10.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a1f0264abcabd4853d4cb9b3d164adbf1565da7dab1da1669e93f3ea60162d79", size = 216706, upload-time = "2025-08-17T00:24:51.528Z" }, + { url = "https://files.pythonhosted.org/packages/6a/8c/51255202ca03d2e7b664770289f80db6f47b05138e06cce112b3957d5dfd/coverage-7.10.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:536cbe6b118a4df231b11af3e0f974a72a095182ff8ec5f4868c931e8043ef3e", size = 216939, upload-time = "2025-08-17T00:24:53.171Z" }, + { url = "https://files.pythonhosted.org/packages/06/7f/df11131483698660f94d3c847dc76461369782d7a7644fcd72ac90da8fd0/coverage-7.10.4-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:9a4c0d84134797b7bf3f080599d0cd501471f6c98b715405166860d79cfaa97e", size = 248429, upload-time = "2025-08-17T00:24:54.934Z" }, + { url = "https://files.pythonhosted.org/packages/eb/fa/13ac5eda7300e160bf98f082e75f5c5b4189bf3a883dd1ee42dbedfdc617/coverage-7.10.4-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7c155fc0f9cee8c9803ea0ad153ab6a3b956baa5d4cd993405dc0b45b2a0b9e0", size = 251178, upload-time = "2025-08-17T00:24:56.353Z" }, + { url = "https://files.pythonhosted.org/packages/9a/bc/f63b56a58ad0bec68a840e7be6b7ed9d6f6288d790760647bb88f5fea41e/coverage-7.10.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a5f2ab6e451d4b07855d8bcf063adf11e199bff421a4ba57f5bb95b7444ca62", size = 252313, upload-time = "2025-08-17T00:24:57.692Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b6/79338f1ea27b01266f845afb4485976211264ab92407d1c307babe3592a7/coverage-7.10.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:685b67d99b945b0c221be0780c336b303a7753b3e0ec0d618c795aada25d5e7a", size = 250230, upload-time = "2025-08-17T00:24:59.293Z" }, + { url = "https://files.pythonhosted.org/packages/bc/93/3b24f1da3e0286a4dc5832427e1d448d5296f8287464b1ff4a222abeeeb5/coverage-7.10.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0c079027e50c2ae44da51c2e294596cbc9dbb58f7ca45b30651c7e411060fc23", size = 248351, upload-time = "2025-08-17T00:25:00.676Z" }, + { url = "https://files.pythonhosted.org/packages/de/5f/d59412f869e49dcc5b89398ef3146c8bfaec870b179cc344d27932e0554b/coverage-7.10.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3749aa72b93ce516f77cf5034d8e3c0dfd45c6e8a163a602ede2dc5f9a0bb927", size = 249788, upload-time = "2025-08-17T00:25:02.354Z" }, + { url = "https://files.pythonhosted.org/packages/cc/52/04a3b733f40a0cc7c4a5b9b010844111dbf906df3e868b13e1ce7b39ac31/coverage-7.10.4-cp312-cp312-win32.whl", hash = "sha256:fecb97b3a52fa9bcd5a7375e72fae209088faf671d39fae67261f37772d5559a", size = 219131, upload-time = "2025-08-17T00:25:03.79Z" }, + { url = "https://files.pythonhosted.org/packages/83/dd/12909fc0b83888197b3ec43a4ac7753589591c08d00d9deda4158df2734e/coverage-7.10.4-cp312-cp312-win_amd64.whl", hash = "sha256:26de58f355626628a21fe6a70e1e1fad95702dafebfb0685280962ae1449f17b", size = 219939, upload-time = "2025-08-17T00:25:05.494Z" }, + { url = "https://files.pythonhosted.org/packages/83/c7/058bb3220fdd6821bada9685eadac2940429ab3c97025ce53549ff423cc1/coverage-7.10.4-cp312-cp312-win_arm64.whl", hash = "sha256:67e8885408f8325198862bc487038a4980c9277d753cb8812510927f2176437a", size = 218572, upload-time = "2025-08-17T00:25:06.897Z" }, + { url = "https://files.pythonhosted.org/packages/bb/78/983efd23200921d9edb6bd40512e1aa04af553d7d5a171e50f9b2b45d109/coverage-7.10.4-py3-none-any.whl", hash = "sha256:065d75447228d05121e5c938ca8f0e91eed60a1eb2d1258d42d5084fecfc3302", size = 208365, upload-time = "2025-08-17T00:26:41.479Z" }, ] [package.optional-dependencies] @@ -667,43 +760,64 @@ toml = [ [[package]] name = "cryptography" -version = "45.0.5" +version = "45.0.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/95/1e/49527ac611af559665f71cbb8f92b332b5ec9c6fbc4e88b0f8e92f5e85df/cryptography-45.0.5.tar.gz", hash = "sha256:72e76caa004ab63accdf26023fccd1d087f6d90ec6048ff33ad0445abf7f605a", size = 744903, upload-time = "2025-07-02T13:06:25.941Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/fb/09e28bc0c46d2c547085e60897fea96310574c70fb21cd58a730a45f3403/cryptography-45.0.5-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:101ee65078f6dd3e5a028d4f19c07ffa4dd22cce6a20eaa160f8b5219911e7d8", size = 7043092, upload-time = "2025-07-02T13:05:01.514Z" }, - { url = "https://files.pythonhosted.org/packages/b1/05/2194432935e29b91fb649f6149c1a4f9e6d3d9fc880919f4ad1bcc22641e/cryptography-45.0.5-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3a264aae5f7fbb089dbc01e0242d3b67dffe3e6292e1f5182122bdf58e65215d", size = 4205926, upload-time = "2025-07-02T13:05:04.741Z" }, - { url = "https://files.pythonhosted.org/packages/07/8b/9ef5da82350175e32de245646b1884fc01124f53eb31164c77f95a08d682/cryptography-45.0.5-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e74d30ec9c7cb2f404af331d5b4099a9b322a8a6b25c4632755c8757345baac5", size = 4429235, upload-time = "2025-07-02T13:05:07.084Z" }, - { url = "https://files.pythonhosted.org/packages/7c/e1/c809f398adde1994ee53438912192d92a1d0fc0f2d7582659d9ef4c28b0c/cryptography-45.0.5-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3af26738f2db354aafe492fb3869e955b12b2ef2e16908c8b9cb928128d42c57", size = 4209785, upload-time = "2025-07-02T13:05:09.321Z" }, - { url = "https://files.pythonhosted.org/packages/d0/8b/07eb6bd5acff58406c5e806eff34a124936f41a4fb52909ffa4d00815f8c/cryptography-45.0.5-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e6c00130ed423201c5bc5544c23359141660b07999ad82e34e7bb8f882bb78e0", size = 3893050, upload-time = "2025-07-02T13:05:11.069Z" }, - { url = "https://files.pythonhosted.org/packages/ec/ef/3333295ed58d900a13c92806b67e62f27876845a9a908c939f040887cca9/cryptography-45.0.5-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:dd420e577921c8c2d31289536c386aaa30140b473835e97f83bc71ea9d2baf2d", size = 4457379, upload-time = "2025-07-02T13:05:13.32Z" }, - { url = "https://files.pythonhosted.org/packages/d9/9d/44080674dee514dbb82b21d6fa5d1055368f208304e2ab1828d85c9de8f4/cryptography-45.0.5-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:d05a38884db2ba215218745f0781775806bde4f32e07b135348355fe8e4991d9", size = 4209355, upload-time = "2025-07-02T13:05:15.017Z" }, - { url = "https://files.pythonhosted.org/packages/c9/d8/0749f7d39f53f8258e5c18a93131919ac465ee1f9dccaf1b3f420235e0b5/cryptography-45.0.5-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:ad0caded895a00261a5b4aa9af828baede54638754b51955a0ac75576b831b27", size = 4456087, upload-time = "2025-07-02T13:05:16.945Z" }, - { url = "https://files.pythonhosted.org/packages/09/d7/92acac187387bf08902b0bf0699816f08553927bdd6ba3654da0010289b4/cryptography-45.0.5-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9024beb59aca9d31d36fcdc1604dd9bbeed0a55bface9f1908df19178e2f116e", size = 4332873, upload-time = "2025-07-02T13:05:18.743Z" }, - { url = "https://files.pythonhosted.org/packages/03/c2/840e0710da5106a7c3d4153c7215b2736151bba60bf4491bdb421df5056d/cryptography-45.0.5-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:91098f02ca81579c85f66df8a588c78f331ca19089763d733e34ad359f474174", size = 4564651, upload-time = "2025-07-02T13:05:21.382Z" }, - { url = "https://files.pythonhosted.org/packages/2e/92/cc723dd6d71e9747a887b94eb3827825c6c24b9e6ce2bb33b847d31d5eaa/cryptography-45.0.5-cp311-abi3-win32.whl", hash = "sha256:926c3ea71a6043921050eaa639137e13dbe7b4ab25800932a8498364fc1abec9", size = 2929050, upload-time = "2025-07-02T13:05:23.39Z" }, - { url = "https://files.pythonhosted.org/packages/1f/10/197da38a5911a48dd5389c043de4aec4b3c94cb836299b01253940788d78/cryptography-45.0.5-cp311-abi3-win_amd64.whl", hash = "sha256:b85980d1e345fe769cfc57c57db2b59cff5464ee0c045d52c0df087e926fbe63", size = 3403224, upload-time = "2025-07-02T13:05:25.202Z" }, - { url = "https://files.pythonhosted.org/packages/fe/2b/160ce8c2765e7a481ce57d55eba1546148583e7b6f85514472b1d151711d/cryptography-45.0.5-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:f3562c2f23c612f2e4a6964a61d942f891d29ee320edb62ff48ffb99f3de9ae8", size = 7017143, upload-time = "2025-07-02T13:05:27.229Z" }, - { url = "https://files.pythonhosted.org/packages/c2/e7/2187be2f871c0221a81f55ee3105d3cf3e273c0a0853651d7011eada0d7e/cryptography-45.0.5-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3fcfbefc4a7f332dece7272a88e410f611e79458fab97b5efe14e54fe476f4fd", size = 4197780, upload-time = "2025-07-02T13:05:29.299Z" }, - { url = "https://files.pythonhosted.org/packages/b9/cf/84210c447c06104e6be9122661159ad4ce7a8190011669afceeaea150524/cryptography-45.0.5-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:460f8c39ba66af7db0545a8c6f2eabcbc5a5528fc1cf6c3fa9a1e44cec33385e", size = 4420091, upload-time = "2025-07-02T13:05:31.221Z" }, - { url = "https://files.pythonhosted.org/packages/3e/6a/cb8b5c8bb82fafffa23aeff8d3a39822593cee6e2f16c5ca5c2ecca344f7/cryptography-45.0.5-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9b4cf6318915dccfe218e69bbec417fdd7c7185aa7aab139a2c0beb7468c89f0", size = 4198711, upload-time = "2025-07-02T13:05:33.062Z" }, - { url = "https://files.pythonhosted.org/packages/04/f7/36d2d69df69c94cbb2473871926daf0f01ad8e00fe3986ac3c1e8c4ca4b3/cryptography-45.0.5-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2089cc8f70a6e454601525e5bf2779e665d7865af002a5dec8d14e561002e135", size = 3883299, upload-time = "2025-07-02T13:05:34.94Z" }, - { url = "https://files.pythonhosted.org/packages/82/c7/f0ea40f016de72f81288e9fe8d1f6748036cb5ba6118774317a3ffc6022d/cryptography-45.0.5-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0027d566d65a38497bc37e0dd7c2f8ceda73597d2ac9ba93810204f56f52ebc7", size = 4450558, upload-time = "2025-07-02T13:05:37.288Z" }, - { url = "https://files.pythonhosted.org/packages/06/ae/94b504dc1a3cdf642d710407c62e86296f7da9e66f27ab12a1ee6fdf005b/cryptography-45.0.5-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:be97d3a19c16a9be00edf79dca949c8fa7eff621763666a145f9f9535a5d7f42", size = 4198020, upload-time = "2025-07-02T13:05:39.102Z" }, - { url = "https://files.pythonhosted.org/packages/05/2b/aaf0adb845d5dabb43480f18f7ca72e94f92c280aa983ddbd0bcd6ecd037/cryptography-45.0.5-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:7760c1c2e1a7084153a0f68fab76e754083b126a47d0117c9ed15e69e2103492", size = 4449759, upload-time = "2025-07-02T13:05:41.398Z" }, - { url = "https://files.pythonhosted.org/packages/91/e4/f17e02066de63e0100a3a01b56f8f1016973a1d67551beaf585157a86b3f/cryptography-45.0.5-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6ff8728d8d890b3dda5765276d1bc6fb099252915a2cd3aff960c4c195745dd0", size = 4319991, upload-time = "2025-07-02T13:05:43.64Z" }, - { url = "https://files.pythonhosted.org/packages/f2/2e/e2dbd629481b499b14516eed933f3276eb3239f7cee2dcfa4ee6b44d4711/cryptography-45.0.5-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7259038202a47fdecee7e62e0fd0b0738b6daa335354396c6ddebdbe1206af2a", size = 4554189, upload-time = "2025-07-02T13:05:46.045Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ea/a78a0c38f4c8736287b71c2ea3799d173d5ce778c7d6e3c163a95a05ad2a/cryptography-45.0.5-cp37-abi3-win32.whl", hash = "sha256:1e1da5accc0c750056c556a93c3e9cb828970206c68867712ca5805e46dc806f", size = 2911769, upload-time = "2025-07-02T13:05:48.329Z" }, - { url = "https://files.pythonhosted.org/packages/79/b3/28ac139109d9005ad3f6b6f8976ffede6706a6478e21c889ce36c840918e/cryptography-45.0.5-cp37-abi3-win_amd64.whl", hash = "sha256:90cb0a7bb35959f37e23303b7eed0a32280510030daba3f7fdfbb65defde6a97", size = 3390016, upload-time = "2025-07-02T13:05:50.811Z" }, - { url = "https://files.pythonhosted.org/packages/c0/71/9bdbcfd58d6ff5084687fe722c58ac718ebedbc98b9f8f93781354e6d286/cryptography-45.0.5-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8c4a6ff8a30e9e3d38ac0539e9a9e02540ab3f827a3394f8852432f6b0ea152e", size = 3587878, upload-time = "2025-07-02T13:06:06.339Z" }, - { url = "https://files.pythonhosted.org/packages/f0/63/83516cfb87f4a8756eaa4203f93b283fda23d210fc14e1e594bd5f20edb6/cryptography-45.0.5-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:bd4c45986472694e5121084c6ebbd112aa919a25e783b87eb95953c9573906d6", size = 4152447, upload-time = "2025-07-02T13:06:08.345Z" }, - { url = "https://files.pythonhosted.org/packages/22/11/d2823d2a5a0bd5802b3565437add16f5c8ce1f0778bf3822f89ad2740a38/cryptography-45.0.5-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:982518cd64c54fcada9d7e5cf28eabd3ee76bd03ab18e08a48cad7e8b6f31b18", size = 4386778, upload-time = "2025-07-02T13:06:10.263Z" }, - { url = "https://files.pythonhosted.org/packages/5f/38/6bf177ca6bce4fe14704ab3e93627c5b0ca05242261a2e43ef3168472540/cryptography-45.0.5-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:12e55281d993a793b0e883066f590c1ae1e802e3acb67f8b442e721e475e6463", size = 4151627, upload-time = "2025-07-02T13:06:13.097Z" }, - { url = "https://files.pythonhosted.org/packages/38/6a/69fc67e5266bff68a91bcb81dff8fb0aba4d79a78521a08812048913e16f/cryptography-45.0.5-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:5aa1e32983d4443e310f726ee4b071ab7569f58eedfdd65e9675484a4eb67bd1", size = 4385593, upload-time = "2025-07-02T13:06:15.689Z" }, - { url = "https://files.pythonhosted.org/packages/f6/34/31a1604c9a9ade0fdab61eb48570e09a796f4d9836121266447b0eaf7feb/cryptography-45.0.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:e357286c1b76403dd384d938f93c46b2b058ed4dfcdce64a770f0537ed3feb6f", size = 3331106, upload-time = "2025-07-02T13:06:18.058Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/d6/0d/d13399c94234ee8f3df384819dc67e0c5ce215fb751d567a55a1f4b028c7/cryptography-45.0.6.tar.gz", hash = "sha256:5c966c732cf6e4a276ce83b6e4c729edda2df6929083a952cc7da973c539c719", size = 744949, upload-time = "2025-08-05T23:59:27.93Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/29/2793d178d0eda1ca4a09a7c4e09a5185e75738cc6d526433e8663b460ea6/cryptography-45.0.6-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:048e7ad9e08cf4c0ab07ff7f36cc3115924e22e2266e034450a890d9e312dd74", size = 7042702, upload-time = "2025-08-05T23:58:23.464Z" }, + { url = "https://files.pythonhosted.org/packages/b3/b6/cabd07410f222f32c8d55486c464f432808abaa1f12af9afcbe8f2f19030/cryptography-45.0.6-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:44647c5d796f5fc042bbc6d61307d04bf29bccb74d188f18051b635f20a9c75f", size = 4206483, upload-time = "2025-08-05T23:58:27.132Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9e/f9c7d36a38b1cfeb1cc74849aabe9bf817990f7603ff6eb485e0d70e0b27/cryptography-45.0.6-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e40b80ecf35ec265c452eea0ba94c9587ca763e739b8e559c128d23bff7ebbbf", size = 4429679, upload-time = "2025-08-05T23:58:29.152Z" }, + { url = "https://files.pythonhosted.org/packages/9c/2a/4434c17eb32ef30b254b9e8b9830cee4e516f08b47fdd291c5b1255b8101/cryptography-45.0.6-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:00e8724bdad672d75e6f069b27970883179bd472cd24a63f6e620ca7e41cc0c5", size = 4210553, upload-time = "2025-08-05T23:58:30.596Z" }, + { url = "https://files.pythonhosted.org/packages/ef/1d/09a5df8e0c4b7970f5d1f3aff1b640df6d4be28a64cae970d56c6cf1c772/cryptography-45.0.6-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7a3085d1b319d35296176af31c90338eeb2ddac8104661df79f80e1d9787b8b2", size = 3894499, upload-time = "2025-08-05T23:58:32.03Z" }, + { url = "https://files.pythonhosted.org/packages/79/62/120842ab20d9150a9d3a6bdc07fe2870384e82f5266d41c53b08a3a96b34/cryptography-45.0.6-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1b7fa6a1c1188c7ee32e47590d16a5a0646270921f8020efc9a511648e1b2e08", size = 4458484, upload-time = "2025-08-05T23:58:33.526Z" }, + { url = "https://files.pythonhosted.org/packages/fd/80/1bc3634d45ddfed0871bfba52cf8f1ad724761662a0c792b97a951fb1b30/cryptography-45.0.6-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:275ba5cc0d9e320cd70f8e7b96d9e59903c815ca579ab96c1e37278d231fc402", size = 4210281, upload-time = "2025-08-05T23:58:35.445Z" }, + { url = "https://files.pythonhosted.org/packages/7d/fe/ffb12c2d83d0ee625f124880a1f023b5878f79da92e64c37962bbbe35f3f/cryptography-45.0.6-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:f4028f29a9f38a2025abedb2e409973709c660d44319c61762202206ed577c42", size = 4456890, upload-time = "2025-08-05T23:58:36.923Z" }, + { url = "https://files.pythonhosted.org/packages/8c/8e/b3f3fe0dc82c77a0deb5f493b23311e09193f2268b77196ec0f7a36e3f3e/cryptography-45.0.6-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ee411a1b977f40bd075392c80c10b58025ee5c6b47a822a33c1198598a7a5f05", size = 4333247, upload-time = "2025-08-05T23:58:38.781Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a6/c3ef2ab9e334da27a1d7b56af4a2417d77e7806b2e0f90d6267ce120d2e4/cryptography-45.0.6-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e2a21a8eda2d86bb604934b6b37691585bd095c1f788530c1fcefc53a82b3453", size = 4565045, upload-time = "2025-08-05T23:58:40.415Z" }, + { url = "https://files.pythonhosted.org/packages/31/c3/77722446b13fa71dddd820a5faab4ce6db49e7e0bf8312ef4192a3f78e2f/cryptography-45.0.6-cp311-abi3-win32.whl", hash = "sha256:d063341378d7ee9c91f9d23b431a3502fc8bfacd54ef0a27baa72a0843b29159", size = 2928923, upload-time = "2025-08-05T23:58:41.919Z" }, + { url = "https://files.pythonhosted.org/packages/38/63/a025c3225188a811b82932a4dcc8457a26c3729d81578ccecbcce2cb784e/cryptography-45.0.6-cp311-abi3-win_amd64.whl", hash = "sha256:833dc32dfc1e39b7376a87b9a6a4288a10aae234631268486558920029b086ec", size = 3403805, upload-time = "2025-08-05T23:58:43.792Z" }, + { url = "https://files.pythonhosted.org/packages/5b/af/bcfbea93a30809f126d51c074ee0fac5bd9d57d068edf56c2a73abedbea4/cryptography-45.0.6-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:3436128a60a5e5490603ab2adbabc8763613f638513ffa7d311c900a8349a2a0", size = 7020111, upload-time = "2025-08-05T23:58:45.316Z" }, + { url = "https://files.pythonhosted.org/packages/98/c6/ea5173689e014f1a8470899cd5beeb358e22bb3cf5a876060f9d1ca78af4/cryptography-45.0.6-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0d9ef57b6768d9fa58e92f4947cea96ade1233c0e236db22ba44748ffedca394", size = 4198169, upload-time = "2025-08-05T23:58:47.121Z" }, + { url = "https://files.pythonhosted.org/packages/ba/73/b12995edc0c7e2311ffb57ebd3b351f6b268fed37d93bfc6f9856e01c473/cryptography-45.0.6-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea3c42f2016a5bbf71825537c2ad753f2870191134933196bee408aac397b3d9", size = 4421273, upload-time = "2025-08-05T23:58:48.557Z" }, + { url = "https://files.pythonhosted.org/packages/f7/6e/286894f6f71926bc0da67408c853dd9ba953f662dcb70993a59fd499f111/cryptography-45.0.6-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:20ae4906a13716139d6d762ceb3e0e7e110f7955f3bc3876e3a07f5daadec5f3", size = 4199211, upload-time = "2025-08-05T23:58:50.139Z" }, + { url = "https://files.pythonhosted.org/packages/de/34/a7f55e39b9623c5cb571d77a6a90387fe557908ffc44f6872f26ca8ae270/cryptography-45.0.6-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dac5ec199038b8e131365e2324c03d20e97fe214af051d20c49db129844e8b3", size = 3883732, upload-time = "2025-08-05T23:58:52.253Z" }, + { url = "https://files.pythonhosted.org/packages/f9/b9/c6d32edbcba0cd9f5df90f29ed46a65c4631c4fbe11187feb9169c6ff506/cryptography-45.0.6-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:18f878a34b90d688982e43f4b700408b478102dd58b3e39de21b5ebf6509c301", size = 4450655, upload-time = "2025-08-05T23:58:53.848Z" }, + { url = "https://files.pythonhosted.org/packages/77/2d/09b097adfdee0227cfd4c699b3375a842080f065bab9014248933497c3f9/cryptography-45.0.6-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5bd6020c80c5b2b2242d6c48487d7b85700f5e0038e67b29d706f98440d66eb5", size = 4198956, upload-time = "2025-08-05T23:58:55.209Z" }, + { url = "https://files.pythonhosted.org/packages/55/66/061ec6689207d54effdff535bbdf85cc380d32dd5377173085812565cf38/cryptography-45.0.6-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:eccddbd986e43014263eda489abbddfbc287af5cddfd690477993dbb31e31016", size = 4449859, upload-time = "2025-08-05T23:58:56.639Z" }, + { url = "https://files.pythonhosted.org/packages/41/ff/e7d5a2ad2d035e5a2af116e1a3adb4d8fcd0be92a18032917a089c6e5028/cryptography-45.0.6-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:550ae02148206beb722cfe4ef0933f9352bab26b087af00e48fdfb9ade35c5b3", size = 4320254, upload-time = "2025-08-05T23:58:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/82/27/092d311af22095d288f4db89fcaebadfb2f28944f3d790a4cf51fe5ddaeb/cryptography-45.0.6-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5b64e668fc3528e77efa51ca70fadcd6610e8ab231e3e06ae2bab3b31c2b8ed9", size = 4554815, upload-time = "2025-08-05T23:59:00.283Z" }, + { url = "https://files.pythonhosted.org/packages/7e/01/aa2f4940262d588a8fdf4edabe4cda45854d00ebc6eaac12568b3a491a16/cryptography-45.0.6-cp37-abi3-win32.whl", hash = "sha256:780c40fb751c7d2b0c6786ceee6b6f871e86e8718a8ff4bc35073ac353c7cd02", size = 2912147, upload-time = "2025-08-05T23:59:01.716Z" }, + { url = "https://files.pythonhosted.org/packages/0a/bc/16e0276078c2de3ceef6b5a34b965f4436215efac45313df90d55f0ba2d2/cryptography-45.0.6-cp37-abi3-win_amd64.whl", hash = "sha256:20d15aed3ee522faac1a39fbfdfee25d17b1284bafd808e1640a74846d7c4d1b", size = 3390459, upload-time = "2025-08-05T23:59:03.358Z" }, + { url = "https://files.pythonhosted.org/packages/61/69/c252de4ec047ba2f567ecb53149410219577d408c2aea9c989acae7eafce/cryptography-45.0.6-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:fc022c1fa5acff6def2fc6d7819bbbd31ccddfe67d075331a65d9cfb28a20983", size = 3584669, upload-time = "2025-08-05T23:59:15.431Z" }, + { url = "https://files.pythonhosted.org/packages/e3/fe/deea71e9f310a31fe0a6bfee670955152128d309ea2d1c79e2a5ae0f0401/cryptography-45.0.6-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:3de77e4df42ac8d4e4d6cdb342d989803ad37707cf8f3fbf7b088c9cbdd46427", size = 4153022, upload-time = "2025-08-05T23:59:16.954Z" }, + { url = "https://files.pythonhosted.org/packages/60/45/a77452f5e49cb580feedba6606d66ae7b82c128947aa754533b3d1bd44b0/cryptography-45.0.6-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:599c8d7df950aa68baa7e98f7b73f4f414c9f02d0e8104a30c0182a07732638b", size = 4386802, upload-time = "2025-08-05T23:59:18.55Z" }, + { url = "https://files.pythonhosted.org/packages/a3/b9/a2f747d2acd5e3075fdf5c145c7c3568895daaa38b3b0c960ef830db6cdc/cryptography-45.0.6-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:31a2b9a10530a1cb04ffd6aa1cd4d3be9ed49f7d77a4dafe198f3b382f41545c", size = 4152706, upload-time = "2025-08-05T23:59:20.044Z" }, + { url = "https://files.pythonhosted.org/packages/81/ec/381b3e8d0685a3f3f304a382aa3dfce36af2d76467da0fd4bb21ddccc7b2/cryptography-45.0.6-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:e5b3dda1b00fb41da3af4c5ef3f922a200e33ee5ba0f0bc9ecf0b0c173958385", size = 4386740, upload-time = "2025-08-05T23:59:21.525Z" }, + { url = "https://files.pythonhosted.org/packages/0a/76/cf8d69da8d0b5ecb0db406f24a63a3f69ba5e791a11b782aeeefef27ccbb/cryptography-45.0.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:629127cfdcdc6806dfe234734d7cb8ac54edaf572148274fa377a7d3405b0043", size = 3331874, upload-time = "2025-08-05T23:59:23.017Z" }, +] + +[[package]] +name = "csaf-tool" +version = "0.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packageurl-python" }, + { name = "rich" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/24/b408082ef806581de3a2094caf0b9ed560a4fb8e8c14bb9c28462ecb023c/csaf_tool-0.3.2-py2.py3-none-any.whl", hash = "sha256:7e5559cb522eb76e3acad39a7bf9ba1b81e5a6224099d511a4c9c2dcf36caa16", size = 17629, upload-time = "2024-06-12T20:10:06.429Z" }, +] + +[[package]] +name = "cvss" +version = "3.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a5/f6/1f7d315de23f39bbc32b94bb6b33a1b6124856037bfaa3f8bdb1a49584fa/cvss-3.6.tar.gz", hash = "sha256:f21d18224efcd3c01b44ff1b37dec2e3208d29a6d0ce6c87a599c73c21ee1a99", size = 30047, upload-time = "2025-08-04T10:50:13.753Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/6d/fe2c65b94a28ae0481dc254e8cd664b82390069003bea945076d8a445f2b/cvss-3.6-py2.py3-none-any.whl", hash = "sha256:e342c6ad9c7eb69d2aebbbc2768a03cabd57eb947c806e145de5b936219833ea", size = 31154, upload-time = "2025-08-04T10:50:12.328Z" }, ] [[package]] @@ -715,6 +829,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, ] +[[package]] +name = "cymem" +version = "2.0.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/4a/1acd761fb6ac4c560e823ce40536a62f886f2d59b2763b5c3fc7e9d92101/cymem-2.0.11.tar.gz", hash = "sha256:efe49a349d4a518be6b6c6b255d4a80f740a341544bde1a807707c058b88d0bd", size = 10346, upload-time = "2025-01-16T21:50:41.045Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/e3/d98e3976f4ffa99cddebc1ce379d4d62e3eb1da22285267f902c99cc3395/cymem-2.0.11-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3ee54039aad3ef65de82d66c40516bf54586287b46d32c91ea0530c34e8a2745", size = 42005, upload-time = "2025-01-16T21:49:34.977Z" }, + { url = "https://files.pythonhosted.org/packages/41/b4/7546faf2ab63e59befc95972316d62276cec153f7d4d60e7b0d5e08f0602/cymem-2.0.11-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4c05ef75b5db217be820604e43a47ccbbafea98ab6659d07cea92fa3c864ea58", size = 41747, upload-time = "2025-01-16T21:49:36.108Z" }, + { url = "https://files.pythonhosted.org/packages/7d/4e/042f372e5b3eb7f5f3dd7677161771d301de2b6fa3f7c74e1cebcd502552/cymem-2.0.11-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8d5381e5793ce531bac0dbc00829c8381f18605bb67e4b61d34f8850463da40", size = 217647, upload-time = "2025-01-16T21:49:37.433Z" }, + { url = "https://files.pythonhosted.org/packages/48/cb/2207679e4b92701f78cf141e1ab4f81f55247dbe154eb426b842a0a993de/cymem-2.0.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2b9d3f42d7249ac81802135cad51d707def058001a32f73fc7fbf3de7045ac7", size = 218857, upload-time = "2025-01-16T21:49:40.09Z" }, + { url = "https://files.pythonhosted.org/packages/31/7a/76ae3b7a39ab2531029d281e43fcfcaad728c2341b150a81a3a1f5587cf3/cymem-2.0.11-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:39b78f2195d20b75c2d465732f6b8e8721c5d4eb012777c2cb89bdb45a043185", size = 206148, upload-time = "2025-01-16T21:49:41.383Z" }, + { url = "https://files.pythonhosted.org/packages/25/f9/d0fc0191ac79f15638ddb59237aa76f234691374d7d7950e10f384bd8a25/cymem-2.0.11-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2203bd6525a80d8fd0c94654a263af21c0387ae1d5062cceaebb652bf9bad7bc", size = 207112, upload-time = "2025-01-16T21:49:43.986Z" }, + { url = "https://files.pythonhosted.org/packages/56/c8/75f75889401b20f4c3a7c5965dda09df42913e904ddc2ffe7ef3bdf25061/cymem-2.0.11-cp311-cp311-win_amd64.whl", hash = "sha256:aa54af7314de400634448da1f935b61323da80a49484074688d344fb2036681b", size = 39360, upload-time = "2025-01-16T21:49:45.479Z" }, + { url = "https://files.pythonhosted.org/packages/71/67/0d74f7e9d79f934368a78fb1d1466b94bebdbff14f8ae94dd3e4ea8738bb/cymem-2.0.11-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a0fbe19ce653cd688842d81e5819dc63f911a26e192ef30b0b89f0ab2b192ff2", size = 42621, upload-time = "2025-01-16T21:49:46.585Z" }, + { url = "https://files.pythonhosted.org/packages/4a/d6/f7a19c63b48efc3f00a3ee8d69070ac90202e1e378f6cf81b8671f0cf762/cymem-2.0.11-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de72101dc0e6326f6a2f73e05a438d1f3c6110d41044236d0fbe62925091267d", size = 42249, upload-time = "2025-01-16T21:49:48.973Z" }, + { url = "https://files.pythonhosted.org/packages/d7/60/cdc434239813eef547fb99b6d0bafe31178501702df9b77c4108c9a216f6/cymem-2.0.11-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bee4395917f6588b8ac1699499128842768b391fe8896e8626950b4da5f9a406", size = 224758, upload-time = "2025-01-16T21:49:51.382Z" }, + { url = "https://files.pythonhosted.org/packages/1d/68/8fa6efae17cd3b2ba9a2f83b824867c5b65b06f7aec3f8a0d0cabdeffb9b/cymem-2.0.11-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b02f2b17d760dc3fe5812737b1ce4f684641cdd751d67761d333a3b5ea97b83", size = 227995, upload-time = "2025-01-16T21:49:54.538Z" }, + { url = "https://files.pythonhosted.org/packages/e4/f3/ceda70bf6447880140602285b7c6fa171cb7c78b623d35345cc32505cd06/cymem-2.0.11-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:04ee6b4041ddec24512d6e969ed6445e57917f01e73b9dabbe17b7e6b27fef05", size = 215325, upload-time = "2025-01-16T21:49:57.229Z" }, + { url = "https://files.pythonhosted.org/packages/d3/47/6915eaa521e1ce7a0ba480eecb6870cb4f681bcd64ced88c2f0ed7a744b4/cymem-2.0.11-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e1048dae7e627ee25f22c87bb670b13e06bc0aecc114b89b959a798d487d1bf4", size = 216447, upload-time = "2025-01-16T21:50:00.432Z" }, + { url = "https://files.pythonhosted.org/packages/7b/be/8e02bdd31e557f642741a06c8e886782ef78f0b00daffd681922dc9bbc88/cymem-2.0.11-cp312-cp312-win_amd64.whl", hash = "sha256:0c269c7a867d74adeb9db65fa1d226342aacf44d64b7931282f0b0eb22eb6275", size = 39283, upload-time = "2025-01-16T21:50:03.384Z" }, +] + +[[package]] +name = "dacite" +version = "1.9.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/55/a0/7ca79796e799a3e782045d29bf052b5cde7439a2bbb17f15ff44f7aacc63/dacite-1.9.2.tar.gz", hash = "sha256:6ccc3b299727c7aa17582f0021f6ae14d5de47c7227932c47fec4cdfefd26f09", size = 22420, upload-time = "2025-02-05T09:27:29.757Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/35/386550fd60316d1e37eccdda609b074113298f23cef5bddb2049823fe666/dacite-1.9.2-py3-none-any.whl", hash = "sha256:053f7c3f5128ca2e9aceb66892b1a3c8936d02c686e707bee96e19deef4bc4a0", size = 16600, upload-time = "2025-02-05T09:27:24.345Z" }, +] + [[package]] name = "dataclasses-json" version = "0.6.7" @@ -730,11 +875,11 @@ wheels = [ [[package]] name = "datasets" -version = "2.2.1" +version = "4.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "aiohttp" }, { name = "dill" }, + { name = "filelock" }, { name = "fsspec", extra = ["http"] }, { name = "huggingface-hub" }, { name = "multiprocess" }, @@ -742,23 +887,79 @@ dependencies = [ { name = "packaging" }, { name = "pandas" }, { name = "pyarrow" }, + { name = "pyyaml" }, { name = "requests" }, - { name = "responses" }, { name = "tqdm" }, { name = "xxhash" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/31/64/1e6fb2a0eb6b0d55117233cf33279ba6d680c0f031ebae81281a47c92760/datasets-2.2.1.tar.gz", hash = "sha256:d362717c4394589b516c8f397ff20a6fe720454aed877ab61d06f3bc05df9544", size = 302132, upload-time = "2022-05-11T17:02:29.543Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/9d/348ed92110ba5f9b70b51ca1078d4809767a835aa2b7ce7e74ad2b98323d/datasets-4.0.0.tar.gz", hash = "sha256:9657e7140a9050db13443ba21cb5de185af8af944479b00e7ff1e00a61c8dbf1", size = 569566, upload-time = "2025-07-09T14:35:52.431Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/62/eb8157afb21bd229c864521c1ab4fa8e9b4f1b06bafdd8c4668a7a31b5dd/datasets-4.0.0-py3-none-any.whl", hash = "sha256:7ef95e62025fd122882dbce6cb904c8cd3fbc829de6669a5eb939c77d50e203d", size = 494825, upload-time = "2025-07-09T14:35:50.658Z" }, +] + +[[package]] +name = "debugpy" +version = "1.8.16" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/d4/722d0bcc7986172ac2ef3c979ad56a1030e3afd44ced136d45f8142b1f4a/debugpy-1.8.16.tar.gz", hash = "sha256:31e69a1feb1cf6b51efbed3f6c9b0ef03bc46ff050679c4be7ea6d2e23540870", size = 1643809, upload-time = "2025-08-06T18:00:02.647Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/d6/ad70ba8b49b23fa286fb21081cf732232cc19374af362051da9c7537ae52/debugpy-1.8.16-cp311-cp311-macosx_14_0_universal2.whl", hash = "sha256:67371b28b79a6a12bcc027d94a06158f2fde223e35b5c4e0783b6f9d3b39274a", size = 2184063, upload-time = "2025-08-06T18:00:11.885Z" }, + { url = "https://files.pythonhosted.org/packages/aa/49/7b03e88dea9759a4c7910143f87f92beb494daaae25560184ff4ae883f9e/debugpy-1.8.16-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2abae6dd02523bec2dee16bd6b0781cccb53fd4995e5c71cc659b5f45581898", size = 3134837, upload-time = "2025-08-06T18:00:13.782Z" }, + { url = "https://files.pythonhosted.org/packages/5d/52/b348930316921de7565fbe37a487d15409041713004f3d74d03eb077dbd4/debugpy-1.8.16-cp311-cp311-win32.whl", hash = "sha256:f8340a3ac2ed4f5da59e064aa92e39edd52729a88fbde7bbaa54e08249a04493", size = 5159142, upload-time = "2025-08-06T18:00:15.391Z" }, + { url = "https://files.pythonhosted.org/packages/d8/ef/9aa9549ce1e10cea696d980292e71672a91ee4a6a691ce5f8629e8f48c49/debugpy-1.8.16-cp311-cp311-win_amd64.whl", hash = "sha256:70f5fcd6d4d0c150a878d2aa37391c52de788c3dc680b97bdb5e529cb80df87a", size = 5183117, upload-time = "2025-08-06T18:00:17.251Z" }, + { url = "https://files.pythonhosted.org/packages/61/fb/0387c0e108d842c902801bc65ccc53e5b91d8c169702a9bbf4f7efcedf0c/debugpy-1.8.16-cp312-cp312-macosx_14_0_universal2.whl", hash = "sha256:b202e2843e32e80b3b584bcebfe0e65e0392920dc70df11b2bfe1afcb7a085e4", size = 2511822, upload-time = "2025-08-06T18:00:18.526Z" }, + { url = "https://files.pythonhosted.org/packages/37/44/19e02745cae22bf96440141f94e15a69a1afaa3a64ddfc38004668fcdebf/debugpy-1.8.16-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64473c4a306ba11a99fe0bb14622ba4fbd943eb004847d9b69b107bde45aa9ea", size = 4230135, upload-time = "2025-08-06T18:00:19.997Z" }, + { url = "https://files.pythonhosted.org/packages/f3/0b/19b1ba5ee4412f303475a2c7ad5858efb99c90eae5ec627aa6275c439957/debugpy-1.8.16-cp312-cp312-win32.whl", hash = "sha256:833a61ed446426e38b0dd8be3e9d45ae285d424f5bf6cd5b2b559c8f12305508", size = 5281271, upload-time = "2025-08-06T18:00:21.281Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e0/bc62e2dc141de53bd03e2c7cb9d7011de2e65e8bdcdaa26703e4d28656ba/debugpy-1.8.16-cp312-cp312-win_amd64.whl", hash = "sha256:75f204684581e9ef3dc2f67687c3c8c183fde2d6675ab131d94084baf8084121", size = 5323149, upload-time = "2025-08-06T18:00:23.033Z" }, + { url = "https://files.pythonhosted.org/packages/52/57/ecc9ae29fa5b2d90107cd1d9bf8ed19aacb74b2264d986ae9d44fe9bdf87/debugpy-1.8.16-py2.py3-none-any.whl", hash = "sha256:19c9521962475b87da6f673514f7fd610328757ec993bf7ec0d8c96f9a325f9e", size = 5287700, upload-time = "2025-08-06T18:00:42.333Z" }, +] + +[[package]] +name = "decorator" +version = "5.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/fa/6d96a0978d19e17b68d634497769987b16c8f4cd0a7a05048bec693caa6b/decorator-5.2.1.tar.gz", hash = "sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360", size = 56711, upload-time = "2025-02-24T04:41:34.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a", size = 9190, upload-time = "2025-02-24T04:41:32.565Z" }, +] + +[[package]] +name = "defusedxml" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, +] + +[[package]] +name = "deprecated" +version = "1.2.18" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/97/06afe62762c9a8a86af0cfb7bfdab22a43ad17138b07af5b1a58442690a2/deprecated-1.2.18.tar.gz", hash = "sha256:422b6f6d859da6f2ef57857761bfb392480502a64c3028ca9bbe86085d72115d", size = 2928744, upload-time = "2025-01-27T10:46:25.7Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/2d/41e8aec8d4bad6f07adfcbc89cf743e0d31c876371d453b2936bcfa7fe34/datasets-2.2.1-py3-none-any.whl", hash = "sha256:1938f3e99599422de50b9b54fe802aca854ed130382dab0b3820c821f7ae6d5e", size = 342193, upload-time = "2022-05-11T17:02:27.047Z" }, + { url = "https://files.pythonhosted.org/packages/6e/c6/ac0b6c1e2d138f1002bcf799d330bd6d85084fece321e662a14223794041/Deprecated-1.2.18-py2.py3-none-any.whl", hash = "sha256:bd5011788200372a32418f888e326a09ff80d0214bd961147cfed01b5c018eec", size = 9998, upload-time = "2025-01-27T10:46:09.186Z" }, ] [[package]] name = "dill" -version = "0.4.0" +version = "0.3.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/17/4d/ac7ffa80c69ea1df30a8aa11b3578692a5118e7cd1aa157e3ef73b092d15/dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca", size = 184847, upload-time = "2024-01-27T23:42:16.145Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/7a/cef76fd8438a42f96db64ddaa85280485a9c395e7df3db8158cfec1eee34/dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7", size = 116252, upload-time = "2024-01-27T23:42:14.239Z" }, +] + +[[package]] +name = "dirtyjson" +version = "1.0.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/12/80/630b4b88364e9a8c8c5797f4602d0f76ef820909ee32f0bacb9f90654042/dill-0.4.0.tar.gz", hash = "sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0", size = 186976, upload-time = "2025-04-16T00:41:48.867Z" } +sdist = { url = "https://files.pythonhosted.org/packages/db/04/d24f6e645ad82ba0ef092fa17d9ef7a21953781663648a01c9371d9e8e98/dirtyjson-1.0.8.tar.gz", hash = "sha256:90ca4a18f3ff30ce849d100dcf4a003953c79d3a2348ef056f1d9c22231a25fd", size = 30782, upload-time = "2022-11-28T23:32:33.319Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/50/3d/9373ad9c56321fdab5b41197068e1d8c25883b3fea29dd361f9b55116869/dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049", size = 119668, upload-time = "2025-04-16T00:41:47.671Z" }, + { url = "https://files.pythonhosted.org/packages/68/69/1bcf70f81de1b4a9f21b3a62ec0c83bdff991c88d6cc2267d02408457e88/dirtyjson-1.0.8-py3-none-any.whl", hash = "sha256:125e27248435a58acace26d5c2c4c11a1c0de0a9c5124c5a94ba78e517d74f53", size = 25197, upload-time = "2022-11-28T23:32:31.219Z" }, ] [[package]] @@ -800,6 +1001,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, ] +[[package]] +name = "eval-type-backport" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/30/ea/8b0ac4469d4c347c6a385ff09dc3c048c2d021696664e26c7ee6791631b5/eval_type_backport-0.2.2.tar.gz", hash = "sha256:f0576b4cf01ebb5bd358d02314d31846af5e07678387486e2c798af0e7d849c1", size = 9079, upload-time = "2024-12-21T20:09:46.005Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/31/55cd413eaccd39125368be33c46de24a1f639f2e12349b0361b4678f3915/eval_type_backport-0.2.2-py3-none-any.whl", hash = "sha256:cb6ad7c393517f476f96d456d0412ea80f0a8cf96f6892834cd9340149111b0a", size = 5830, upload-time = "2024-12-21T20:09:44.175Z" }, +] + +[[package]] +name = "executing" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/50/a9d80c47ff289c611ff12e63f7c5d13942c65d68125160cefd768c73e6e4/executing-2.2.0.tar.gz", hash = "sha256:5d108c028108fe2551d1a7b2e8b713341e2cb4fc0aa7dcf966fa4327a5226755", size = 978693, upload-time = "2025-01-22T15:41:29.403Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/8f/c4d9bafc34ad7ad5d8dc16dd1347ee0e507a52c3adb6bfa8887e1c6a26ba/executing-2.2.0-py2.py3-none-any.whl", hash = "sha256:11387150cad388d62750327a53d3339fad4888b39a6fe233c3afbb54ecffd3aa", size = 26702, upload-time = "2025-01-22T15:41:25.929Z" }, +] + [[package]] name = "expandvars" version = "1.1.1" @@ -809,6 +1028,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2b/ca/0753ba3a81255ac49748ec8b665ab01f8efcf711f74bbccb5457a6193acc/expandvars-1.1.1-py3-none-any.whl", hash = "sha256:09ca39e6bfcb0d899db8778a00dd3d89cfeb0080795c54f16f6279afd0ef8c5b", size = 7522, upload-time = "2025-07-12T07:46:18.984Z" }, ] +[[package]] +name = "exploit-iq-commons" +version = "0.1.0" +source = { editable = "src/exploit_iq_commons" } +dependencies = [ + { name = "esprima" }, + { name = "gitpython" }, + { name = "langchain" }, + { name = "langchain-community" }, + { name = "pandas" }, + { name = "pydantic" }, + { name = "tqdm" }, + { name = "tree-sitter" }, + { name = "tree-sitter-languages" }, +] + +[package.metadata] +requires-dist = [ + { name = "esprima", specifier = "==4.0.1" }, + { name = "gitpython", specifier = "==3.1.44" }, + { name = "langchain", specifier = "~=0.3.0" }, + { name = "langchain-community", specifier = "~=0.3.0" }, + { name = "pandas", specifier = "==2.3.1" }, + { name = "pydantic", specifier = ">=2.0" }, + { name = "tqdm", specifier = "==4.67.1" }, + { name = "tree-sitter", specifier = "==0.21.3" }, + { name = "tree-sitter-languages", specifier = "==1.10.2" }, +] + [[package]] name = "extratools" version = "0.8.2.1" @@ -827,7 +1075,6 @@ dependencies = [ { name = "numpy" }, { name = "packaging" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/92/57/f5af8d68f9a8ec943496aa8fa9b01d7c4e654e918b246f5a0e6c85df6e0d/faiss_cpu-1.9.0.tar.gz", hash = "sha256:587fcea9fa478e9307a388754824a032849d317894a607586c3cdd8c8aeb7233", size = 67785, upload-time = "2024-10-08T07:07:00.355Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f5/a2/b346d976c0adcdeab7d23bf6f58eef6d1c5c9c0bf919353923cf91553049/faiss_cpu-1.9.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:b0e9208a36da519dc2eb90e4c44c66a6812a5b68457582d8ed21d04e910e3d1f", size = 7661789, upload-time = "2024-10-08T07:06:26.259Z" }, { url = "https://files.pythonhosted.org/packages/6b/9f/f0a39439a938818f1add48bd7b79d4b8e12e60f2f0c1e4a0b37b295625e0/faiss_cpu-1.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6a4b2871057560020b83ad7bb5aaf3b97b64f980f9af2ca99ba34eeb4fe38bdf", size = 3215612, upload-time = "2024-10-08T07:06:28.359Z" }, @@ -857,20 +1104,32 @@ wheels = [ [[package]] name = "fastjsonschema" -version = "2.21.1" +version = "2.21.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/b5/23b216d9d985a956623b6bd12d4086b60f0059b27799f23016af04a74ea1/fastjsonschema-2.21.2.tar.gz", hash = "sha256:b1eb43748041c880796cd077f1a07c3d94e93ae84bba5ed36800a33554ae05de", size = 374130, upload-time = "2025-08-14T18:49:36.666Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/a8/20d0723294217e47de6d9e2e40fd4a9d2f7c4b6ef974babd482a59743694/fastjsonschema-2.21.2-py3-none-any.whl", hash = "sha256:1c797122d0a86c5cace2e54bf4e819c36223b552017172f32c5c024a6b77e463", size = 24024, upload-time = "2025-08-14T18:49:34.776Z" }, +] + +[[package]] +name = "fickling" +version = "0.1.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8b/50/4b769ce1ac4071a1ef6d86b1a3fb56cdc3a37615e8c5519e1af96cdac366/fastjsonschema-2.21.1.tar.gz", hash = "sha256:794d4f0a58f848961ba16af7b9c85a3e88cd360df008c59aac6fc5ae9323b5d4", size = 373939, upload-time = "2024-12-02T10:55:15.133Z" } +dependencies = [ + { name = "stdlib-list" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/23/0a03d2d01c004ab3f0181bbda3642c7d88226b4a25f47675ef948326504f/fickling-0.1.4.tar.gz", hash = "sha256:cb06bbb7b6a1c443eacf230ab7e212d8b4f3bb2333f307a8c94a144537018888", size = 40956, upload-time = "2025-07-07T13:17:59.572Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/90/2b/0817a2b257fe88725c25589d89aec060581aabf668707a8d03b2e9e0cb2a/fastjsonschema-2.21.1-py3-none-any.whl", hash = "sha256:c9e5b7e908310918cf494a434eeb31384dd84a98b57a30bcb1f535015b554667", size = 23924, upload-time = "2024-12-02T10:55:07.599Z" }, + { url = "https://files.pythonhosted.org/packages/38/40/059cd7c6913cc20b029dd5c8f38578d185f71737c5a62387df4928cd10fe/fickling-0.1.4-py3-none-any.whl", hash = "sha256:110522385a30b7936c50c3860ba42b0605254df9d0ef6cbdaf0ad8fb455a6672", size = 42573, upload-time = "2025-07-07T13:17:58.071Z" }, ] [[package]] name = "filelock" -version = "3.18.0" +version = "3.19.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0a/10/c23352565a6544bdc5353e0b15fc1c563352101f30e24bf500207a54df9a/filelock-3.18.0.tar.gz", hash = "sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2", size = 18075, upload-time = "2025-03-14T07:11:40.47Z" } +sdist = { url = "https://files.pythonhosted.org/packages/40/bb/0ab3e58d22305b6f5440629d20683af28959bf793d98d11950e305c1c326/filelock-3.19.1.tar.gz", hash = "sha256:66eda1888b0171c998b35be2bcc0f6d75c388a7ce20c3f3f37aa8e96c2dddf58", size = 17687, upload-time = "2025-08-14T16:56:03.016Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/36/2a115987e2d8c300a974597416d9de88f2444426de9571f4b59b2cca3acc/filelock-3.18.0-py3-none-any.whl", hash = "sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de", size = 16215, upload-time = "2025-03-14T07:11:39.145Z" }, + { url = "https://files.pythonhosted.org/packages/42/14/42b2651a2f46b022ccd948bca9f2d5af0fd8929c4eec235b8d6d844fbe67/filelock-3.19.1-py3-none-any.whl", hash = "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d", size = 15988, upload-time = "2025-08-14T16:56:01.633Z" }, ] [[package]] @@ -909,27 +1168,27 @@ wheels = [ [[package]] name = "fonttools" -version = "4.59.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8a/27/ec3c723bfdf86f34c5c82bf6305df3e0f0d8ea798d2d3a7cb0c0a866d286/fonttools-4.59.0.tar.gz", hash = "sha256:be392ec3529e2f57faa28709d60723a763904f71a2b63aabe14fee6648fe3b14", size = 3532521, upload-time = "2025-07-16T12:04:54.613Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/96/520733d9602fa1bf6592e5354c6721ac6fc9ea72bc98d112d0c38b967199/fonttools-4.59.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:841b2186adce48903c0fef235421ae21549020eca942c1da773ac380b056ab3c", size = 2782387, upload-time = "2025-07-16T12:03:51.424Z" }, - { url = "https://files.pythonhosted.org/packages/87/6a/170fce30b9bce69077d8eec9bea2cfd9f7995e8911c71be905e2eba6368b/fonttools-4.59.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9bcc1e77fbd1609198966ded6b2a9897bd6c6bcbd2287a2fc7d75f1a254179c5", size = 2342194, upload-time = "2025-07-16T12:03:53.295Z" }, - { url = "https://files.pythonhosted.org/packages/b0/b6/7c8166c0066856f1408092f7968ac744060cf72ca53aec9036106f57eeca/fonttools-4.59.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:37c377f7cb2ab2eca8a0b319c68146d34a339792f9420fca6cd49cf28d370705", size = 5032333, upload-time = "2025-07-16T12:03:55.177Z" }, - { url = "https://files.pythonhosted.org/packages/eb/0c/707c5a19598eafcafd489b73c4cb1c142102d6197e872f531512d084aa76/fonttools-4.59.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa39475eaccb98f9199eccfda4298abaf35ae0caec676ffc25b3a5e224044464", size = 4974422, upload-time = "2025-07-16T12:03:57.406Z" }, - { url = "https://files.pythonhosted.org/packages/f6/e7/6d33737d9fe632a0f59289b6f9743a86d2a9d0673de2a0c38c0f54729822/fonttools-4.59.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d3972b13148c1d1fbc092b27678a33b3080d1ac0ca305742b0119b75f9e87e38", size = 5010631, upload-time = "2025-07-16T12:03:59.449Z" }, - { url = "https://files.pythonhosted.org/packages/63/e1/a4c3d089ab034a578820c8f2dff21ef60daf9668034a1e4fb38bb1cc3398/fonttools-4.59.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a408c3c51358c89b29cfa5317cf11518b7ce5de1717abb55c5ae2d2921027de6", size = 5122198, upload-time = "2025-07-16T12:04:01.542Z" }, - { url = "https://files.pythonhosted.org/packages/09/77/ca82b9c12fa4de3c520b7760ee61787640cf3fde55ef1b0bfe1de38c8153/fonttools-4.59.0-cp311-cp311-win32.whl", hash = "sha256:6770d7da00f358183d8fd5c4615436189e4f683bdb6affb02cad3d221d7bb757", size = 2214216, upload-time = "2025-07-16T12:04:03.515Z" }, - { url = "https://files.pythonhosted.org/packages/ab/25/5aa7ca24b560b2f00f260acf32c4cf29d7aaf8656e159a336111c18bc345/fonttools-4.59.0-cp311-cp311-win_amd64.whl", hash = "sha256:84fc186980231a287b28560d3123bd255d3c6b6659828c642b4cf961e2b923d0", size = 2261879, upload-time = "2025-07-16T12:04:05.015Z" }, - { url = "https://files.pythonhosted.org/packages/e2/77/b1c8af22f4265e951cd2e5535dbef8859efcef4fb8dee742d368c967cddb/fonttools-4.59.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:f9b3a78f69dcbd803cf2fb3f972779875b244c1115481dfbdd567b2c22b31f6b", size = 2767562, upload-time = "2025-07-16T12:04:06.895Z" }, - { url = "https://files.pythonhosted.org/packages/ff/5a/aeb975699588176bb357e8b398dfd27e5d3a2230d92b81ab8cbb6187358d/fonttools-4.59.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:57bb7e26928573ee7c6504f54c05860d867fd35e675769f3ce01b52af38d48e2", size = 2335168, upload-time = "2025-07-16T12:04:08.695Z" }, - { url = "https://files.pythonhosted.org/packages/54/97/c6101a7e60ae138c4ef75b22434373a0da50a707dad523dd19a4889315bf/fonttools-4.59.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4536f2695fe5c1ffb528d84a35a7d3967e5558d2af58b4775e7ab1449d65767b", size = 4909850, upload-time = "2025-07-16T12:04:10.761Z" }, - { url = "https://files.pythonhosted.org/packages/bd/6c/fa4d18d641054f7bff878cbea14aa9433f292b9057cb1700d8e91a4d5f4f/fonttools-4.59.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:885bde7d26e5b40e15c47bd5def48b38cbd50830a65f98122a8fb90962af7cd1", size = 4955131, upload-time = "2025-07-16T12:04:12.846Z" }, - { url = "https://files.pythonhosted.org/packages/20/5c/331947fc1377deb928a69bde49f9003364f5115e5cbe351eea99e39412a2/fonttools-4.59.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6801aeddb6acb2c42eafa45bc1cb98ba236871ae6f33f31e984670b749a8e58e", size = 4899667, upload-time = "2025-07-16T12:04:14.558Z" }, - { url = "https://files.pythonhosted.org/packages/8a/46/b66469dfa26b8ff0baa7654b2cc7851206c6d57fe3abdabbaab22079a119/fonttools-4.59.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:31003b6a10f70742a63126b80863ab48175fb8272a18ca0846c0482968f0588e", size = 5051349, upload-time = "2025-07-16T12:04:16.388Z" }, - { url = "https://files.pythonhosted.org/packages/2e/05/ebfb6b1f3a4328ab69787d106a7d92ccde77ce66e98659df0f9e3f28d93d/fonttools-4.59.0-cp312-cp312-win32.whl", hash = "sha256:fbce6dae41b692a5973d0f2158f782b9ad05babc2c2019a970a1094a23909b1b", size = 2201315, upload-time = "2025-07-16T12:04:18.557Z" }, - { url = "https://files.pythonhosted.org/packages/09/45/d2bdc9ea20bbadec1016fd0db45696d573d7a26d95ab5174ffcb6d74340b/fonttools-4.59.0-cp312-cp312-win_amd64.whl", hash = "sha256:332bfe685d1ac58ca8d62b8d6c71c2e52a6c64bc218dc8f7825c9ea51385aa01", size = 2249408, upload-time = "2025-07-16T12:04:20.489Z" }, - { url = "https://files.pythonhosted.org/packages/d0/9c/df0ef2c51845a13043e5088f7bb988ca6cd5bb82d5d4203d6a158aa58cf2/fonttools-4.59.0-py3-none-any.whl", hash = "sha256:241313683afd3baacb32a6bd124d0bce7404bc5280e12e291bae1b9bba28711d", size = 1128050, upload-time = "2025-07-16T12:04:52.687Z" }, +version = "4.59.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/11/7f/29c9c3fe4246f6ad96fee52b88d0dc3a863c7563b0afc959e36d78b965dc/fonttools-4.59.1.tar.gz", hash = "sha256:74995b402ad09822a4c8002438e54940d9f1ecda898d2bb057729d7da983e4cb", size = 3534394, upload-time = "2025-08-14T16:28:14.266Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/62/9667599561f623d4a523cc9eb4f66f3b94b6155464110fa9aebbf90bbec7/fonttools-4.59.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4909cce2e35706f3d18c54d3dcce0414ba5e0fb436a454dffec459c61653b513", size = 2778815, upload-time = "2025-08-14T16:26:28.484Z" }, + { url = "https://files.pythonhosted.org/packages/8f/78/cc25bcb2ce86033a9df243418d175e58f1956a35047c685ef553acae67d6/fonttools-4.59.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:efbec204fa9f877641747f2d9612b2b656071390d7a7ef07a9dbf0ecf9c7195c", size = 2341631, upload-time = "2025-08-14T16:26:30.396Z" }, + { url = "https://files.pythonhosted.org/packages/a4/cc/fcbb606dd6871f457ac32f281c20bcd6cc77d9fce77b5a4e2b2afab1f500/fonttools-4.59.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:39dfd42cc2dc647b2c5469bc7a5b234d9a49e72565b96dd14ae6f11c2c59ef15", size = 5022222, upload-time = "2025-08-14T16:26:32.447Z" }, + { url = "https://files.pythonhosted.org/packages/61/96/c0b1cf2b74d08eb616a80dbf5564351fe4686147291a25f7dce8ace51eb3/fonttools-4.59.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b11bc177a0d428b37890825d7d025040d591aa833f85f8d8878ed183354f47df", size = 4966512, upload-time = "2025-08-14T16:26:34.621Z" }, + { url = "https://files.pythonhosted.org/packages/a4/26/51ce2e3e0835ffc2562b1b11d1fb9dafd0aca89c9041b64a9e903790a761/fonttools-4.59.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5b9b4c35b3be45e5bc774d3fc9608bbf4f9a8d371103b858c80edbeed31dd5aa", size = 5001645, upload-time = "2025-08-14T16:26:36.876Z" }, + { url = "https://files.pythonhosted.org/packages/36/11/ef0b23f4266349b6d5ccbd1a07b7adc998d5bce925792aa5d1ec33f593e3/fonttools-4.59.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:01158376b8a418a0bae9625c476cebfcfcb5e6761e9d243b219cd58341e7afbb", size = 5113777, upload-time = "2025-08-14T16:26:39.002Z" }, + { url = "https://files.pythonhosted.org/packages/d0/da/b398fe61ef433da0a0472cdb5d4399124f7581ffe1a31b6242c91477d802/fonttools-4.59.1-cp311-cp311-win32.whl", hash = "sha256:cf7c5089d37787387123f1cb8f1793a47c5e1e3d1e4e7bfbc1cc96e0f925eabe", size = 2215076, upload-time = "2025-08-14T16:26:41.196Z" }, + { url = "https://files.pythonhosted.org/packages/94/bd/e2624d06ab94e41c7c77727b2941f1baed7edb647e63503953e6888020c9/fonttools-4.59.1-cp311-cp311-win_amd64.whl", hash = "sha256:c866eef7a0ba320486ade6c32bfc12813d1a5db8567e6904fb56d3d40acc5116", size = 2262779, upload-time = "2025-08-14T16:26:43.483Z" }, + { url = "https://files.pythonhosted.org/packages/ac/fe/6e069cc4cb8881d164a9bd956e9df555bc62d3eb36f6282e43440200009c/fonttools-4.59.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:43ab814bbba5f02a93a152ee61a04182bb5809bd2bc3609f7822e12c53ae2c91", size = 2769172, upload-time = "2025-08-14T16:26:45.729Z" }, + { url = "https://files.pythonhosted.org/packages/b9/98/ec4e03f748fefa0dd72d9d95235aff6fef16601267f4a2340f0e16b9330f/fonttools-4.59.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4f04c3ffbfa0baafcbc550657cf83657034eb63304d27b05cff1653b448ccff6", size = 2337281, upload-time = "2025-08-14T16:26:47.921Z" }, + { url = "https://files.pythonhosted.org/packages/8b/b1/890360a7e3d04a30ba50b267aca2783f4c1364363797e892e78a4f036076/fonttools-4.59.1-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d601b153e51a5a6221f0d4ec077b6bfc6ac35bfe6c19aeaa233d8990b2b71726", size = 4909215, upload-time = "2025-08-14T16:26:49.682Z" }, + { url = "https://files.pythonhosted.org/packages/8a/ec/2490599550d6c9c97a44c1e36ef4de52d6acf742359eaa385735e30c05c4/fonttools-4.59.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c735e385e30278c54f43a0d056736942023c9043f84ee1021eff9fd616d17693", size = 4951958, upload-time = "2025-08-14T16:26:51.616Z" }, + { url = "https://files.pythonhosted.org/packages/d1/40/bd053f6f7634234a9b9805ff8ae4f32df4f2168bee23cafd1271ba9915a9/fonttools-4.59.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1017413cdc8555dce7ee23720da490282ab7ec1cf022af90a241f33f9a49afc4", size = 4894738, upload-time = "2025-08-14T16:26:53.836Z" }, + { url = "https://files.pythonhosted.org/packages/ac/a1/3cd12a010d288325a7cfcf298a84825f0f9c29b01dee1baba64edfe89257/fonttools-4.59.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5c6d8d773470a5107052874341ed3c487c16ecd179976d81afed89dea5cd7406", size = 5045983, upload-time = "2025-08-14T16:26:56.153Z" }, + { url = "https://files.pythonhosted.org/packages/a2/af/8a2c3f6619cc43cf87951405337cc8460d08a4e717bb05eaa94b335d11dc/fonttools-4.59.1-cp312-cp312-win32.whl", hash = "sha256:2a2d0d33307f6ad3a2086a95dd607c202ea8852fa9fb52af9b48811154d1428a", size = 2203407, upload-time = "2025-08-14T16:26:58.165Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f2/a19b874ddbd3ebcf11d7e25188ef9ac3f68b9219c62263acb34aca8cde05/fonttools-4.59.1-cp312-cp312-win_amd64.whl", hash = "sha256:0b9e4fa7eaf046ed6ac470f6033d52c052481ff7a6e0a92373d14f556f298dc0", size = 2251561, upload-time = "2025-08-14T16:27:00.646Z" }, + { url = "https://files.pythonhosted.org/packages/0f/64/9d606e66d498917cd7a2ff24f558010d42d6fd4576d9dd57f0bd98333f5a/fonttools-4.59.1-py3-none-any.whl", hash = "sha256:647db657073672a8330608970a984d51573557f328030566521bc03415535042", size = 1130094, upload-time = "2025-08-14T16:28:12.048Z" }, ] [[package]] @@ -977,11 +1236,11 @@ wheels = [ [[package]] name = "fsspec" -version = "2025.7.0" +version = "2025.3.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8b/02/0835e6ab9cfc03916fe3f78c0956cfcdb6ff2669ffa6651065d5ebf7fc98/fsspec-2025.7.0.tar.gz", hash = "sha256:786120687ffa54b8283d942929540d8bc5ccfa820deb555a2b5d0ed2b737bf58", size = 304432, upload-time = "2025-07-15T16:05:21.19Z" } +sdist = { url = "https://files.pythonhosted.org/packages/34/f4/5721faf47b8c499e776bc34c6a8fc17efdf7fdef0b00f398128bc5dcb4ac/fsspec-2025.3.0.tar.gz", hash = "sha256:a935fd1ea872591f2b5148907d103488fc523295e6c64b835cfad8c3eca44972", size = 298491, upload-time = "2025-03-07T21:47:56.461Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2f/e0/014d5d9d7a4564cf1c40b5039bc882db69fd881111e03ab3657ac0b218e2/fsspec-2025.7.0-py3-none-any.whl", hash = "sha256:8b012e39f63c7d5f10474de957f3ab793b47b45ae7d39f2fb735f8bbe25c0e21", size = 199597, upload-time = "2025-07-15T16:05:19.529Z" }, + { url = "https://files.pythonhosted.org/packages/56/53/eb690efa8513166adef3e0669afd31e95ffde69fb3c52ec2ac7223ed6018/fsspec-2025.3.0-py3-none-any.whl", hash = "sha256:efb87af3efa9103f94ca91a7f8cb7a4df91af9f74fc106c9c7ea0efd7277c1b3", size = 193615, upload-time = "2025-03-07T21:47:54.809Z" }, ] [package.optional-dependencies] @@ -1014,26 +1273,89 @@ wheels = [ ] [[package]] -name = "google-search-results" -version = "2.4.0" +name = "google-auth" +version = "2.40.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "requests" }, + { name = "cachetools" }, + { name = "pyasn1-modules" }, + { name = "rsa" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/9b/e92ef23b84fa10a64ce4831390b7a4c2e53c0132568d99d4ae61d04c8855/google_auth-2.40.3.tar.gz", hash = "sha256:500c3a29adedeb36ea9cf24b8d10858e152f2412e3ca37829b3fa18e33d63b77", size = 281029, upload-time = "2025-06-04T18:04:57.577Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/63/b19553b658a1692443c62bd07e5868adaa0ad746a0751ba62c59568cd45b/google_auth-2.40.3-py2.py3-none-any.whl", hash = "sha256:1370d4593e86213563547f97a92752fc658456fe4514c809544f330fed45a7ca", size = 216137, upload-time = "2025-06-04T18:04:55.573Z" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c2/35/d60b766cf8ffbbb86c701dc2042e360d224180ba32056b7a5f56bfab2697/google_search_results-2.4.0.tar.gz", hash = "sha256:fa3305468c7ecaff3d39b38810455a4256404f4c01fa90e31dc352b798df0ac5", size = 10104, upload-time = "2021-07-26T16:58:32.324Z" } [[package]] -name = "googleapis-common-protos" -version = "1.70.0" +name = "google-genai" +version = "1.31.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/39/24/33db22342cf4a2ea27c9955e6713140fedd51e8b141b5ce5260897020f1a/googleapis_common_protos-1.70.0.tar.gz", hash = "sha256:0e1b44e0ea153e6594f9f394fef15193a68aaaea2d843f83e2742717ca753257", size = 145903, upload-time = "2025-04-14T10:17:02.924Z" } + { name = "anyio" }, + { name = "google-auth" }, + { name = "httpx" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "tenacity" }, + { name = "typing-extensions" }, + { name = "websockets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e0/1b/da30fa6e2966942d7028a58eb7aa7d04544dcc3aa66194365b2e0adac570/google_genai-1.31.0.tar.gz", hash = "sha256:8572b47aa684357c3e5e10d290ec772c65414114939e3ad2955203e27cd2fcbc", size = 233482, upload-time = "2025-08-18T23:40:21.733Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/27/1525bc9cbec58660f0842ebcbfe910a1dde908c2672373804879666e0bb8/google_genai-1.31.0-py3-none-any.whl", hash = "sha256:5c6959bcf862714e8ed0922db3aaf41885bacf6318751b3421bf1e459f78892f", size = 231876, upload-time = "2025-08-18T23:40:20.385Z" }, +] + +[[package]] +name = "google-search-results" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/35/d60b766cf8ffbbb86c701dc2042e360d224180ba32056b7a5f56bfab2697/google_search_results-2.4.0.tar.gz", hash = "sha256:fa3305468c7ecaff3d39b38810455a4256404f4c01fa90e31dc352b798df0ac5", size = 10104, upload-time = "2021-07-26T16:58:32.324Z" } + +[[package]] +name = "googleapis-common-protos" +version = "1.70.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/24/33db22342cf4a2ea27c9955e6713140fedd51e8b141b5ce5260897020f1a/googleapis_common_protos-1.70.0.tar.gz", hash = "sha256:0e1b44e0ea153e6594f9f394fef15193a68aaaea2d843f83e2742717ca753257", size = 145903, upload-time = "2025-04-14T10:17:02.924Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/86/f1/62a193f0227cf15a920390abe675f386dec35f7ae3ffe6da582d3ade42c7/googleapis_common_protos-1.70.0-py3-none-any.whl", hash = "sha256:b8bfcca8c25a2bb253e0e0b0adaf8c00773e5e6af6fd92397576680b807e0fd8", size = 294530, upload-time = "2025-04-14T10:17:01.271Z" }, ] +[[package]] +name = "gputil" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ed/0e/5c61eedde9f6c87713e89d794f01e378cfd9565847d4576fa627d758c554/GPUtil-1.4.0.tar.gz", hash = "sha256:099e52c65e512cdfa8c8763fca67f5a5c2afb63469602d5dcb4d296b3661efb9", size = 5545, upload-time = "2018-12-18T09:12:13.63Z" } + +[[package]] +name = "gql" +version = "4.1.0b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "backoff" }, + { name = "graphql-core" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/e7/6a97577e9e1bd679ea31cb375908558497a423f1257f9f6947399069f7b6/gql-4.1.0b0.tar.gz", hash = "sha256:40116b8461345c98c416f516a5a5db9abe5c9d6bde04055e2b4e942e9c76adc3", size = 215609, upload-time = "2025-08-17T14:59:04.722Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/f9/129acd1856ec69daf1562685d9603c2cb4dda6f7862bd018bc45cdede9ed/gql-4.1.0b0-py3-none-any.whl", hash = "sha256:ca71a6fa3571acbad273df3616acac5f320c69c22f69de736a43a3e9b81b5e44", size = 89933, upload-time = "2025-08-17T14:59:03.316Z" }, +] + +[package.optional-dependencies] +aiohttp = [ + { name = "aiohttp" }, +] +requests = [ + { name = "requests" }, + { name = "requests-toolbelt" }, +] + [[package]] name = "graphql-core" version = "3.3.0a9" @@ -1043,30 +1365,72 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bd/48/98ccc5fde6e4f7060cbce766d58e3364425af1374ea7379244b2c34cf635/graphql_core-3.3.0a9-py3-none-any.whl", hash = "sha256:73eeb24b8d0256cea62b70f0a7caa8bfc290e2e87769e0ccfdf4e9e4f06911cc", size = 227535, upload-time = "2025-06-18T21:37:17.579Z" }, ] +[[package]] +name = "graphviz" +version = "0.21" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/b3/3ac91e9be6b761a4b30d66ff165e54439dcd48b83f4e20d644867215f6ca/graphviz-0.21.tar.gz", hash = "sha256:20743e7183be82aaaa8ad6c93f8893c923bd6658a04c32ee115edb3c8a835f78", size = 200434, upload-time = "2025-06-15T09:35:05.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/4c/e0ce1ef95d4000ebc1c11801f9b944fa5910ecc15b5e351865763d8657f8/graphviz-0.21-py3-none-any.whl", hash = "sha256:54f33de9f4f911d7e84e4191749cac8cc5653f815b06738c54db9a15ab8b1e42", size = 47300, upload-time = "2025-06-15T09:35:04.433Z" }, +] + [[package]] name = "greenlet" -version = "3.2.3" +version = "3.2.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/03/b8/704d753a5a45507a7aab61f18db9509302ed3d0a27ac7e0359ec2905b1a6/greenlet-3.2.4.tar.gz", hash = "sha256:0dca0d95ff849f9a364385f36ab49f50065d76964944638be9691e1832e9f86d", size = 188260, upload-time = "2025-08-07T13:24:33.51Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/de/f28ced0a67749cac23fecb02b694f6473f47686dff6afaa211d186e2ef9c/greenlet-3.2.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:96378df1de302bc38e99c3a9aa311967b7dc80ced1dcc6f171e99842987882a2", size = 272305, upload-time = "2025-08-07T13:15:41.288Z" }, + { url = "https://files.pythonhosted.org/packages/09/16/2c3792cba130000bf2a31c5272999113f4764fd9d874fb257ff588ac779a/greenlet-3.2.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1ee8fae0519a337f2329cb78bd7a8e128ec0f881073d43f023c7b8d4831d5246", size = 632472, upload-time = "2025-08-07T13:42:55.044Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8f/95d48d7e3d433e6dae5b1682e4292242a53f22df82e6d3dda81b1701a960/greenlet-3.2.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94abf90142c2a18151632371140b3dba4dee031633fe614cb592dbb6c9e17bc3", size = 644646, upload-time = "2025-08-07T13:45:26.523Z" }, + { url = "https://files.pythonhosted.org/packages/d5/5e/405965351aef8c76b8ef7ad370e5da58d57ef6068df197548b015464001a/greenlet-3.2.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:4d1378601b85e2e5171b99be8d2dc85f594c79967599328f95c1dc1a40f1c633", size = 640519, upload-time = "2025-08-07T13:53:13.928Z" }, + { url = "https://files.pythonhosted.org/packages/25/5d/382753b52006ce0218297ec1b628e048c4e64b155379331f25a7316eb749/greenlet-3.2.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0db5594dce18db94f7d1650d7489909b57afde4c580806b8d9203b6e79cdc079", size = 639707, upload-time = "2025-08-07T13:18:27.146Z" }, + { url = "https://files.pythonhosted.org/packages/1f/8e/abdd3f14d735b2929290a018ecf133c901be4874b858dd1c604b9319f064/greenlet-3.2.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2523e5246274f54fdadbce8494458a2ebdcdbc7b802318466ac5606d3cded1f8", size = 587684, upload-time = "2025-08-07T13:18:25.164Z" }, + { url = "https://files.pythonhosted.org/packages/5d/65/deb2a69c3e5996439b0176f6651e0052542bb6c8f8ec2e3fba97c9768805/greenlet-3.2.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1987de92fec508535687fb807a5cea1560f6196285a4cde35c100b8cd632cc52", size = 1116647, upload-time = "2025-08-07T13:42:38.655Z" }, + { url = "https://files.pythonhosted.org/packages/3f/cc/b07000438a29ac5cfb2194bfc128151d52f333cee74dd7dfe3fb733fc16c/greenlet-3.2.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:55e9c5affaa6775e2c6b67659f3a71684de4c549b3dd9afca3bc773533d284fa", size = 1142073, upload-time = "2025-08-07T13:18:21.737Z" }, + { url = "https://files.pythonhosted.org/packages/67/24/28a5b2fa42d12b3d7e5614145f0bd89714c34c08be6aabe39c14dd52db34/greenlet-3.2.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c9c6de1940a7d828635fbd254d69db79e54619f165ee7ce32fda763a9cb6a58c", size = 1548385, upload-time = "2025-11-04T12:42:11.067Z" }, + { url = "https://files.pythonhosted.org/packages/6a/05/03f2f0bdd0b0ff9a4f7b99333d57b53a7709c27723ec8123056b084e69cd/greenlet-3.2.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03c5136e7be905045160b1b9fdca93dd6727b180feeafda6818e6496434ed8c5", size = 1613329, upload-time = "2025-11-04T12:42:12.928Z" }, + { url = "https://files.pythonhosted.org/packages/d8/0f/30aef242fcab550b0b3520b8e3561156857c94288f0332a79928c31a52cf/greenlet-3.2.4-cp311-cp311-win_amd64.whl", hash = "sha256:9c40adce87eaa9ddb593ccb0fa6a07caf34015a29bf8d344811665b573138db9", size = 299100, upload-time = "2025-08-07T13:44:12.287Z" }, + { url = "https://files.pythonhosted.org/packages/44/69/9b804adb5fd0671f367781560eb5eb586c4d495277c93bde4307b9e28068/greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd", size = 274079, upload-time = "2025-08-07T13:15:45.033Z" }, + { url = "https://files.pythonhosted.org/packages/46/e9/d2a80c99f19a153eff70bc451ab78615583b8dac0754cfb942223d2c1a0d/greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb", size = 640997, upload-time = "2025-08-07T13:42:56.234Z" }, + { url = "https://files.pythonhosted.org/packages/3b/16/035dcfcc48715ccd345f3a93183267167cdd162ad123cd93067d86f27ce4/greenlet-3.2.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f28588772bb5fb869a8eb331374ec06f24a83a9c25bfa1f38b6993afe9c1e968", size = 655185, upload-time = "2025-08-07T13:45:27.624Z" }, + { url = "https://files.pythonhosted.org/packages/31/da/0386695eef69ffae1ad726881571dfe28b41970173947e7c558d9998de0f/greenlet-3.2.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:5c9320971821a7cb77cfab8d956fa8e39cd07ca44b6070db358ceb7f8797c8c9", size = 649926, upload-time = "2025-08-07T13:53:15.251Z" }, + { url = "https://files.pythonhosted.org/packages/68/88/69bf19fd4dc19981928ceacbc5fd4bb6bc2215d53199e367832e98d1d8fe/greenlet-3.2.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c60a6d84229b271d44b70fb6e5fa23781abb5d742af7b808ae3f6efd7c9c60f6", size = 651839, upload-time = "2025-08-07T13:18:30.281Z" }, + { url = "https://files.pythonhosted.org/packages/19/0d/6660d55f7373b2ff8152401a83e02084956da23ae58cddbfb0b330978fe9/greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0", size = 607586, upload-time = "2025-08-07T13:18:28.544Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1a/c953fdedd22d81ee4629afbb38d2f9d71e37d23caace44775a3a969147d4/greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0", size = 1123281, upload-time = "2025-08-07T13:42:39.858Z" }, + { url = "https://files.pythonhosted.org/packages/3f/c7/12381b18e21aef2c6bd3a636da1088b888b97b7a0362fac2e4de92405f97/greenlet-3.2.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:20fb936b4652b6e307b8f347665e2c615540d4b42b3b4c8a321d8286da7e520f", size = 1151142, upload-time = "2025-08-07T13:18:22.981Z" }, + { url = "https://files.pythonhosted.org/packages/27/45/80935968b53cfd3f33cf99ea5f08227f2646e044568c9b1555b58ffd61c2/greenlet-3.2.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ee7a6ec486883397d70eec05059353b8e83eca9168b9f3f9a361971e77e0bcd0", size = 1564846, upload-time = "2025-11-04T12:42:15.191Z" }, + { url = "https://files.pythonhosted.org/packages/69/02/b7c30e5e04752cb4db6202a3858b149c0710e5453b71a3b2aec5d78a1aab/greenlet-3.2.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:326d234cbf337c9c3def0676412eb7040a35a768efc92504b947b3e9cfc7543d", size = 1633814, upload-time = "2025-11-04T12:42:17.175Z" }, + { url = "https://files.pythonhosted.org/packages/e9/08/b0814846b79399e585f974bbeebf5580fbe59e258ea7be64d9dfb253c84f/greenlet-3.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7d4e128405eea3814a12cc2605e0e6aedb4035bf32697f72deca74de4105e02", size = 299899, upload-time = "2025-08-07T13:38:53.448Z" }, +] + +[[package]] +name = "griffe" +version = "1.12.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/ca/29f36e00c74844ae50d139cf5a8b1751887b2f4d5023af65d460268ad7aa/griffe-1.12.1.tar.gz", hash = "sha256:29f5a6114c0aeda7d9c86a570f736883f8a2c5b38b57323d56b3d1c000565567", size = 411863, upload-time = "2025-08-14T21:08:15.38Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/f2/4fab6c3e5bcaf38a44cc8a974d2752eaad4c129e45d6533d926a30edd133/griffe-1.12.1-py3-none-any.whl", hash = "sha256:2d7c12334de00089c31905424a00abcfd931b45b8b516967f224133903d302cc", size = 138940, upload-time = "2025-08-14T21:08:13.382Z" }, +] + +[[package]] +name = "groq" +version = "0.31.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c9/92/bb85bd6e80148a4d2e0c59f7c0c2891029f8fd510183afc7d8d2feeed9b6/greenlet-3.2.3.tar.gz", hash = "sha256:8b0dd8ae4c0d6f5e54ee55ba935eeb3d735a9b58a8a1e5b5cbab64e01a39f365", size = 185752, upload-time = "2025-06-05T16:16:09.955Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/2e/d4fcb2978f826358b673f779f78fa8a32ee37df11920dc2bb5589cbeecef/greenlet-3.2.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:784ae58bba89fa1fa5733d170d42486580cab9decda3484779f4759345b29822", size = 270219, upload-time = "2025-06-05T16:10:10.414Z" }, - { url = "https://files.pythonhosted.org/packages/16/24/929f853e0202130e4fe163bc1d05a671ce8dcd604f790e14896adac43a52/greenlet-3.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0921ac4ea42a5315d3446120ad48f90c3a6b9bb93dd9b3cf4e4d84a66e42de83", size = 630383, upload-time = "2025-06-05T16:38:51.785Z" }, - { url = "https://files.pythonhosted.org/packages/d1/b2/0320715eb61ae70c25ceca2f1d5ae620477d246692d9cc284c13242ec31c/greenlet-3.2.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:d2971d93bb99e05f8c2c0c2f4aa9484a18d98c4c3bd3c62b65b7e6ae33dfcfaf", size = 642422, upload-time = "2025-06-05T16:41:35.259Z" }, - { url = "https://files.pythonhosted.org/packages/bd/49/445fd1a210f4747fedf77615d941444349c6a3a4a1135bba9701337cd966/greenlet-3.2.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c667c0bf9d406b77a15c924ef3285e1e05250948001220368e039b6aa5b5034b", size = 638375, upload-time = "2025-06-05T16:48:18.235Z" }, - { url = "https://files.pythonhosted.org/packages/7e/c8/ca19760cf6eae75fa8dc32b487e963d863b3ee04a7637da77b616703bc37/greenlet-3.2.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:592c12fb1165be74592f5de0d70f82bc5ba552ac44800d632214b76089945147", size = 637627, upload-time = "2025-06-05T16:13:02.858Z" }, - { url = "https://files.pythonhosted.org/packages/65/89/77acf9e3da38e9bcfca881e43b02ed467c1dedc387021fc4d9bd9928afb8/greenlet-3.2.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29e184536ba333003540790ba29829ac14bb645514fbd7e32af331e8202a62a5", size = 585502, upload-time = "2025-06-05T16:12:49.642Z" }, - { url = "https://files.pythonhosted.org/packages/97/c6/ae244d7c95b23b7130136e07a9cc5aadd60d59b5951180dc7dc7e8edaba7/greenlet-3.2.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:93c0bb79844a367782ec4f429d07589417052e621aa39a5ac1fb99c5aa308edc", size = 1114498, upload-time = "2025-06-05T16:36:46.598Z" }, - { url = "https://files.pythonhosted.org/packages/89/5f/b16dec0cbfd3070658e0d744487919740c6d45eb90946f6787689a7efbce/greenlet-3.2.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:751261fc5ad7b6705f5f76726567375bb2104a059454e0226e1eef6c756748ba", size = 1139977, upload-time = "2025-06-05T16:12:38.262Z" }, - { url = "https://files.pythonhosted.org/packages/66/77/d48fb441b5a71125bcac042fc5b1494c806ccb9a1432ecaa421e72157f77/greenlet-3.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:83a8761c75312361aa2b5b903b79da97f13f556164a7dd2d5448655425bd4c34", size = 297017, upload-time = "2025-06-05T16:25:05.225Z" }, - { url = "https://files.pythonhosted.org/packages/f3/94/ad0d435f7c48debe960c53b8f60fb41c2026b1d0fa4a99a1cb17c3461e09/greenlet-3.2.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:25ad29caed5783d4bd7a85c9251c651696164622494c00802a139c00d639242d", size = 271992, upload-time = "2025-06-05T16:11:23.467Z" }, - { url = "https://files.pythonhosted.org/packages/93/5d/7c27cf4d003d6e77749d299c7c8f5fd50b4f251647b5c2e97e1f20da0ab5/greenlet-3.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88cd97bf37fe24a6710ec6a3a7799f3f81d9cd33317dcf565ff9950c83f55e0b", size = 638820, upload-time = "2025-06-05T16:38:52.882Z" }, - { url = "https://files.pythonhosted.org/packages/c6/7e/807e1e9be07a125bb4c169144937910bf59b9d2f6d931578e57f0bce0ae2/greenlet-3.2.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:baeedccca94880d2f5666b4fa16fc20ef50ba1ee353ee2d7092b383a243b0b0d", size = 653046, upload-time = "2025-06-05T16:41:36.343Z" }, - { url = "https://files.pythonhosted.org/packages/9d/ab/158c1a4ea1068bdbc78dba5a3de57e4c7aeb4e7fa034320ea94c688bfb61/greenlet-3.2.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:be52af4b6292baecfa0f397f3edb3c6092ce071b499dd6fe292c9ac9f2c8f264", size = 647701, upload-time = "2025-06-05T16:48:19.604Z" }, - { url = "https://files.pythonhosted.org/packages/cc/0d/93729068259b550d6a0288da4ff72b86ed05626eaf1eb7c0d3466a2571de/greenlet-3.2.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0cc73378150b8b78b0c9fe2ce56e166695e67478550769536a6742dca3651688", size = 649747, upload-time = "2025-06-05T16:13:04.628Z" }, - { url = "https://files.pythonhosted.org/packages/f6/f6/c82ac1851c60851302d8581680573245c8fc300253fc1ff741ae74a6c24d/greenlet-3.2.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:706d016a03e78df129f68c4c9b4c4f963f7d73534e48a24f5f5a7101ed13dbbb", size = 605461, upload-time = "2025-06-05T16:12:50.792Z" }, - { url = "https://files.pythonhosted.org/packages/98/82/d022cf25ca39cf1200650fc58c52af32c90f80479c25d1cbf57980ec3065/greenlet-3.2.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:419e60f80709510c343c57b4bb5a339d8767bf9aef9b8ce43f4f143240f88b7c", size = 1121190, upload-time = "2025-06-05T16:36:48.59Z" }, - { url = "https://files.pythonhosted.org/packages/f5/e1/25297f70717abe8104c20ecf7af0a5b82d2f5a980eb1ac79f65654799f9f/greenlet-3.2.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:93d48533fade144203816783373f27a97e4193177ebaaf0fc396db19e5d61163", size = 1149055, upload-time = "2025-06-05T16:12:40.457Z" }, - { url = "https://files.pythonhosted.org/packages/1f/8f/8f9e56c5e82eb2c26e8cde787962e66494312dc8cb261c460e1f3a9c88bc/greenlet-3.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:7454d37c740bb27bdeddfc3f358f26956a07d5220818ceb467a483197d84f849", size = 297817, upload-time = "2025-06-05T16:29:49.244Z" }, +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/a2/77fd1460e7d55859219223719aa44ae8902a3a1ad333cd5faf330eb0b894/groq-0.31.0.tar.gz", hash = "sha256:182252e9bf0d696df607c137cbafa851d2c84aaf94bcfe9165c0bc231043490c", size = 136237, upload-time = "2025-08-05T23:14:01.183Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/f8/14672d69a91495f43462c5490067eeafc30346e81bda1a62848e897f9bc3/groq-0.31.0-py3-none-any.whl", hash = "sha256:5e3c7ec9728b7cccf913da982a9b5ebb46dc18a070b35e12a3d6a1e12d6b0f7f", size = 131365, upload-time = "2025-08-05T23:13:59.768Z" }, ] [[package]] @@ -1083,28 +1447,45 @@ wheels = [ [[package]] name = "grpcio" -version = "1.67.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/20/53/d9282a66a5db45981499190b77790570617a604a38f3d103d0400974aeb5/grpcio-1.67.1.tar.gz", hash = "sha256:3dc2ed4cabea4dc14d5e708c2b426205956077cc5de419b4d4079315017e9732", size = 12580022, upload-time = "2024-10-29T06:30:07.787Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/59/2c/b60d6ea1f63a20a8d09c6db95c4f9a16497913fb3048ce0990ed81aeeca0/grpcio-1.67.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:7818c0454027ae3384235a65210bbf5464bd715450e30a3d40385453a85a70cb", size = 5119075, upload-time = "2024-10-29T06:24:04.696Z" }, - { url = "https://files.pythonhosted.org/packages/b3/9a/e1956f7ca582a22dd1f17b9e26fcb8229051b0ce6d33b47227824772feec/grpcio-1.67.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ea33986b70f83844cd00814cee4451055cd8cab36f00ac64a31f5bb09b31919e", size = 11009159, upload-time = "2024-10-29T06:24:07.781Z" }, - { url = "https://files.pythonhosted.org/packages/43/a8/35fbbba580c4adb1d40d12e244cf9f7c74a379073c0a0ca9d1b5338675a1/grpcio-1.67.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:c7a01337407dd89005527623a4a72c5c8e2894d22bead0895306b23c6695698f", size = 5629476, upload-time = "2024-10-29T06:24:11.444Z" }, - { url = "https://files.pythonhosted.org/packages/77/c9/864d336e167263d14dfccb4dbfa7fce634d45775609895287189a03f1fc3/grpcio-1.67.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80b866f73224b0634f4312a4674c1be21b2b4afa73cb20953cbbb73a6b36c3cc", size = 6239901, upload-time = "2024-10-29T06:24:14.2Z" }, - { url = "https://files.pythonhosted.org/packages/f7/1e/0011408ebabf9bd69f4f87cc1515cbfe2094e5a32316f8714a75fd8ddfcb/grpcio-1.67.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fff78ba10d4250bfc07a01bd6254a6d87dc67f9627adece85c0b2ed754fa96", size = 5881010, upload-time = "2024-10-29T06:24:17.451Z" }, - { url = "https://files.pythonhosted.org/packages/b4/7d/fbca85ee9123fb296d4eff8df566f458d738186d0067dec6f0aa2fd79d71/grpcio-1.67.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8a23cbcc5bb11ea7dc6163078be36c065db68d915c24f5faa4f872c573bb400f", size = 6580706, upload-time = "2024-10-29T06:24:20.038Z" }, - { url = "https://files.pythonhosted.org/packages/75/7a/766149dcfa2dfa81835bf7df623944c1f636a15fcb9b6138ebe29baf0bc6/grpcio-1.67.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1a65b503d008f066e994f34f456e0647e5ceb34cfcec5ad180b1b44020ad4970", size = 6161799, upload-time = "2024-10-29T06:24:22.604Z" }, - { url = "https://files.pythonhosted.org/packages/09/13/5b75ae88810aaea19e846f5380611837de411181df51fd7a7d10cb178dcb/grpcio-1.67.1-cp311-cp311-win32.whl", hash = "sha256:e29ca27bec8e163dca0c98084040edec3bc49afd10f18b412f483cc68c712744", size = 3616330, upload-time = "2024-10-29T06:24:25.775Z" }, - { url = "https://files.pythonhosted.org/packages/aa/39/38117259613f68f072778c9638a61579c0cfa5678c2558706b10dd1d11d3/grpcio-1.67.1-cp311-cp311-win_amd64.whl", hash = "sha256:786a5b18544622bfb1e25cc08402bd44ea83edfb04b93798d85dca4d1a0b5be5", size = 4354535, upload-time = "2024-10-29T06:24:28.614Z" }, - { url = "https://files.pythonhosted.org/packages/6e/25/6f95bd18d5f506364379eabc0d5874873cc7dbdaf0757df8d1e82bc07a88/grpcio-1.67.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:267d1745894200e4c604958da5f856da6293f063327cb049a51fe67348e4f953", size = 5089809, upload-time = "2024-10-29T06:24:31.24Z" }, - { url = "https://files.pythonhosted.org/packages/10/3f/d79e32e5d0354be33a12db2267c66d3cfeff700dd5ccdd09fd44a3ff4fb6/grpcio-1.67.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:85f69fdc1d28ce7cff8de3f9c67db2b0ca9ba4449644488c1e0303c146135ddb", size = 10981985, upload-time = "2024-10-29T06:24:34.942Z" }, - { url = "https://files.pythonhosted.org/packages/21/f2/36fbc14b3542e3a1c20fb98bd60c4732c55a44e374a4eb68f91f28f14aab/grpcio-1.67.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:f26b0b547eb8d00e195274cdfc63ce64c8fc2d3e2d00b12bf468ece41a0423a0", size = 5588770, upload-time = "2024-10-29T06:24:38.145Z" }, - { url = "https://files.pythonhosted.org/packages/0d/af/bbc1305df60c4e65de8c12820a942b5e37f9cf684ef5e49a63fbb1476a73/grpcio-1.67.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4422581cdc628f77302270ff839a44f4c24fdc57887dc2a45b7e53d8fc2376af", size = 6214476, upload-time = "2024-10-29T06:24:41.006Z" }, - { url = "https://files.pythonhosted.org/packages/92/cf/1d4c3e93efa93223e06a5c83ac27e32935f998bc368e276ef858b8883154/grpcio-1.67.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d7616d2ded471231c701489190379e0c311ee0a6c756f3c03e6a62b95a7146e", size = 5850129, upload-time = "2024-10-29T06:24:43.553Z" }, - { url = "https://files.pythonhosted.org/packages/ae/ca/26195b66cb253ac4d5ef59846e354d335c9581dba891624011da0e95d67b/grpcio-1.67.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8a00efecde9d6fcc3ab00c13f816313c040a28450e5e25739c24f432fc6d3c75", size = 6568489, upload-time = "2024-10-29T06:24:46.453Z" }, - { url = "https://files.pythonhosted.org/packages/d1/94/16550ad6b3f13b96f0856ee5dfc2554efac28539ee84a51d7b14526da985/grpcio-1.67.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:699e964923b70f3101393710793289e42845791ea07565654ada0969522d0a38", size = 6149369, upload-time = "2024-10-29T06:24:49.112Z" }, - { url = "https://files.pythonhosted.org/packages/33/0d/4c3b2587e8ad7f121b597329e6c2620374fccbc2e4e1aa3c73ccc670fde4/grpcio-1.67.1-cp312-cp312-win32.whl", hash = "sha256:4e7b904484a634a0fff132958dabdb10d63e0927398273917da3ee103e8d1f78", size = 3599176, upload-time = "2024-10-29T06:24:51.443Z" }, - { url = "https://files.pythonhosted.org/packages/7d/36/0c03e2d80db69e2472cf81c6123aa7d14741de7cf790117291a703ae6ae1/grpcio-1.67.1-cp312-cp312-win_amd64.whl", hash = "sha256:5721e66a594a6c4204458004852719b38f3d5522082be9061d6510b455c90afc", size = 4346574, upload-time = "2024-10-29T06:24:54.587Z" }, +version = "1.74.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/b4/35feb8f7cab7239c5b94bd2db71abb3d6adb5f335ad8f131abb6060840b6/grpcio-1.74.0.tar.gz", hash = "sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1", size = 12756048, upload-time = "2025-07-24T18:54:23.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/77/b2f06db9f240a5abeddd23a0e49eae2b6ac54d85f0e5267784ce02269c3b/grpcio-1.74.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31", size = 5487368, upload-time = "2025-07-24T18:53:03.548Z" }, + { url = "https://files.pythonhosted.org/packages/48/99/0ac8678a819c28d9a370a663007581744a9f2a844e32f0fa95e1ddda5b9e/grpcio-1.74.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4", size = 10999804, upload-time = "2025-07-24T18:53:05.095Z" }, + { url = "https://files.pythonhosted.org/packages/45/c6/a2d586300d9e14ad72e8dc211c7aecb45fe9846a51e558c5bca0c9102c7f/grpcio-1.74.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce", size = 5987667, upload-time = "2025-07-24T18:53:07.157Z" }, + { url = "https://files.pythonhosted.org/packages/c9/57/5f338bf56a7f22584e68d669632e521f0de460bb3749d54533fc3d0fca4f/grpcio-1.74.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3", size = 6655612, upload-time = "2025-07-24T18:53:09.244Z" }, + { url = "https://files.pythonhosted.org/packages/82/ea/a4820c4c44c8b35b1903a6c72a5bdccec92d0840cf5c858c498c66786ba5/grpcio-1.74.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182", size = 6219544, upload-time = "2025-07-24T18:53:11.221Z" }, + { url = "https://files.pythonhosted.org/packages/a4/17/0537630a921365928f5abb6d14c79ba4dcb3e662e0dbeede8af4138d9dcf/grpcio-1.74.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d", size = 6334863, upload-time = "2025-07-24T18:53:12.925Z" }, + { url = "https://files.pythonhosted.org/packages/e2/a6/85ca6cb9af3f13e1320d0a806658dca432ff88149d5972df1f7b51e87127/grpcio-1.74.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f", size = 7019320, upload-time = "2025-07-24T18:53:15.002Z" }, + { url = "https://files.pythonhosted.org/packages/4f/a7/fe2beab970a1e25d2eff108b3cf4f7d9a53c185106377a3d1989216eba45/grpcio-1.74.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4", size = 6514228, upload-time = "2025-07-24T18:53:16.999Z" }, + { url = "https://files.pythonhosted.org/packages/6a/c2/2f9c945c8a248cebc3ccda1b7a1bf1775b9d7d59e444dbb18c0014e23da6/grpcio-1.74.0-cp311-cp311-win32.whl", hash = "sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b", size = 3817216, upload-time = "2025-07-24T18:53:20.564Z" }, + { url = "https://files.pythonhosted.org/packages/ff/d1/a9cf9c94b55becda2199299a12b9feef0c79946b0d9d34c989de6d12d05d/grpcio-1.74.0-cp311-cp311-win_amd64.whl", hash = "sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11", size = 4495380, upload-time = "2025-07-24T18:53:22.058Z" }, + { url = "https://files.pythonhosted.org/packages/4c/5d/e504d5d5c4469823504f65687d6c8fb97b7f7bf0b34873b7598f1df24630/grpcio-1.74.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8", size = 5445551, upload-time = "2025-07-24T18:53:23.641Z" }, + { url = "https://files.pythonhosted.org/packages/43/01/730e37056f96f2f6ce9f17999af1556df62ee8dab7fa48bceeaab5fd3008/grpcio-1.74.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6", size = 10979810, upload-time = "2025-07-24T18:53:25.349Z" }, + { url = "https://files.pythonhosted.org/packages/79/3d/09fd100473ea5c47083889ca47ffd356576173ec134312f6aa0e13111dee/grpcio-1.74.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5", size = 5941946, upload-time = "2025-07-24T18:53:27.387Z" }, + { url = "https://files.pythonhosted.org/packages/8a/99/12d2cca0a63c874c6d3d195629dcd85cdf5d6f98a30d8db44271f8a97b93/grpcio-1.74.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49", size = 6621763, upload-time = "2025-07-24T18:53:29.193Z" }, + { url = "https://files.pythonhosted.org/packages/9d/2c/930b0e7a2f1029bbc193443c7bc4dc2a46fedb0203c8793dcd97081f1520/grpcio-1.74.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7", size = 6180664, upload-time = "2025-07-24T18:53:30.823Z" }, + { url = "https://files.pythonhosted.org/packages/db/d5/ff8a2442180ad0867717e670f5ec42bfd8d38b92158ad6bcd864e6d4b1ed/grpcio-1.74.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3", size = 6301083, upload-time = "2025-07-24T18:53:32.454Z" }, + { url = "https://files.pythonhosted.org/packages/b0/ba/b361d390451a37ca118e4ec7dccec690422e05bc85fba2ec72b06cefec9f/grpcio-1.74.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707", size = 6994132, upload-time = "2025-07-24T18:53:34.506Z" }, + { url = "https://files.pythonhosted.org/packages/3b/0c/3a5fa47d2437a44ced74141795ac0251bbddeae74bf81df3447edd767d27/grpcio-1.74.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b", size = 6489616, upload-time = "2025-07-24T18:53:36.217Z" }, + { url = "https://files.pythonhosted.org/packages/ae/95/ab64703b436d99dc5217228babc76047d60e9ad14df129e307b5fec81fd0/grpcio-1.74.0-cp312-cp312-win32.whl", hash = "sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c", size = 3807083, upload-time = "2025-07-24T18:53:37.911Z" }, + { url = "https://files.pythonhosted.org/packages/84/59/900aa2445891fc47a33f7d2f76e00ca5d6ae6584b20d19af9c06fa09bf9a/grpcio-1.74.0-cp312-cp312-win_amd64.whl", hash = "sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc", size = 4490123, upload-time = "2025-07-24T18:53:39.528Z" }, +] + +[[package]] +name = "gssapi" +version = "1.11.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "decorator" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/23/52/c1e90623c259a42ab0587078bb04f959867b970add46ff66750ead8fc7c5/gssapi-1.11.1.tar.gz", hash = "sha256:2049ee4b1d0c363163a1344b7282a363f9f4094e51d2c36de0cf01d4735e0ae2", size = 95233, upload-time = "2026-01-26T21:01:39.463Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/75/3cc18f2d084d19fbba38dc684588cf5f674c647e754f9cf1625bd78c39f8/gssapi-1.11.1-cp311-abi3-macosx_10_9_x86_64.whl", hash = "sha256:2298e5909a8f2d27c29352885a24e4026cfd3fa24fc38d4a0a3743fa5a3e7667", size = 611712, upload-time = "2026-01-26T21:01:19.626Z" }, + { url = "https://files.pythonhosted.org/packages/81/f9/ac0f8c43c209d56c89655f80cd4ae43379f88370d01a7e11f264f081eef5/gssapi-1.11.1-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:5b60b1f8d8d3e36c025bd3494105de1dfccee578e8de001f423cc094468e3022", size = 642782, upload-time = "2026-01-26T21:01:21.462Z" }, + { url = "https://files.pythonhosted.org/packages/f8/97/f4ea9248bfdf5fcde2c5bf0bc0e573d212748724a32a5aa1002e11edb760/gssapi-1.11.1-cp311-abi3-win32.whl", hash = "sha256:9738fe0ba163c28ccf521de7520640bde4b135c1b6e87a5ac5a90435369e89c0", size = 699801, upload-time = "2026-01-26T21:01:23.226Z" }, + { url = "https://files.pythonhosted.org/packages/54/ca/7f839880baf7c365768884161c246a3b6201738722fc7581a995190ec431/gssapi-1.11.1-cp311-abi3-win_amd64.whl", hash = "sha256:96a102ad1ec266e2d843468bf03149982969fc70344f303f81ea20197b80d7a1", size = 781646, upload-time = "2026-01-26T21:01:24.618Z" }, ] [[package]] @@ -1118,17 +1499,17 @@ wheels = [ [[package]] name = "hf-xet" -version = "1.1.6rc2" +version = "1.1.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/68/02/9590927b54d8eca803d5f2c22398c91907bb430fe9082351449ba1b47d21/hf_xet-1.1.6rc2.tar.gz", hash = "sha256:f59e54beb49e3c424d0c8d6a8df74349192e994f7739c18030b32330589d7671", size = 494316, upload-time = "2025-07-14T20:09:24.083Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7a/49/91010b59debc7c862a5fd426d343134dd9a68778dbe570234b6495a4e204/hf_xet-1.1.8.tar.gz", hash = "sha256:62a0043e441753bbc446dcb5a3fe40a4d03f5fb9f13589ef1df9ab19252beb53", size = 484065, upload-time = "2025-08-18T22:01:03.584Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/21/55/5e30c2c63122301f184cc442c5aed077bd6f5ae2079a0f4b701a373d81fb/hf_xet-1.1.6rc2-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e93026949c596b2b644730d2a1a007a63d1d1c80f293247962c368c777d00c90", size = 2700641, upload-time = "2025-07-14T20:09:19.086Z" }, - { url = "https://files.pythonhosted.org/packages/b3/a0/9b493c6373e8a16a67527449a4719b8dbe3177aa17ed591bc2ec91600bc3/hf_xet-1.1.6rc2-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:cf88c08a4fd8dbe62ee468e9dc1c4f0e37abcfa8cac130189fcffc6ca8d53526", size = 2565067, upload-time = "2025-07-14T20:09:17.852Z" }, - { url = "https://files.pythonhosted.org/packages/56/de/da1c3e840fcb078d20957d69ef150a2492a6b8c8c0272e591770762cffc2/hf_xet-1.1.6rc2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c37fff1cc2509f861706af6f5e5d27178bade8a02d84eb069c1b942952cc7b92", size = 3115719, upload-time = "2025-07-14T20:09:16.604Z" }, - { url = "https://files.pythonhosted.org/packages/3a/c9/1719b01de254868cb1e0a6f64dc6f763bfe0e23c56114b392452c07152e4/hf_xet-1.1.6rc2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9b1926694628bffe2589d213596703cce9ac7383712ab15dde638a0b778e9e97", size = 3007969, upload-time = "2025-07-14T20:09:13.963Z" }, - { url = "https://files.pythonhosted.org/packages/66/65/65576be66ecf94f853cdbd5a37babb9693655b8dcf37468f59ed7594f877/hf_xet-1.1.6rc2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:acd8497a218b5db57594aaa9ba39f3fec86440b80d08f84e2595ab2a654f1559", size = 3170371, upload-time = "2025-07-14T20:09:20.981Z" }, - { url = "https://files.pythonhosted.org/packages/bb/3a/86750fd0ec6eb7c4de1394e3921f3357f44ef64c033a5d86129b85cf7dac/hf_xet-1.1.6rc2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38cd088fc7f187ca735c9f47d1a6b79587334b7ffd111494cf867920574194b9", size = 3280665, upload-time = "2025-07-14T20:09:22.566Z" }, - { url = "https://files.pythonhosted.org/packages/f7/72/a45ae75ad1065c05ab79a1f431d14c3e8fb1b64abae95da7e0dcb5265ba5/hf_xet-1.1.6rc2-cp37-abi3-win_amd64.whl", hash = "sha256:3676ed0f45b3f0332e135e99f467a34f1aabe8a49172ca6e8d7e011ddad305fa", size = 2751450, upload-time = "2025-07-14T20:09:25.35Z" }, + { url = "https://files.pythonhosted.org/packages/9c/91/5814db3a0d4a65fb6a87f0931ae28073b87f06307701fe66e7c41513bfb4/hf_xet-1.1.8-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:3d5f82e533fc51c7daad0f9b655d9c7811b5308e5890236828bd1dd3ed8fea74", size = 2752357, upload-time = "2025-08-18T22:00:58.777Z" }, + { url = "https://files.pythonhosted.org/packages/70/72/ce898516e97341a7a9d450609e130e108643389110261eaee6deb1ba8545/hf_xet-1.1.8-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:8e2dba5896bca3ab61d0bef4f01a1647004de59640701b37e37eaa57087bbd9d", size = 2613142, upload-time = "2025-08-18T22:00:57.252Z" }, + { url = "https://files.pythonhosted.org/packages/b7/d6/13af5f916cef795ac2b5e4cc1de31f2e0e375f4475d50799915835f301c2/hf_xet-1.1.8-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfe5700bc729be3d33d4e9a9b5cc17a951bf8c7ada7ba0c9198a6ab2053b7453", size = 3175859, upload-time = "2025-08-18T22:00:55.978Z" }, + { url = "https://files.pythonhosted.org/packages/4c/ed/34a193c9d1d72b7c3901b3b5153b1be9b2736b832692e1c3f167af537102/hf_xet-1.1.8-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:09e86514c3c4284ed8a57d6b0f3d089f9836a0af0a1ceb3c9dd664f1f3eaefef", size = 3074178, upload-time = "2025-08-18T22:00:54.147Z" }, + { url = "https://files.pythonhosted.org/packages/4a/1b/de6817b4bf65385280252dff5c9cceeedfbcb27ddb93923639323c1034a4/hf_xet-1.1.8-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4a9b99ab721d385b83f4fc8ee4e0366b0b59dce03b5888a86029cc0ca634efbf", size = 3238122, upload-time = "2025-08-18T22:01:00.546Z" }, + { url = "https://files.pythonhosted.org/packages/b7/13/874c85c7ed519ec101deb654f06703d9e5e68d34416730f64c4755ada36a/hf_xet-1.1.8-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:25b9d43333bbef39aeae1616789ec329c21401a7fe30969d538791076227b591", size = 3344325, upload-time = "2025-08-18T22:01:02.013Z" }, + { url = "https://files.pythonhosted.org/packages/9e/d3/0aaf279f4f3dea58e99401b92c31c0f752924ba0e6c7d7bb07b1dbd7f35e/hf_xet-1.1.8-cp37-abi3-win_amd64.whl", hash = "sha256:4171f31d87b13da4af1ed86c98cf763292e4720c088b4957cf9d564f92904ca9", size = 2801689, upload-time = "2025-08-18T22:01:04.81Z" }, ] [[package]] @@ -1192,7 +1573,7 @@ wheels = [ [[package]] name = "huggingface-hub" -version = "0.33.4" +version = "0.35.0rc0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock" }, @@ -1204,9 +1585,9 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4b/9e/9366b7349fc125dd68b9d384a0fea84d67b7497753fe92c71b67e13f47c4/huggingface_hub-0.33.4.tar.gz", hash = "sha256:6af13478deae120e765bfd92adad0ae1aec1ad8c439b46f23058ad5956cbca0a", size = 426674, upload-time = "2025-07-11T12:32:48.694Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3c/6b/d2234ceb0842fb36345e0c36290662f30429cfca3e9726e705f6f079e7ee/huggingface_hub-0.35.0rc0.tar.gz", hash = "sha256:1a8d599a28f8071d45e82f94b5e3437880afda153934198b91ce2d7eea724722", size = 456850, upload-time = "2025-07-24T14:02:10.538Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/46/7b/98daa50a2db034cab6cd23a3de04fa2358cb691593d28e9130203eb7a805/huggingface_hub-0.33.4-py3-none-any.whl", hash = "sha256:09f9f4e7ca62547c70f8b82767eefadd2667f4e116acba2e3e62a5a81815a7bb", size = 515339, upload-time = "2025-07-11T12:32:46.346Z" }, + { url = "https://files.pythonhosted.org/packages/e4/56/4d882ae1577879616e620bc5a4c20c05bd8e4a2367ca630331c48b3193ed/huggingface_hub-0.35.0rc0-py3-none-any.whl", hash = "sha256:812de58926a5ff57b1a1393fff26f26ea8e5aa9de126da62607094018d811296", size = 558670, upload-time = "2025-07-24T14:02:08.744Z" }, ] [[package]] @@ -1239,6 +1620,85 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, ] +[[package]] +name = "intervaltree" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/fb/396d568039d21344639db96d940d40eb62befe704ef849b27949ded5c3bb/intervaltree-3.1.0.tar.gz", hash = "sha256:902b1b88936918f9b2a19e0e5eb7ccb430ae45cde4f39ea4b36932920d33952d", size = 32861, upload-time = "2020-08-03T08:01:11.392Z" } + +[[package]] +name = "ipykernel" +version = "7.0.0a2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "appnope", marker = "sys_platform == 'darwin'" }, + { name = "comm" }, + { name = "debugpy" }, + { name = "ipython" }, + { name = "jupyter-client" }, + { name = "jupyter-core" }, + { name = "matplotlib-inline" }, + { name = "nest-asyncio" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyzmq" }, + { name = "tornado" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c1/1a/baaccb506ef660e9e3d420088e3a68f7873b1bcc152c0d069541641fd230/ipykernel-7.0.0a2.tar.gz", hash = "sha256:b3bd996d94ca09012b6df6654a086ddb3a01a40bcd1ad8cc1f5e80b327e0f120", size = 170826, upload-time = "2025-08-13T09:51:57.264Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/e3/c471afccaedd89576f5591f1622d70aa9f610194b9c4e91f4ff4b6d9d909/ipykernel-7.0.0a2-py3-none-any.whl", hash = "sha256:58336ba0d4c5ad0b9833392d544d882baf520ce8af1b714f89e746197bd7c644", size = 118216, upload-time = "2025-08-13T09:51:55.464Z" }, +] + +[[package]] +name = "ipynbname" +version = "2025.8.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ipykernel" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/e9/e791d0ec7125a41c5625dc348ee28c2e335bd79ff8b31e9971333f129337/ipynbname-2025.8.0.0.tar.gz", hash = "sha256:dcc8367c64c4a9f0baa6acea3e509fd6696bf5d81c9abceb386e2ad6efcac935", size = 4374, upload-time = "2025-08-17T16:43:28.581Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/93/aece384e71b25385bd6a176f09865ee52faa7bda8221acafff8753f3f30a/ipynbname-2025.8.0.0-py3-none-any.whl", hash = "sha256:66f6249ff26d99b0057db2aac5ced4ca8a2ccaad067b9a66d17cfc1725207966", size = 4534, upload-time = "2025-08-17T16:43:27.219Z" }, +] + +[[package]] +name = "ipython" +version = "9.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/54/80/406f9e3bde1c1fd9bf5a0be9d090f8ae623e401b7670d8f6fdf2ab679891/ipython-9.4.0.tar.gz", hash = "sha256:c033c6d4e7914c3d9768aabe76bbe87ba1dc66a92a05db6bfa1125d81f2ee270", size = 4385338, upload-time = "2025-07-01T11:11:30.606Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/f8/0031ee2b906a15a33d6bfc12dd09c3dfa966b3cb5b284ecfb7549e6ac3c4/ipython-9.4.0-py3-none-any.whl", hash = "sha256:25850f025a446d9b359e8d296ba175a36aedd32e83ca9b5060430fe16801f066", size = 611021, upload-time = "2025-07-01T11:11:27.85Z" }, +] + +[[package]] +name = "ipython-pygments-lexers" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl", hash = "sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c", size = 8074, upload-time = "2025-01-17T11:24:33.271Z" }, +] + [[package]] name = "isort" version = "5.12.0" @@ -1257,6 +1717,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" }, ] +[[package]] +name = "jedi" +version = "0.19.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "parso" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/3a/79a912fbd4d8dd6fbb02bf69afd3bb72cf0c729bb3063c6f4498603db17a/jedi-0.19.2.tar.gz", hash = "sha256:4770dc3de41bde3966b02eb84fbcf557fb33cce26ad23da12c742fb50ecb11f0", size = 1231287, upload-time = "2024-11-11T01:41:42.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl", hash = "sha256:a8ef22bde8490f57fe5c7681a3c83cb58874daf72b4784de3cce5b6ef6edb5b9", size = 1572278, upload-time = "2024-11-11T01:41:40.175Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -1321,11 +1793,11 @@ wheels = [ [[package]] name = "json5" -version = "0.12.0" +version = "0.12.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/12/be/c6c745ec4c4539b25a278b70e29793f10382947df0d9efba2fa09120895d/json5-0.12.0.tar.gz", hash = "sha256:0b4b6ff56801a1c7dc817b0241bca4ce474a0e6a163bfef3fc594d3fd263ff3a", size = 51907, upload-time = "2025-04-03T16:33:13.201Z" } +sdist = { url = "https://files.pythonhosted.org/packages/12/ae/929aee9619e9eba9015207a9d2c1c54db18311da7eb4dcf6d41ad6f0eb67/json5-0.12.1.tar.gz", hash = "sha256:b2743e77b3242f8d03c143dd975a6ec7c52e2f2afe76ed934e53503dd4ad4990", size = 52191, upload-time = "2025-08-12T19:47:42.583Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/41/9f/3500910d5a98549e3098807493851eeef2b89cdd3032227558a104dfe926/json5-0.12.0-py3-none-any.whl", hash = "sha256:6d37aa6c08b0609f16e1ec5ff94697e2cbbfbad5ac112afa05794da9ab7810db", size = 36079, upload-time = "2025-04-03T16:33:11.927Z" }, + { url = "https://files.pythonhosted.org/packages/85/e2/05328bd2621be49a6fed9e3030b1e51a2d04537d3f816d211b9cc53c5262/json5-0.12.1-py3-none-any.whl", hash = "sha256:d9c9b3bc34a5f54d43c35e11ef7cb87d8bdd098c6ace87117a7b7e83e705c1d5", size = 36119, upload-time = "2025-08-12T19:47:41.131Z" }, ] [[package]] @@ -1363,7 +1835,7 @@ wheels = [ [[package]] name = "jsonschema" -version = "4.24.0" +version = "4.25.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, @@ -1371,9 +1843,9 @@ dependencies = [ { name = "referencing" }, { name = "rpds-py" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bf/d3/1cf5326b923a53515d8f3a2cd442e6d7e94fcc444716e879ea70a0ce3177/jsonschema-4.24.0.tar.gz", hash = "sha256:0b4e8069eb12aedfa881333004bccaec24ecef5a8a6a4b6df142b2cc9599d196", size = 353480, upload-time = "2025-05-26T18:48:10.459Z" } +sdist = { url = "https://files.pythonhosted.org/packages/74/69/f7185de793a29082a9f3c7728268ffb31cb5095131a9c139a74078e27336/jsonschema-4.25.1.tar.gz", hash = "sha256:e4a9655ce0da0c0b67a085847e00a3a51449e1157f4f75e9fb5aa545e122eb85", size = 357342, upload-time = "2025-08-18T17:03:50.038Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/3d/023389198f69c722d039351050738d6755376c8fd343e91dc493ea485905/jsonschema-4.24.0-py3-none-any.whl", hash = "sha256:a462455f19f5faf404a7902952b6f0e3ce868f3ee09a359b05eca6673bd8412d", size = 88709, upload-time = "2025-05-26T18:48:08.417Z" }, + { url = "https://files.pythonhosted.org/packages/bf/9c/8c95d856233c1f82500c2450b8c68576b4cf1c871db3afac5c34ff84e6fd/jsonschema-4.25.1-py3-none-any.whl", hash = "sha256:3fba0169e345c7175110351d456342c364814cfcf3b964ba4587f22915230a63", size = 90040, upload-time = "2025-08-18T17:03:48.373Z" }, ] [[package]] @@ -1388,6 +1860,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/01/0e/b27cdbaccf30b890c40ed1da9fd4a3593a5cf94dae54fb34f8a4b74fcd3f/jsonschema_specifications-2025.4.1-py3-none-any.whl", hash = "sha256:4653bffbd6584f7de83a67e0d620ef16900b390ddc7939d56684d6c81e33f1af", size = 18437, upload-time = "2025-04-23T12:34:05.422Z" }, ] +[[package]] +name = "jupyter-client" +version = "8.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jupyter-core" }, + { name = "python-dateutil" }, + { name = "pyzmq" }, + { name = "tornado" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/22/bf9f12fdaeae18019a468b68952a60fe6dbab5d67cd2a103cac7659b41ca/jupyter_client-8.6.3.tar.gz", hash = "sha256:35b3a0947c4a6e9d589eb97d7d4cd5e90f910ee73101611f01283732bd6d9419", size = 342019, upload-time = "2024-09-17T10:44:17.613Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/85/b0394e0b6fcccd2c1eeefc230978a6f8cb0c5df1e4cd3e7625735a0d7d1e/jupyter_client-8.6.3-py3-none-any.whl", hash = "sha256:e8a19cc986cc45905ac3362915f410f3af85424b4c0905e94fa5f2cb08e8f23f", size = 106105, upload-time = "2024-09-17T10:44:15.218Z" }, +] + [[package]] name = "jupyter-core" version = "5.8.1" @@ -1402,47 +1890,73 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2f/57/6bffd4b20b88da3800c5d691e0337761576ee688eb01299eae865689d2df/jupyter_core-5.8.1-py3-none-any.whl", hash = "sha256:c28d268fc90fb53f1338ded2eb410704c5449a358406e8a948b75706e24863d0", size = 28880, upload-time = "2025-05-27T07:38:15.137Z" }, ] +[[package]] +name = "kaitaistruct" +version = "0.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/04/dd60b9cb65d580ef6cb6eaee975ad1bdd22d46a3f51b07a1e0606710ea88/kaitaistruct-0.10.tar.gz", hash = "sha256:a044dee29173d6afbacf27bcac39daf89b654dd418cfa009ab82d9178a9ae52a", size = 7061, upload-time = "2022-07-09T00:34:06.729Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/bf/88ad23efc08708bda9a2647169828e3553bb2093a473801db61f75356395/kaitaistruct-0.10-py2.py3-none-any.whl", hash = "sha256:a97350919adbf37fda881f75e9365e2fb88d04832b7a4e57106ec70119efb235", size = 7013, upload-time = "2022-07-09T00:34:03.905Z" }, +] + [[package]] name = "kiwisolver" -version = "1.4.8" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/59/7c91426a8ac292e1cdd53a63b6d9439abd573c875c3f92c146767dd33faf/kiwisolver-1.4.8.tar.gz", hash = "sha256:23d5f023bdc8c7e54eb65f03ca5d5bb25b601eac4d7f1a042888a1f45237987e", size = 97538, upload-time = "2024-12-24T18:30:51.519Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/ed/c913ee28936c371418cb167b128066ffb20bbf37771eecc2c97edf8a6e4c/kiwisolver-1.4.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a4d3601908c560bdf880f07d94f31d734afd1bb71e96585cace0e38ef44c6d84", size = 124635, upload-time = "2024-12-24T18:28:51.826Z" }, - { url = "https://files.pythonhosted.org/packages/4c/45/4a7f896f7467aaf5f56ef093d1f329346f3b594e77c6a3c327b2d415f521/kiwisolver-1.4.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:856b269c4d28a5c0d5e6c1955ec36ebfd1651ac00e1ce0afa3e28da95293b561", size = 66717, upload-time = "2024-12-24T18:28:54.256Z" }, - { url = "https://files.pythonhosted.org/packages/5f/b4/c12b3ac0852a3a68f94598d4c8d569f55361beef6159dce4e7b624160da2/kiwisolver-1.4.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c2b9a96e0f326205af81a15718a9073328df1173a2619a68553decb7097fd5d7", size = 65413, upload-time = "2024-12-24T18:28:55.184Z" }, - { url = "https://files.pythonhosted.org/packages/a9/98/1df4089b1ed23d83d410adfdc5947245c753bddfbe06541c4aae330e9e70/kiwisolver-1.4.8-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c5020c83e8553f770cb3b5fc13faac40f17e0b205bd237aebd21d53d733adb03", size = 1343994, upload-time = "2024-12-24T18:28:57.493Z" }, - { url = "https://files.pythonhosted.org/packages/8d/bf/b4b169b050c8421a7c53ea1ea74e4ef9c335ee9013216c558a047f162d20/kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dace81d28c787956bfbfbbfd72fdcef014f37d9b48830829e488fdb32b49d954", size = 1434804, upload-time = "2024-12-24T18:29:00.077Z" }, - { url = "https://files.pythonhosted.org/packages/66/5a/e13bd341fbcf73325ea60fdc8af752addf75c5079867af2e04cc41f34434/kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:11e1022b524bd48ae56c9b4f9296bce77e15a2e42a502cceba602f804b32bb79", size = 1450690, upload-time = "2024-12-24T18:29:01.401Z" }, - { url = "https://files.pythonhosted.org/packages/9b/4f/5955dcb376ba4a830384cc6fab7d7547bd6759fe75a09564910e9e3bb8ea/kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b9b4d2892fefc886f30301cdd80debd8bb01ecdf165a449eb6e78f79f0fabd6", size = 1376839, upload-time = "2024-12-24T18:29:02.685Z" }, - { url = "https://files.pythonhosted.org/packages/3a/97/5edbed69a9d0caa2e4aa616ae7df8127e10f6586940aa683a496c2c280b9/kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a96c0e790ee875d65e340ab383700e2b4891677b7fcd30a699146f9384a2bb0", size = 1435109, upload-time = "2024-12-24T18:29:04.113Z" }, - { url = "https://files.pythonhosted.org/packages/13/fc/e756382cb64e556af6c1809a1bbb22c141bbc2445049f2da06b420fe52bf/kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:23454ff084b07ac54ca8be535f4174170c1094a4cff78fbae4f73a4bcc0d4dab", size = 2245269, upload-time = "2024-12-24T18:29:05.488Z" }, - { url = "https://files.pythonhosted.org/packages/76/15/e59e45829d7f41c776d138245cabae6515cb4eb44b418f6d4109c478b481/kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:87b287251ad6488e95b4f0b4a79a6d04d3ea35fde6340eb38fbd1ca9cd35bbbc", size = 2393468, upload-time = "2024-12-24T18:29:06.79Z" }, - { url = "https://files.pythonhosted.org/packages/e9/39/483558c2a913ab8384d6e4b66a932406f87c95a6080112433da5ed668559/kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b21dbe165081142b1232a240fc6383fd32cdd877ca6cc89eab93e5f5883e1c25", size = 2355394, upload-time = "2024-12-24T18:29:08.24Z" }, - { url = "https://files.pythonhosted.org/packages/01/aa/efad1fbca6570a161d29224f14b082960c7e08268a133fe5dc0f6906820e/kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:768cade2c2df13db52475bd28d3a3fac8c9eff04b0e9e2fda0f3760f20b3f7fc", size = 2490901, upload-time = "2024-12-24T18:29:09.653Z" }, - { url = "https://files.pythonhosted.org/packages/c9/4f/15988966ba46bcd5ab9d0c8296914436720dd67fca689ae1a75b4ec1c72f/kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d47cfb2650f0e103d4bf68b0b5804c68da97272c84bb12850d877a95c056bd67", size = 2312306, upload-time = "2024-12-24T18:29:12.644Z" }, - { url = "https://files.pythonhosted.org/packages/2d/27/bdf1c769c83f74d98cbc34483a972f221440703054894a37d174fba8aa68/kiwisolver-1.4.8-cp311-cp311-win_amd64.whl", hash = "sha256:ed33ca2002a779a2e20eeb06aea7721b6e47f2d4b8a8ece979d8ba9e2a167e34", size = 71966, upload-time = "2024-12-24T18:29:14.089Z" }, - { url = "https://files.pythonhosted.org/packages/4a/c9/9642ea855604aeb2968a8e145fc662edf61db7632ad2e4fb92424be6b6c0/kiwisolver-1.4.8-cp311-cp311-win_arm64.whl", hash = "sha256:16523b40aab60426ffdebe33ac374457cf62863e330a90a0383639ce14bf44b2", size = 65311, upload-time = "2024-12-24T18:29:15.892Z" }, - { url = "https://files.pythonhosted.org/packages/fc/aa/cea685c4ab647f349c3bc92d2daf7ae34c8e8cf405a6dcd3a497f58a2ac3/kiwisolver-1.4.8-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d6af5e8815fd02997cb6ad9bbed0ee1e60014438ee1a5c2444c96f87b8843502", size = 124152, upload-time = "2024-12-24T18:29:16.85Z" }, - { url = "https://files.pythonhosted.org/packages/c5/0b/8db6d2e2452d60d5ebc4ce4b204feeb16176a851fd42462f66ade6808084/kiwisolver-1.4.8-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bade438f86e21d91e0cf5dd7c0ed00cda0f77c8c1616bd83f9fc157fa6760d31", size = 66555, upload-time = "2024-12-24T18:29:19.146Z" }, - { url = "https://files.pythonhosted.org/packages/60/26/d6a0db6785dd35d3ba5bf2b2df0aedc5af089962c6eb2cbf67a15b81369e/kiwisolver-1.4.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b83dc6769ddbc57613280118fb4ce3cd08899cc3369f7d0e0fab518a7cf37fdb", size = 65067, upload-time = "2024-12-24T18:29:20.096Z" }, - { url = "https://files.pythonhosted.org/packages/c9/ed/1d97f7e3561e09757a196231edccc1bcf59d55ddccefa2afc9c615abd8e0/kiwisolver-1.4.8-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:111793b232842991be367ed828076b03d96202c19221b5ebab421ce8bcad016f", size = 1378443, upload-time = "2024-12-24T18:29:22.843Z" }, - { url = "https://files.pythonhosted.org/packages/29/61/39d30b99954e6b46f760e6289c12fede2ab96a254c443639052d1b573fbc/kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:257af1622860e51b1a9d0ce387bf5c2c4f36a90594cb9514f55b074bcc787cfc", size = 1472728, upload-time = "2024-12-24T18:29:24.463Z" }, - { url = "https://files.pythonhosted.org/packages/0c/3e/804163b932f7603ef256e4a715e5843a9600802bb23a68b4e08c8c0ff61d/kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:69b5637c3f316cab1ec1c9a12b8c5f4750a4c4b71af9157645bf32830e39c03a", size = 1478388, upload-time = "2024-12-24T18:29:25.776Z" }, - { url = "https://files.pythonhosted.org/packages/8a/9e/60eaa75169a154700be74f875a4d9961b11ba048bef315fbe89cb6999056/kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:782bb86f245ec18009890e7cb8d13a5ef54dcf2ebe18ed65f795e635a96a1c6a", size = 1413849, upload-time = "2024-12-24T18:29:27.202Z" }, - { url = "https://files.pythonhosted.org/packages/bc/b3/9458adb9472e61a998c8c4d95cfdfec91c73c53a375b30b1428310f923e4/kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc978a80a0db3a66d25767b03688f1147a69e6237175c0f4ffffaaedf744055a", size = 1475533, upload-time = "2024-12-24T18:29:28.638Z" }, - { url = "https://files.pythonhosted.org/packages/e4/7a/0a42d9571e35798de80aef4bb43a9b672aa7f8e58643d7bd1950398ffb0a/kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:36dbbfd34838500a31f52c9786990d00150860e46cd5041386f217101350f0d3", size = 2268898, upload-time = "2024-12-24T18:29:30.368Z" }, - { url = "https://files.pythonhosted.org/packages/d9/07/1255dc8d80271400126ed8db35a1795b1a2c098ac3a72645075d06fe5c5d/kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:eaa973f1e05131de5ff3569bbba7f5fd07ea0595d3870ed4a526d486fe57fa1b", size = 2425605, upload-time = "2024-12-24T18:29:33.151Z" }, - { url = "https://files.pythonhosted.org/packages/84/df/5a3b4cf13780ef6f6942df67b138b03b7e79e9f1f08f57c49957d5867f6e/kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a66f60f8d0c87ab7f59b6fb80e642ebb29fec354a4dfad687ca4092ae69d04f4", size = 2375801, upload-time = "2024-12-24T18:29:34.584Z" }, - { url = "https://files.pythonhosted.org/packages/8f/10/2348d068e8b0f635c8c86892788dac7a6b5c0cb12356620ab575775aad89/kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:858416b7fb777a53f0c59ca08190ce24e9abbd3cffa18886a5781b8e3e26f65d", size = 2520077, upload-time = "2024-12-24T18:29:36.138Z" }, - { url = "https://files.pythonhosted.org/packages/32/d8/014b89fee5d4dce157d814303b0fce4d31385a2af4c41fed194b173b81ac/kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:085940635c62697391baafaaeabdf3dd7a6c3643577dde337f4d66eba021b2b8", size = 2338410, upload-time = "2024-12-24T18:29:39.991Z" }, - { url = "https://files.pythonhosted.org/packages/bd/72/dfff0cc97f2a0776e1c9eb5bef1ddfd45f46246c6533b0191887a427bca5/kiwisolver-1.4.8-cp312-cp312-win_amd64.whl", hash = "sha256:01c3d31902c7db5fb6182832713d3b4122ad9317c2c5877d0539227d96bb2e50", size = 71853, upload-time = "2024-12-24T18:29:42.006Z" }, - { url = "https://files.pythonhosted.org/packages/dc/85/220d13d914485c0948a00f0b9eb419efaf6da81b7d72e88ce2391f7aed8d/kiwisolver-1.4.8-cp312-cp312-win_arm64.whl", hash = "sha256:a3c44cb68861de93f0c4a8175fbaa691f0aa22550c331fefef02b618a9dcb476", size = 65424, upload-time = "2024-12-24T18:29:44.38Z" }, +version = "1.4.10rc0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/de/354c903d772c1cc0a9310344e077b31c6c893cc5a664019b907a04997099/kiwisolver-1.4.10rc0.tar.gz", hash = "sha256:d321718aaa2583577be9836e8cc0ed9fd0863e57a85b1b73b328aac063bc9903", size = 97614, upload-time = "2025-08-10T20:22:27.702Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/21/019c64c58655e6a3b3783197238c92edf8090e30d094d9c2770e50312c85/kiwisolver-1.4.10rc0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7cc57b12496996be722d18b93fb145098c98b7f49cbe06eb92f1a8839fec851f", size = 124257, upload-time = "2025-08-10T20:20:23.798Z" }, + { url = "https://files.pythonhosted.org/packages/07/c5/a1459a96995a804ffcb587fa53006b4f598bd2b757e60811cc3538829852/kiwisolver-1.4.10rc0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ed726e9bf4ab6d8bf63f214189a080b1ccc2236de8f69a65518ec2e08792d809", size = 66646, upload-time = "2025-08-10T20:20:24.831Z" }, + { url = "https://files.pythonhosted.org/packages/f8/0f/6312411778e5d84df581bc67036e5da913013169d3e819ff3fde02766fa9/kiwisolver-1.4.10rc0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:22954717039cdf96df76c2583be41fc211a87375e4d32eb2bbdbd071d2ce0dbb", size = 65386, upload-time = "2025-08-10T20:20:25.845Z" }, + { url = "https://files.pythonhosted.org/packages/a3/03/5e917f0bbfcabcab2d46f22da2eda240fb18a15098e1ff75cc4242dc6807/kiwisolver-1.4.10rc0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6d1c2b921dcb8dd3c8b8caeb942e8dde2f9efbb4cce5a7cc4ecb119156dc6f63", size = 1435659, upload-time = "2025-08-10T20:20:26.96Z" }, + { url = "https://files.pythonhosted.org/packages/5c/92/e47df418bf28fb7371fd6cc716ac3765b6048010c3f79ca46c97b4f67631/kiwisolver-1.4.10rc0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8655f912cf62c65176ca04e15b937c03c41da59090fb177885a9aa413f3e27c0", size = 1246594, upload-time = "2025-08-10T20:20:28.807Z" }, + { url = "https://files.pythonhosted.org/packages/b2/f8/ff201f9d0a620f830fa624808376a502cd7ab70f3db75e8f0bef8e22854a/kiwisolver-1.4.10rc0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0a6487724ff217c96fcd072289a5b9e48b88ded72d48cfa48105382ac5cae7b6", size = 1263691, upload-time = "2025-08-10T20:20:30.512Z" }, + { url = "https://files.pythonhosted.org/packages/af/6b/1fa0c58baa765b93e50809c6d05d451592e23b4b1f3ed7c3aa84e5043543/kiwisolver-1.4.10rc0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fc089ea753b3d23063c5791cfbf781049a885372d8090ed3176f14aa95e8b133", size = 1317497, upload-time = "2025-08-10T20:20:32.318Z" }, + { url = "https://files.pythonhosted.org/packages/11/82/7be342e8384ed7efb91c1a568190d64eab42456619266121047d82da802e/kiwisolver-1.4.10rc0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0ef8610afeec79ae3cbd9cfa36d4520c672fe0615a2a0ddbce235a510736b35d", size = 2195834, upload-time = "2025-08-10T20:20:34.101Z" }, + { url = "https://files.pythonhosted.org/packages/d8/e9/d8a0fd5c4aa341f06258d544ac568f9fcdf4f67ee662b9475e72b42c8808/kiwisolver-1.4.10rc0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:481e143b69de966593116e54e5033ee1fbf6d0c08d557b45fd4319b2011773ba", size = 2290888, upload-time = "2025-08-10T20:20:36.981Z" }, + { url = "https://files.pythonhosted.org/packages/a4/1f/5d4abbceee0facefb7a90ab4fec6966886448323143bd967518c45c92e79/kiwisolver-1.4.10rc0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:f339d6f008fe5a175908e2c5d1a7204bfce3da4250f9bf70df5a60a4b16a8760", size = 2461638, upload-time = "2025-08-10T20:20:38.418Z" }, + { url = "https://files.pythonhosted.org/packages/86/d8/9c5f0bc3222ae58674b24ce8262206f59f79aecaf0b1583ff5fe495e9ff0/kiwisolver-1.4.10rc0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f80f88fa20f14817018eab0f3db70e93da4719434f5c16a0058bde6544c26675", size = 2268133, upload-time = "2025-08-10T20:20:40.384Z" }, + { url = "https://files.pythonhosted.org/packages/ec/d1/82c1e2964bb1b11160afcc692d932be95ebdfe2bc0756cd1f8dee79a2346/kiwisolver-1.4.10rc0-cp311-cp311-win_amd64.whl", hash = "sha256:ef95d952120e64a55d633a1f2973392acd42bced2e7bf3b76816f7dcca5a88a6", size = 73888, upload-time = "2025-08-10T20:20:42.37Z" }, + { url = "https://files.pythonhosted.org/packages/61/a2/d4867a3ae81e2fe28de05f25266d5a6e0f4950e63f050d3cd78cb48bdfb4/kiwisolver-1.4.10rc0-cp311-cp311-win_arm64.whl", hash = "sha256:9453be53a21f813c0472db8a5cea26e8cadc315c5002aa9882fad8bf5212483e", size = 65199, upload-time = "2025-08-10T20:20:43.372Z" }, + { url = "https://files.pythonhosted.org/packages/dc/87/3df31abf12db3ccabfa52a96dc49e6defe233d8ffca1361091a1ea3a109c/kiwisolver-1.4.10rc0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1cb9ae443b2dba2229ac3b8a771420ee76bfce56f610dcb4998676cebed79346", size = 123742, upload-time = "2025-08-10T20:20:44.391Z" }, + { url = "https://files.pythonhosted.org/packages/7b/62/fc9adfd88082b95971969736d777762f7940f3d49f5ffae37c439699156d/kiwisolver-1.4.10rc0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2727d842ff63050315d2a69334d64ed4ada261c04155c09d9534fd4033d38641", size = 66522, upload-time = "2025-08-10T20:20:45.81Z" }, + { url = "https://files.pythonhosted.org/packages/7d/d1/3802735c705ffa861bbb568ed4226936fcfc917a179bf3998fdf97e48e57/kiwisolver-1.4.10rc0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f9563b4c98c23f52d98e15be40cbf0c215c824e7268d5222e9ad1803e8c1156d", size = 65016, upload-time = "2025-08-10T20:20:46.911Z" }, + { url = "https://files.pythonhosted.org/packages/72/76/29b4d717f5614fc91cb4542cd67f42b5bcb6c946201a9cf9d4a34231efc2/kiwisolver-1.4.10rc0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:75cef7209e3c71dc81d1d5bc8d5eb1b34be7a8d2cd94b83f0eb15533512a45ef", size = 1474820, upload-time = "2025-08-10T20:20:48.447Z" }, + { url = "https://files.pythonhosted.org/packages/f1/9b/828761ad3841b0e7d464514939a980eef7547b5952fb9a33232b17ed1540/kiwisolver-1.4.10rc0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:374c11179eaacd3b8bfef677aa28a0a6d703b3474ea399f3b08b8e4d67522016", size = 1276468, upload-time = "2025-08-10T20:20:49.775Z" }, + { url = "https://files.pythonhosted.org/packages/92/6d/cb780c0ebad56e2b7b3c2c0376e9d5c25e90680d2b3b254afaec2507a62f/kiwisolver-1.4.10rc0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9abc647f46a322fd19e0564ce59fcc0f14b9934540ddc7404ed7f3eea54d0d11", size = 1294484, upload-time = "2025-08-10T20:20:51.207Z" }, + { url = "https://files.pythonhosted.org/packages/55/70/44814b447c4e38da6f466b6a3e992d330c3e2c1c9c29731c436997b78f68/kiwisolver-1.4.10rc0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7e3c788da96b50e60f484d9dd790554a80800c22a468cd059b9a7a9c753d273", size = 1343677, upload-time = "2025-08-10T20:20:52.906Z" }, + { url = "https://files.pythonhosted.org/packages/5c/54/44eee9dc53be9c4c6ac3b099aedc482f1a1a6b193d0f258ccfa955c291df/kiwisolver-1.4.10rc0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:468814c0e8f41b8f7b537da0c77a05a70f89aa4b7cfff96aa47f7936e57add9b", size = 2225010, upload-time = "2025-08-10T20:20:54.365Z" }, + { url = "https://files.pythonhosted.org/packages/50/5e/e05e24f858352e6985ace1f9ab8ece32b0962f4c5074ddb38fc91617809a/kiwisolver-1.4.10rc0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:aa62a0aa711b1f94f80f1407a668362718c64f27e176ac6952d983ec1a5cd745", size = 2321356, upload-time = "2025-08-10T20:20:55.803Z" }, + { url = "https://files.pythonhosted.org/packages/f1/a7/1d21b9aa468bdbd4be190043d73ad07756c078003e43fdc4af5ccb3df75a/kiwisolver-1.4.10rc0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:6e8697067105c536bcc14a33f5c9c0f0c155cf909818d01475f45d012ac441c1", size = 2488062, upload-time = "2025-08-10T20:20:57.371Z" }, + { url = "https://files.pythonhosted.org/packages/88/9b/17512a47070d022499f19078b980531b7be5d50eb9990dfc4ec29aa554ca/kiwisolver-1.4.10rc0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4c168de06cd8a4947a8af1e49d63341face78aca8e9e6b7685625701147ab22d", size = 2291890, upload-time = "2025-08-10T20:20:59.237Z" }, + { url = "https://files.pythonhosted.org/packages/85/c1/084d9b537e33555d8bf5d41ffaee88cf0ee49fa42587fdee181d31a40b61/kiwisolver-1.4.10rc0-cp312-cp312-win_amd64.whl", hash = "sha256:0b8bb7b6b3964d0454f8504e003097f2ae628679a1054ecb63578feeb7671cab", size = 73949, upload-time = "2025-08-10T20:21:00.845Z" }, + { url = "https://files.pythonhosted.org/packages/ff/fa/538442202d639add2f52a814bdfc58207ee6fbb6d1ecd1a6e867f48ec1af/kiwisolver-1.4.10rc0-cp312-cp312-win_arm64.whl", hash = "sha256:44cd6dfea8a6c2becac4f3d60ebdcfe4fed858bbf7fe9cd38ffea7b58df66435", size = 65077, upload-time = "2025-08-10T20:21:01.882Z" }, + { url = "https://files.pythonhosted.org/packages/bb/31/93f123de35b4e62a9debc1d4f35e20e5da42cc8b99824a13b670beaf426e/kiwisolver-1.4.10rc0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:81b5d3e873fed82e9ff7b45311c5fa961a191dda6f19898fce6385ecc82ea65e", size = 60156, upload-time = "2025-08-10T20:22:21.821Z" }, + { url = "https://files.pythonhosted.org/packages/bd/92/2777e966406f0ba01e3f7faae07a79d7a0cc530937fdb2883679a5d10eed/kiwisolver-1.4.10rc0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:c5645e473af736949870d303d51d6edf2925afcd56ad5779b29bf7e5c620dc86", size = 58657, upload-time = "2025-08-10T20:22:22.903Z" }, + { url = "https://files.pythonhosted.org/packages/86/90/3b73b9a069cf64bd761c12ffd53b93e212a82f7fd36057ad1b8ef0a27399/kiwisolver-1.4.10rc0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:81c6204ddf60b409e384a9dfd12f70980bd98556e9c46f681ea40c90201ac236", size = 80335, upload-time = "2025-08-10T20:22:24.371Z" }, + { url = "https://files.pythonhosted.org/packages/42/0e/08307c00313e305336f1faa8c332b8042877670bbf950535643b3310f7d1/kiwisolver-1.4.10rc0-pp311-pypy311_pp73-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b52feaf694434ba3aea0cfb9515b72d5cbe8c555473f14fc5dba121ced58d73d", size = 78068, upload-time = "2025-08-10T20:22:25.433Z" }, + { url = "https://files.pythonhosted.org/packages/26/05/ed2ad330b22530772f0498431d6f589a18c5eb3bd858da577f1b2ef5980e/kiwisolver-1.4.10rc0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:7b4aa27204cb091a4ef96438773c9609f24f217fb3cd53612c41394f39b0d8b6", size = 73983, upload-time = "2025-08-10T20:22:26.498Z" }, +] + +[[package]] +name = "koji" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "defusedxml" }, + { name = "python-dateutil" }, + { name = "requests" }, + { name = "requests-gssapi" }, + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/4d/6ccfb241b9b8a2809cbbd74dcc51ec2520b87222c5883aee0efc0ee88435/koji-1.36.0.tar.gz", hash = "sha256:481552b9ec9aad55e393c6c5f92e834bc2288e1da01ea67ee48ca63466c4049e", size = 225085, upload-time = "2026-03-23T22:13:34.242Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/8a/37cf0308e10684e36505e870b37fe22142b4cd2d2c16a894c4fda7891b72/koji-1.36.0-py3-none-any.whl", hash = "sha256:7aeb7932731106e489cd61c471b689aeb80c16583d5eded46f4566696fd5a157", size = 232156, upload-time = "2026-03-23T22:13:32.824Z" }, ] [[package]] name = "langchain" -version = "0.3.26" +version = "0.3.27" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, @@ -1453,14 +1967,14 @@ dependencies = [ { name = "requests" }, { name = "sqlalchemy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7f/13/a9931800ee42bbe0f8850dd540de14e80dda4945e7ee36e20b5d5964286e/langchain-0.3.26.tar.gz", hash = "sha256:8ff034ee0556d3e45eff1f1e96d0d745ced57858414dba7171c8ebdbeb5580c9", size = 10226808, upload-time = "2025-06-20T22:23:01.174Z" } +sdist = { url = "https://files.pythonhosted.org/packages/83/f6/f4f7f3a56626fe07e2bb330feb61254dbdf06c506e6b59a536a337da51cf/langchain-0.3.27.tar.gz", hash = "sha256:aa6f1e6274ff055d0fd36254176770f356ed0a8994297d1df47df341953cec62", size = 10233809, upload-time = "2025-07-24T14:42:32.959Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f1/f2/c09a2e383283e3af1db669ab037ac05a45814f4b9c472c48dc24c0cef039/langchain-0.3.26-py3-none-any.whl", hash = "sha256:361bb2e61371024a8c473da9f9c55f4ee50f269c5ab43afdb2b1309cb7ac36cf", size = 1012336, upload-time = "2025-06-20T22:22:58.874Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d5/4861816a95b2f6993f1360cfb605aacb015506ee2090433a71de9cca8477/langchain-0.3.27-py3-none-any.whl", hash = "sha256:7b20c4f338826acb148d885b20a73a16e410ede9ee4f19bb02011852d5f98798", size = 1018194, upload-time = "2025-07-24T14:42:30.23Z" }, ] [[package]] name = "langchain-aws" -version = "0.2.27" +version = "0.2.30" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "boto3" }, @@ -1468,9 +1982,9 @@ dependencies = [ { name = "numpy" }, { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/60/ff/7bfa3e508d7b4f7406c755f93d1db03f5822e91a29a31035bd369d7878fd/langchain_aws-0.2.27.tar.gz", hash = "sha256:3d08c099dafb6536c74cba7de393845a314086ead73a5821fde52d435be34a47", size = 99349, upload-time = "2025-06-24T13:39:46.012Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/58/a3071e27315017fe2320a3d39a1a6fb081a78868034488f3fefec61df971/langchain_aws-0.2.30.tar.gz", hash = "sha256:67c31f0784045a4a73ef78a2c18f392e14c1e9f7b55870e62f18039cadfb925c", size = 109960, upload-time = "2025-07-31T22:02:11.4Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/44/3dec0bfe8666b2b906a3bbec8739a2cb93aee516c032bce8a3ed41957ccd/langchain_aws-0.2.27-py3-none-any.whl", hash = "sha256:d2912c4488ff5ef0361661c1a42476b3586c27c8e86c0c7e3a198e5dc9b8ae6f", size = 121204, upload-time = "2025-06-24T13:39:45.046Z" }, + { url = "https://files.pythonhosted.org/packages/c6/01/368ddf5c488e1a7186a8e72c8687b7c35479ad66f0fe2dcc00f23f35109e/langchain_aws-0.2.30-py3-none-any.whl", hash = "sha256:947fe4ece30bde0a37ea721b87049d7642607bd2ba9d2d93c9890736972b4274", size = 133900, upload-time = "2025-07-31T22:02:10.38Z" }, ] [[package]] @@ -1498,7 +2012,7 @@ wheels = [ [[package]] name = "langchain-core" -version = "0.3.69" +version = "0.3.74" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonpatch" }, @@ -1509,9 +2023,9 @@ dependencies = [ { name = "tenacity" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/82/26/c4770d3933237cde2918d502e3b0a8b6ce100b296840b632658f3e59b341/langchain_core-0.3.69.tar.gz", hash = "sha256:c132961117cc7f0227a4c58dd3e209674a6dd5b7e74abc61a0df93b0d736e283", size = 563824, upload-time = "2025-07-15T21:19:56.626Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/c6/5d755a0f1f4857abbe5ea6f5907ed0e2b5df52bf4dde0a0fd768290e3084/langchain_core-0.3.74.tar.gz", hash = "sha256:ff604441aeade942fbcc0a3860a592daba7671345230c2078ba2eb5f82b6ba76", size = 569553, upload-time = "2025-08-07T20:47:05.094Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/51/7b/bb7b088440ff9cc55e9e6eba94162cbdcd3b1693c194e1ad4764acba29b9/langchain_core-0.3.69-py3-none-any.whl", hash = "sha256:383e9cb4919f7ef4b24bf8552ef42e4323c064924fea88b28dd5d7ddb740d3b8", size = 441556, upload-time = "2025-07-15T21:19:55.342Z" }, + { url = "https://files.pythonhosted.org/packages/4d/26/545283681ac0379d31c7ad0bac5f195e1982092d76c65ca048db9e3cec0e/langchain_core-0.3.74-py3-none-any.whl", hash = "sha256:088338b5bc2f6a66892f9afc777992c24ee3188f41cbc603d09181e34a228ce7", size = 443453, upload-time = "2025-08-07T20:47:03.853Z" }, ] [[package]] @@ -1529,42 +2043,54 @@ wheels = [ [[package]] name = "langchain-nvidia-ai-endpoints" -version = "0.3.11" +version = "0.3.16" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, { name = "filetype" }, { name = "langchain-core" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d1/69/255f8468d12cf77cca14debc350bbeb3ed42092775fe353b3eb3e6eacbc4/langchain_nvidia_ai_endpoints-0.3.11.tar.gz", hash = "sha256:df06727a7876813e262591c5ca0b1f48052df8dad80de9b2730c799bdb854ed1", size = 38741, upload-time = "2025-07-14T20:52:21.601Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/f0/9be96dbd50faf9d5053fa82beb51c27f19d9da00e07a43755e4fc21c728d/langchain_nvidia_ai_endpoints-0.3.16.tar.gz", hash = "sha256:8c4aafd125284ef12668e5428e18b83864fb44a4677dcf8b456454e45cb1e7b0", size = 40184, upload-time = "2025-08-08T22:18:22.016Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/5f/47377adf27756b5b469ec1342113edbcfaa86660cca02637a8eddedd853b/langchain_nvidia_ai_endpoints-0.3.11-py3-none-any.whl", hash = "sha256:73253b56e9349dffe5bac633aeb3945375952238bb194623bf043c9f54489e62", size = 42343, upload-time = "2025-07-14T20:52:20.684Z" }, + { url = "https://files.pythonhosted.org/packages/95/68/e40626a8c78895e5fa8cea0f29b80cbdaffba3c9c6eb90532780315ee8e8/langchain_nvidia_ai_endpoints-0.3.16-py3-none-any.whl", hash = "sha256:a8c1c8a316668ff8402b89a97ace5f978ee71e351a487abbc5aa8c47f576e7d0", size = 43572, upload-time = "2025-08-08T22:18:20.873Z" }, ] [[package]] name = "langchain-openai" -version = "0.3.28" +version = "0.3.30" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, { name = "openai" }, { name = "tiktoken" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6b/1d/90cd764c62d5eb822113d3debc3abe10c8807d2c0af90917bfe09acd6f86/langchain_openai-0.3.28.tar.gz", hash = "sha256:6c669548dbdea325c034ae5ef699710e2abd054c7354fdb3ef7bf909dc739d9e", size = 753951, upload-time = "2025-07-14T10:50:44.076Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ad/21/6b2024cdd907812d33d31d42c05baa6a3fc6b341d76f7a982730b6985501/langchain_openai-0.3.30.tar.gz", hash = "sha256:90df37509b2dcf5e057f491326fcbf78cf2a71caff5103a5a7de560320171842", size = 766426, upload-time = "2025-08-12T17:05:55.587Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/91/56/75f3d84b69b8bdae521a537697375e1241377627c32b78edcae337093502/langchain_openai-0.3.28-py3-none-any.whl", hash = "sha256:4cd6d80a5b2ae471a168017bc01b2e0f01548328d83532400a001623624ede67", size = 70571, upload-time = "2025-07-14T10:50:42.492Z" }, + { url = "https://files.pythonhosted.org/packages/23/36/cd370071243ae321c22bfafbf75fef1601dd22d0baeeedb71835954ed0ad/langchain_openai-0.3.30-py3-none-any.whl", hash = "sha256:280f1f31004393228e3f75ff8353b1aae86bbc282abc7890a05beb5f43b89923", size = 74362, upload-time = "2025-08-12T17:05:54.415Z" }, ] [[package]] name = "langchain-text-splitters" -version = "0.3.8" +version = "0.3.9" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e7/ac/b4a25c5716bb0103b1515f1f52cc69ffb1035a5a225ee5afe3aed28bf57b/langchain_text_splitters-0.3.8.tar.gz", hash = "sha256:116d4b9f2a22dda357d0b79e30acf005c5518177971c66a9f1ab0edfdb0f912e", size = 42128, upload-time = "2025-04-04T14:03:51.521Z" } +sdist = { url = "https://files.pythonhosted.org/packages/91/52/d43ad77acae169210cc476cbc1e4ab37a701017c950211a11ab500fe7d7e/langchain_text_splitters-0.3.9.tar.gz", hash = "sha256:7cd1e5a3aaf609979583eeca2eb34177622570b8fa8f586a605c6b1c34e7ebdb", size = 45260, upload-time = "2025-07-24T14:38:45.14Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/52/7638394b88bc15083fd2c3752a843784d9d2d110d68fed6437c8607fb749/langchain_text_splitters-0.3.9-py3-none-any.whl", hash = "sha256:cee0bb816211584ea79cc79927317c358543f40404bcfdd69e69ba3ccde54401", size = 33314, upload-time = "2025-07-24T14:38:43.953Z" }, +] + +[[package]] +name = "langcodes" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "language-data" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/7a/5a97e327063409a5caa21541e6d08ae4a0f2da328447e9f2c7b39e179226/langcodes-3.5.0.tar.gz", hash = "sha256:1eef8168d07e51e131a2497ffecad4b663f6208e7c3ae3b8dc15c51734a6f801", size = 191030, upload-time = "2024-11-19T10:23:45.546Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/a3/3696ff2444658053c01b6b7443e761f28bb71217d82bb89137a978c5f66f/langchain_text_splitters-0.3.8-py3-none-any.whl", hash = "sha256:e75cc0f4ae58dcf07d9f18776400cf8ade27fadd4ff6d264df6278bb302f6f02", size = 32440, upload-time = "2025-04-04T14:03:50.6Z" }, + { url = "https://files.pythonhosted.org/packages/c3/6b/068c2ea7a712bf805c62445bd9e9c06d7340358ef2824150eceac027444b/langcodes-3.5.0-py3-none-any.whl", hash = "sha256:853c69d1a35e0e13da2f427bb68fb2fa4a8f4fb899e0c62ad8df8d073dcfed33", size = 182974, upload-time = "2024-11-19T10:23:42.824Z" }, ] [[package]] @@ -1583,33 +2109,33 @@ wheels = [ [[package]] name = "langgraph-checkpoint" -version = "2.1.0" +version = "2.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, { name = "ormsgpack" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f9/30/c04abcb2ac30f200dbfde5839ca3832552fe2bd852d9e85a68e47418a11c/langgraph_checkpoint-2.1.0.tar.gz", hash = "sha256:cdaa2f0b49aa130ab185c02d82f02b40299a1fbc9ac59ac20cecce09642a1abe", size = 135501, upload-time = "2025-06-16T22:05:01.918Z" } +sdist = { url = "https://files.pythonhosted.org/packages/73/3e/d00eb2b56c3846a0cabd2e5aa71c17a95f882d4f799a6ffe96a19b55eba9/langgraph_checkpoint-2.1.1.tar.gz", hash = "sha256:72038c0f9e22260cb9bff1f3ebe5eb06d940b7ee5c1e4765019269d4f21cf92d", size = 136256, upload-time = "2025-07-17T13:07:52.411Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/41/390a97d9d0abe5b71eea2f6fb618d8adadefa674e97f837bae6cda670bc7/langgraph_checkpoint-2.1.0-py3-none-any.whl", hash = "sha256:4cea3e512081da1241396a519cbfe4c5d92836545e2c64e85b6f5c34a1b8bc61", size = 43844, upload-time = "2025-06-16T22:05:00.758Z" }, + { url = "https://files.pythonhosted.org/packages/4c/dd/64686797b0927fb18b290044be12ae9d4df01670dce6bb2498d5ab65cb24/langgraph_checkpoint-2.1.1-py3-none-any.whl", hash = "sha256:5a779134fd28134a9a83d078be4450bbf0e0c79fdf5e992549658899e6fc5ea7", size = 43925, upload-time = "2025-07-17T13:07:51.023Z" }, ] [[package]] name = "langgraph-sdk" -version = "0.1.73" +version = "0.1.74" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, { name = "orjson" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ba/e8/daf0271f91e93b10566533955c00ee16e471066755c2efd1ba9a887a7eab/langgraph_sdk-0.1.73.tar.gz", hash = "sha256:6e6dcdf66bcf8710739899616856527a72a605ce15beb76fbac7f4ce0e2ad080", size = 72157, upload-time = "2025-07-14T23:57:22.765Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/f7/3807b72988f7eef5e0eb41e7e695eca50f3ed31f7cab5602db3b651c85ff/langgraph_sdk-0.1.74.tar.gz", hash = "sha256:7450e0db5b226cc2e5328ca22c5968725873630ef47c4206a30707cb25dc3ad6", size = 72190, upload-time = "2025-07-21T16:36:50.032Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/77/86/56e01e715e5b0028cdaff1492a89e54fa12e18c21e03b805a10ea36ecd5a/langgraph_sdk-0.1.73-py3-none-any.whl", hash = "sha256:a60ac33f70688ad07051edff1d5ed8089c8f0de1f69dc900be46e095ca20eed8", size = 50222, upload-time = "2025-07-14T23:57:21.42Z" }, + { url = "https://files.pythonhosted.org/packages/1f/1a/3eacc4df8127781ee4b0b1e5cad7dbaf12510f58c42cbcb9d1e2dba2a164/langgraph_sdk-0.1.74-py3-none-any.whl", hash = "sha256:3a265c3757fe0048adad4391d10486db63ef7aa5a2cbd22da22d4503554cb890", size = 50254, upload-time = "2025-07-21T16:36:49.134Z" }, ] [[package]] name = "langsmith" -version = "0.4.6" +version = "0.4.14" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, @@ -1620,147 +2146,469 @@ dependencies = [ { name = "requests-toolbelt" }, { name = "zstandard" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fc/9e/11536528c6e351820ad3fca0d2807f0e0f0619ff907529c78f68ba648497/langsmith-0.4.6.tar.gz", hash = "sha256:9189dbc9c60f2086ca3a1f0110cfe3aff6b0b7c2e0e3384f9572e70502e7933c", size = 352364, upload-time = "2025-07-15T19:43:18.541Z" } +sdist = { url = "https://files.pythonhosted.org/packages/85/b0/1def3c6d12eb5e412213e39f1ba4ac64a47ec3102cf42a3a1ff86af1402d/langsmith-0.4.14.tar.gz", hash = "sha256:4d29c7a9c85b20ba813ab9c855407bccdf5eb4f397f512ffa89959b2a2cb83ed", size = 921872, upload-time = "2025-08-12T20:39:43.704Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/9b/f2be47db823e89448ea41bfd8fc5ce6a995556bd25be4c23e5b3bb5b6c9b/langsmith-0.4.6-py3-none-any.whl", hash = "sha256:900e83fe59ee672bcf2f75c8bb47cd012bf8154d92a99c0355fc38b6485cbd3e", size = 367901, upload-time = "2025-07-15T19:43:16.508Z" }, + { url = "https://files.pythonhosted.org/packages/9e/08/3f0fb3e2f7cc6fd91c4d06d7abc6607425a66973bee79d04018bac41dd4f/langsmith-0.4.14-py3-none-any.whl", hash = "sha256:b6d070ac425196947d2a98126fb0e35f3b8c001a2e6e5b7049dd1c56f0767d0b", size = 373249, upload-time = "2025-08-12T20:39:41.992Z" }, ] [[package]] -name = "mako" -version = "1.3.10" +name = "language-data" +version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "markupsafe" }, + { name = "marisa-trie" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9e/38/bd5b78a920a64d708fe6bc8e0a2c075e1389d53bef8413725c63ba041535/mako-1.3.10.tar.gz", hash = "sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28", size = 392474, upload-time = "2025-04-10T12:44:31.16Z" } +sdist = { url = "https://files.pythonhosted.org/packages/dd/ce/3f144716a9f2cbf42aa86ebc8b085a184be25c80aa453eea17c294d239c1/language_data-1.3.0.tar.gz", hash = "sha256:7600ef8aa39555145d06c89f0c324bf7dab834ea0b0a439d8243762e3ebad7ec", size = 5129310, upload-time = "2024-11-19T10:21:37.912Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59", size = 78509, upload-time = "2025-04-10T12:50:53.297Z" }, + { url = "https://files.pythonhosted.org/packages/5d/e9/5a5ffd9b286db82be70d677d0a91e4d58f7912bb8dd026ddeeb4abe70679/language_data-1.3.0-py3-none-any.whl", hash = "sha256:e2ee943551b5ae5f89cd0e801d1fc3835bb0ef5b7e9c3a4e8e17b2b214548fbf", size = 5385760, upload-time = "2024-11-19T10:21:36.005Z" }, ] [[package]] -name = "markdown-it-py" -version = "3.0.0" +name = "litellm" +version = "1.75.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mdurl" }, + { name = "aiohttp" }, + { name = "click" }, + { name = "httpx" }, + { name = "importlib-metadata" }, + { name = "jinja2" }, + { name = "jsonschema" }, + { name = "openai" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "tiktoken" }, + { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb", size = 74596, upload-time = "2023-06-03T06:41:14.443Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8d/4e/48e3d6de19afe713223e3bc7009a2003501420de2a5d823c569cefbd9731/litellm-1.75.8.tar.gz", hash = "sha256:92061bd263ff8c33c8fff70ba92cd046adb7ea041a605826a915d108742fe59e", size = 10140384, upload-time = "2025-08-16T21:42:24.23Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528, upload-time = "2023-06-03T06:41:11.019Z" }, + { url = "https://files.pythonhosted.org/packages/5e/82/c4d00fbeafd93c00dab6ea03f33cadd6a97adeb720ba1d89fc319e5cb10b/litellm-1.75.8-py3-none-any.whl", hash = "sha256:0bf004488df8506381ec6e35e1486e2870e8d578a7c3f2427cd497558ce07a2e", size = 8916305, upload-time = "2025-08-16T21:42:21.387Z" }, ] [[package]] -name = "markupsafe" -version = "3.0.2" +name = "llama-cloud" +version = "0.1.35" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b2/97/5d42485e71dfc078108a86d6de8fa46db44a1a9295e89c5d6d4a06e23a62/markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", size = 20537, upload-time = "2024-10-18T15:21:54.129Z" } +dependencies = [ + { name = "certifi" }, + { name = "httpx" }, + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/72/816e6e900448e1b4a8137d90e65876b296c5264a23db6ae888bd3e6660ba/llama_cloud-0.1.35.tar.gz", hash = "sha256:200349d5d57424d7461f304cdb1355a58eea3e6ca1e6b0d75c66b2e937216983", size = 106403, upload-time = "2025-07-28T17:22:06.41Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/28/bbf83e3f76936960b850435576dd5e67034e200469571be53f69174a2dfd/MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d", size = 14353, upload-time = "2024-10-18T15:21:02.187Z" }, - { url = "https://files.pythonhosted.org/packages/6c/30/316d194b093cde57d448a4c3209f22e3046c5bb2fb0820b118292b334be7/MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93", size = 12392, upload-time = "2024-10-18T15:21:02.941Z" }, - { url = "https://files.pythonhosted.org/packages/f2/96/9cdafba8445d3a53cae530aaf83c38ec64c4d5427d975c974084af5bc5d2/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832", size = 23984, upload-time = "2024-10-18T15:21:03.953Z" }, - { url = "https://files.pythonhosted.org/packages/f1/a4/aefb044a2cd8d7334c8a47d3fb2c9f328ac48cb349468cc31c20b539305f/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84", size = 23120, upload-time = "2024-10-18T15:21:06.495Z" }, - { url = "https://files.pythonhosted.org/packages/8d/21/5e4851379f88f3fad1de30361db501300d4f07bcad047d3cb0449fc51f8c/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca", size = 23032, upload-time = "2024-10-18T15:21:07.295Z" }, - { url = "https://files.pythonhosted.org/packages/00/7b/e92c64e079b2d0d7ddf69899c98842f3f9a60a1ae72657c89ce2655c999d/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798", size = 24057, upload-time = "2024-10-18T15:21:08.073Z" }, - { url = "https://files.pythonhosted.org/packages/f9/ac/46f960ca323037caa0a10662ef97d0a4728e890334fc156b9f9e52bcc4ca/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e", size = 23359, upload-time = "2024-10-18T15:21:09.318Z" }, - { url = "https://files.pythonhosted.org/packages/69/84/83439e16197337b8b14b6a5b9c2105fff81d42c2a7c5b58ac7b62ee2c3b1/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4", size = 23306, upload-time = "2024-10-18T15:21:10.185Z" }, - { url = "https://files.pythonhosted.org/packages/9a/34/a15aa69f01e2181ed8d2b685c0d2f6655d5cca2c4db0ddea775e631918cd/MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d", size = 15094, upload-time = "2024-10-18T15:21:11.005Z" }, - { url = "https://files.pythonhosted.org/packages/da/b8/3a3bd761922d416f3dc5d00bfbed11f66b1ab89a0c2b6e887240a30b0f6b/MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b", size = 15521, upload-time = "2024-10-18T15:21:12.911Z" }, - { url = "https://files.pythonhosted.org/packages/22/09/d1f21434c97fc42f09d290cbb6350d44eb12f09cc62c9476effdb33a18aa/MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf", size = 14274, upload-time = "2024-10-18T15:21:13.777Z" }, - { url = "https://files.pythonhosted.org/packages/6b/b0/18f76bba336fa5aecf79d45dcd6c806c280ec44538b3c13671d49099fdd0/MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225", size = 12348, upload-time = "2024-10-18T15:21:14.822Z" }, - { url = "https://files.pythonhosted.org/packages/e0/25/dd5c0f6ac1311e9b40f4af06c78efde0f3b5cbf02502f8ef9501294c425b/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028", size = 24149, upload-time = "2024-10-18T15:21:15.642Z" }, - { url = "https://files.pythonhosted.org/packages/f3/f0/89e7aadfb3749d0f52234a0c8c7867877876e0a20b60e2188e9850794c17/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8", size = 23118, upload-time = "2024-10-18T15:21:17.133Z" }, - { url = "https://files.pythonhosted.org/packages/d5/da/f2eeb64c723f5e3777bc081da884b414671982008c47dcc1873d81f625b6/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c", size = 22993, upload-time = "2024-10-18T15:21:18.064Z" }, - { url = "https://files.pythonhosted.org/packages/da/0e/1f32af846df486dce7c227fe0f2398dc7e2e51d4a370508281f3c1c5cddc/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557", size = 24178, upload-time = "2024-10-18T15:21:18.859Z" }, - { url = "https://files.pythonhosted.org/packages/c4/f6/bb3ca0532de8086cbff5f06d137064c8410d10779c4c127e0e47d17c0b71/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22", size = 23319, upload-time = "2024-10-18T15:21:19.671Z" }, - { url = "https://files.pythonhosted.org/packages/a2/82/8be4c96ffee03c5b4a034e60a31294daf481e12c7c43ab8e34a1453ee48b/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48", size = 23352, upload-time = "2024-10-18T15:21:20.971Z" }, - { url = "https://files.pythonhosted.org/packages/51/ae/97827349d3fcffee7e184bdf7f41cd6b88d9919c80f0263ba7acd1bbcb18/MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30", size = 15097, upload-time = "2024-10-18T15:21:22.646Z" }, - { url = "https://files.pythonhosted.org/packages/c1/80/a61f99dc3a936413c3ee4e1eecac96c0da5ed07ad56fd975f1a9da5bc630/MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87", size = 15601, upload-time = "2024-10-18T15:21:23.499Z" }, + { url = "https://files.pythonhosted.org/packages/1d/d2/8d18a021ab757cea231428404f21fe3186bf1ebaac3f57a73c379483fd3f/llama_cloud-0.1.35-py3-none-any.whl", hash = "sha256:b7abab4423118e6f638d2f326749e7a07c6426543bea6da99b623c715b22af71", size = 303280, upload-time = "2025-07-28T17:22:04.946Z" }, ] [[package]] -name = "marshmallow" -version = "3.26.1" +name = "llama-cloud-services" +version = "0.6.15" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "packaging" }, + { name = "click" }, + { name = "llama-cloud" }, + { name = "llama-index-core" }, + { name = "platformdirs" }, + { name = "pydantic" }, + { name = "python-dotenv" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ab/5e/5e53d26b42ab75491cda89b871dab9e97c840bf12c63ec58a1919710cd06/marshmallow-3.26.1.tar.gz", hash = "sha256:e6d8affb6cb61d39d26402096dc0aee12d5a26d490a121f118d2e81dc0719dc6", size = 221825, upload-time = "2025-02-03T15:32:25.093Z" } +sdist = { url = "https://files.pythonhosted.org/packages/56/4e/da311d13340d22705d6ae48732c78a580039f132dfcaa68a7063b066c38c/llama_cloud_services-0.6.15.tar.gz", hash = "sha256:912799d9cdcf48074145c6781f40a6dd7dadb6344ecb30b715407db85a0e675e", size = 31701, upload-time = "2025-04-24T03:39:46.551Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/34/75/51952c7b2d3873b44a0028b1bd26a25078c18f92f256608e8d1dc61b39fd/marshmallow-3.26.1-py3-none-any.whl", hash = "sha256:3350409f20a70a7e4e11a27661187b77cdcaeb20abca41c1454fe33636bea09c", size = 50878, upload-time = "2025-02-03T15:32:22.295Z" }, + { url = "https://files.pythonhosted.org/packages/f5/0d/88805be6a13b368c9e7a2b2cede60fd0298e0e3abc9a6a6923d414c1ab14/llama_cloud_services-0.6.15-py3-none-any.whl", hash = "sha256:c4e24dd41f2cde17eeba7750d41cc70fe26e1179c03ae832122d762572e53de6", size = 36676, upload-time = "2025-04-24T03:39:45.217Z" }, ] [[package]] -name = "matplotlib" -version = "3.10.3" +name = "llama-index" +version = "0.12.42" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "contourpy" }, - { name = "cycler" }, - { name = "fonttools" }, - { name = "kiwisolver" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pillow" }, - { name = "pyparsing" }, - { name = "python-dateutil" }, + { name = "llama-index-agent-openai" }, + { name = "llama-index-cli" }, + { name = "llama-index-core" }, + { name = "llama-index-embeddings-openai" }, + { name = "llama-index-indices-managed-llama-cloud" }, + { name = "llama-index-llms-openai" }, + { name = "llama-index-multi-modal-llms-openai" }, + { name = "llama-index-program-openai" }, + { name = "llama-index-question-gen-openai" }, + { name = "llama-index-readers-file" }, + { name = "llama-index-readers-llama-parse" }, + { name = "nltk" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/26/91/d49359a21893183ed2a5b6c76bec40e0b1dcbf8ca148f864d134897cfc75/matplotlib-3.10.3.tar.gz", hash = "sha256:2f82d2c5bb7ae93aaaa4cd42aca65d76ce6376f83304fa3a630b569aca274df0", size = 34799811, upload-time = "2025-05-08T19:10:54.39Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/86/7e36d3f7b0606cba5e4862bb62f8e6bef91b7c45229f15262faf2c92f6e3/llama_index-0.12.42.tar.gz", hash = "sha256:d2bd2b9ea06bc42ebe74697aaa0b66c79bd3dee8ff5d06c53294d49ca44512e7", size = 8073, upload-time = "2025-06-12T04:59:38.313Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/bd/af9f655456f60fe1d575f54fb14704ee299b16e999704817a7645dfce6b0/matplotlib-3.10.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0ef061f74cd488586f552d0c336b2f078d43bc00dc473d2c3e7bfee2272f3fa8", size = 8178873, upload-time = "2025-05-08T19:09:53.857Z" }, - { url = "https://files.pythonhosted.org/packages/c2/86/e1c86690610661cd716eda5f9d0b35eaf606ae6c9b6736687cfc8f2d0cd8/matplotlib-3.10.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d96985d14dc5f4a736bbea4b9de9afaa735f8a0fc2ca75be2fa9e96b2097369d", size = 8052205, upload-time = "2025-05-08T19:09:55.684Z" }, - { url = "https://files.pythonhosted.org/packages/54/51/a9f8e49af3883dacddb2da1af5fca1f7468677f1188936452dd9aaaeb9ed/matplotlib-3.10.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c5f0283da91e9522bdba4d6583ed9d5521566f63729ffb68334f86d0bb98049", size = 8465823, upload-time = "2025-05-08T19:09:57.442Z" }, - { url = "https://files.pythonhosted.org/packages/e7/e3/c82963a3b86d6e6d5874cbeaa390166458a7f1961bab9feb14d3d1a10f02/matplotlib-3.10.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdfa07c0ec58035242bc8b2c8aae37037c9a886370eef6850703d7583e19964b", size = 8606464, upload-time = "2025-05-08T19:09:59.471Z" }, - { url = "https://files.pythonhosted.org/packages/0e/34/24da1027e7fcdd9e82da3194c470143c551852757a4b473a09a012f5b945/matplotlib-3.10.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c0b9849a17bce080a16ebcb80a7b714b5677d0ec32161a2cc0a8e5a6030ae220", size = 9413103, upload-time = "2025-05-08T19:10:03.208Z" }, - { url = "https://files.pythonhosted.org/packages/a6/da/948a017c3ea13fd4a97afad5fdebe2f5bbc4d28c0654510ce6fd6b06b7bd/matplotlib-3.10.3-cp311-cp311-win_amd64.whl", hash = "sha256:eef6ed6c03717083bc6d69c2d7ee8624205c29a8e6ea5a31cd3492ecdbaee1e1", size = 8065492, upload-time = "2025-05-08T19:10:05.271Z" }, - { url = "https://files.pythonhosted.org/packages/eb/43/6b80eb47d1071f234ef0c96ca370c2ca621f91c12045f1401b5c9b28a639/matplotlib-3.10.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0ab1affc11d1f495ab9e6362b8174a25afc19c081ba5b0775ef00533a4236eea", size = 8179689, upload-time = "2025-05-08T19:10:07.602Z" }, - { url = "https://files.pythonhosted.org/packages/0f/70/d61a591958325c357204870b5e7b164f93f2a8cca1dc6ce940f563909a13/matplotlib-3.10.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2a818d8bdcafa7ed2eed74487fdb071c09c1ae24152d403952adad11fa3c65b4", size = 8050466, upload-time = "2025-05-08T19:10:09.383Z" }, - { url = "https://files.pythonhosted.org/packages/e7/75/70c9d2306203148cc7902a961240c5927dd8728afedf35e6a77e105a2985/matplotlib-3.10.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:748ebc3470c253e770b17d8b0557f0aa85cf8c63fd52f1a61af5b27ec0b7ffee", size = 8456252, upload-time = "2025-05-08T19:10:11.958Z" }, - { url = "https://files.pythonhosted.org/packages/c4/91/ba0ae1ff4b3f30972ad01cd4a8029e70a0ec3b8ea5be04764b128b66f763/matplotlib-3.10.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed70453fd99733293ace1aec568255bc51c6361cb0da94fa5ebf0649fdb2150a", size = 8601321, upload-time = "2025-05-08T19:10:14.47Z" }, - { url = "https://files.pythonhosted.org/packages/d2/88/d636041eb54a84b889e11872d91f7cbf036b3b0e194a70fa064eb8b04f7a/matplotlib-3.10.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbed9917b44070e55640bd13419de83b4c918e52d97561544814ba463811cbc7", size = 9406972, upload-time = "2025-05-08T19:10:16.569Z" }, - { url = "https://files.pythonhosted.org/packages/b1/79/0d1c165eac44405a86478082e225fce87874f7198300bbebc55faaf6d28d/matplotlib-3.10.3-cp312-cp312-win_amd64.whl", hash = "sha256:cf37d8c6ef1a48829443e8ba5227b44236d7fcaf7647caa3178a4ff9f7a5be05", size = 8067954, upload-time = "2025-05-08T19:10:18.663Z" }, + { url = "https://files.pythonhosted.org/packages/64/9a/3044c50dd7cd7340763ed25e5ee7ee59917281735725eb73f62fe110333e/llama_index-0.12.42-py3-none-any.whl", hash = "sha256:9a304b2bd71d6772fc6c2e9995e119815c00716fc98813b15ea79caee09f1fe2", size = 7085, upload-time = "2025-06-12T04:59:37.071Z" }, ] [[package]] -name = "mccabe" -version = "0.7.0" +name = "llama-index-agent-openai" +version = "0.4.12" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658, upload-time = "2022-01-24T01:14:51.113Z" } +dependencies = [ + { name = "llama-index-core" }, + { name = "llama-index-llms-openai" }, + { name = "openai" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/94/69decc46d11e954c6a8c64999cc237af5932d116eeb7a06515856641a6d4/llama_index_agent_openai-0.4.12.tar.gz", hash = "sha256:d2fe53feb69cfe45752edb7328bf0d25f6a9071b3c056787e661b93e5b748a28", size = 12443, upload-time = "2025-06-29T00:52:03.606Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, + { url = "https://files.pythonhosted.org/packages/89/f5/857ea1c136f422234e298e868af74094a71bf98687be40a365ad6551a660/llama_index_agent_openai-0.4.12-py3-none-any.whl", hash = "sha256:6dbb6276b2e5330032a726b28d5eef5140825f36d72d472b231f08ad3af99665", size = 14704, upload-time = "2025-06-29T00:52:02.528Z" }, ] [[package]] -name = "mcp" -version = "1.11.0" +name = "llama-index-cli" +version = "0.4.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, - { name = "httpx" }, - { name = "httpx-sse" }, - { name = "jsonschema" }, - { name = "pydantic" }, - { name = "pydantic-settings" }, - { name = "python-multipart" }, - { name = "pywin32", marker = "sys_platform == 'win32'" }, - { name = "sse-starlette" }, - { name = "starlette" }, - { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, + { name = "llama-index-core" }, + { name = "llama-index-embeddings-openai" }, + { name = "llama-index-llms-openai" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3a/f5/9506eb5578d5bbe9819ee8ba3198d0ad0e2fbe3bab8b257e4131ceb7dfb6/mcp-1.11.0.tar.gz", hash = "sha256:49a213df56bb9472ff83b3132a4825f5c8f5b120a90246f08b0dac6bedac44c8", size = 406907, upload-time = "2025-07-10T16:41:09.388Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/44/6acba0b8425d15682def89a4dbba68c782fd74ce6e74a4fa48beb08632f6/llama_index_cli-0.4.4.tar.gz", hash = "sha256:c3af0cf1e2a7e5ef44d0bae5aa8e8872b54c5dd6b731afbae9f13ffeb4997be0", size = 25308, upload-time = "2025-07-07T05:17:40.556Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/92/9c/c9ca79f9c512e4113a5d07043013110bb3369fc7770040c61378c7fbcf70/mcp-1.11.0-py3-none-any.whl", hash = "sha256:58deac37f7483e4b338524b98bc949b7c2b7c33d978f5fafab5bde041c5e2595", size = 155880, upload-time = "2025-07-10T16:41:07.935Z" }, + { url = "https://files.pythonhosted.org/packages/cc/21/89989b7fa8ce4b9bc6f0f7326ae7f959a887c1281f007a718eafd6ef614f/llama_index_cli-0.4.4-py3-none-any.whl", hash = "sha256:1070593cf79407054735ab7a23c5a65a26fc18d264661e42ef38fc549b4b7658", size = 28598, upload-time = "2025-07-07T05:17:39.522Z" }, ] [[package]] -name = "mdurl" -version = "0.1.2" +name = "llama-index-core" +version = "0.12.42" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, -] - -[[package]] +dependencies = [ + { name = "aiohttp" }, + { name = "aiosqlite" }, + { name = "banks" }, + { name = "dataclasses-json" }, + { name = "deprecated" }, + { name = "dirtyjson" }, + { name = "filetype" }, + { name = "fsspec" }, + { name = "httpx" }, + { name = "nest-asyncio" }, + { name = "networkx" }, + { name = "nltk" }, + { name = "numpy" }, + { name = "pillow" }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "sqlalchemy", extra = ["asyncio"] }, + { name = "tenacity" }, + { name = "tiktoken" }, + { name = "tqdm" }, + { name = "typing-extensions" }, + { name = "typing-inspect" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d8/67/ef1fc8c3b0676b524fd6b67a0e3e082996d768d227ed2b4b39f78119253b/llama_index_core-0.12.42.tar.gz", hash = "sha256:cff21fe15610826997c876a6b1d28d52727932c5f9c2af04b23e041a10f40a24", size = 7292833, upload-time = "2025-06-12T03:07:06.791Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/92/dc47b702da141d0f617ee86d26b5bdb62ed9a52518d1a5f271e91337e4a7/llama_index_core-0.12.42-py3-none-any.whl", hash = "sha256:0534cd9a4f6113175aa406a47ae9a683b5a43fd55532e9dbbffa96838ff18e07", size = 7665930, upload-time = "2025-06-12T03:06:58.387Z" }, +] + +[[package]] +name = "llama-index-embeddings-openai" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llama-index-core" }, + { name = "openai" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/02/a2604ef3a167131fdd701888f45f16c8efa6d523d02efe8c4e640238f4ea/llama_index_embeddings_openai-0.3.1.tar.gz", hash = "sha256:1368aad3ce24cbaed23d5ad251343cef1eb7b4a06d6563d6606d59cb347fef20", size = 5492, upload-time = "2024-11-27T16:04:17.017Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/45/ca55b91c4ac1b6251d4099fa44121a6c012129822906cadcc27b8cfb33a4/llama_index_embeddings_openai-0.3.1-py3-none-any.whl", hash = "sha256:f15a3d13da9b6b21b8bd51d337197879a453d1605e625a1c6d45e741756c0290", size = 6177, upload-time = "2024-11-27T16:04:15.981Z" }, +] + +[[package]] +name = "llama-index-indices-managed-llama-cloud" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llama-cloud" }, + { name = "llama-index-core" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f1/2a/0b42b533e6891150eb9d1189492a08be155e9656889178009bbc4cdf44b9/llama_index_indices_managed_llama_cloud-0.8.0.tar.gz", hash = "sha256:762de10d3949e04997766f6a665ed4503394d82ea4b3339139a365edde6e753e", size = 14722, upload-time = "2025-07-28T19:53:08.979Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/50/c259cc8b8497ab8f3e245c9bc828e8a269951b222760b5cac072acba3811/llama_index_indices_managed_llama_cloud-0.8.0-py3-none-any.whl", hash = "sha256:817d6bd4715d45522e7165d29208e093d06179cc1bc5f9590382245f73dfd7aa", size = 16451, upload-time = "2025-07-28T19:53:07.826Z" }, +] + +[[package]] +name = "llama-index-llms-openai" +version = "0.4.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llama-index-core" }, + { name = "openai" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d9/39/a7ce514fb500951e9edb713ed918a9ffe49f1a76fccfc531a4ec5c7fe15a/llama_index_llms_openai-0.4.7.tar.gz", hash = "sha256:564af8ab39fb3f3adfeae73a59c0dca46c099ab844a28e725eee0c551d4869f8", size = 24251, upload-time = "2025-06-16T03:38:47.175Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/e9/391926dad180ced6bb37a62edddb8483fbecde411239bd5e726841bb77b4/llama_index_llms_openai-0.4.7-py3-none-any.whl", hash = "sha256:3b8d9d3c1bcadc2cff09724de70f074f43eafd5b7048a91247c9a41b7cd6216d", size = 25365, upload-time = "2025-06-16T03:38:45.72Z" }, +] + +[[package]] +name = "llama-index-multi-modal-llms-openai" +version = "0.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llama-index-core" }, + { name = "llama-index-llms-openai" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7b/77/0ece3c7887e9c7242fc7cc2a184374bbeb08a35783d71a8466e1dd18ccfc/llama_index_multi_modal_llms_openai-0.5.1.tar.gz", hash = "sha256:df3aff00c36023c5f8c49f972a325f71823ed0f4dd9cd479955d76afc146575f", size = 3708, upload-time = "2025-05-30T23:08:09.915Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/a5/6181a385af1caef88e1d0d3919c399ba7e031190fc0be4fd5242875c5b0e/llama_index_multi_modal_llms_openai-0.5.1-py3-none-any.whl", hash = "sha256:69bb9c310c323ce51038f0d40719f546d0d4e0853835a4f5cfa21f07dbb0b91e", size = 3362, upload-time = "2025-05-30T23:08:08.977Z" }, +] + +[[package]] +name = "llama-index-program-openai" +version = "0.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llama-index-agent-openai" }, + { name = "llama-index-core" }, + { name = "llama-index-llms-openai" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/81/9caa34e80adce1adb715ae083a54ad45c8fc0d9aef0f2d80d61c1b805ab6/llama_index_program_openai-0.3.2.tar.gz", hash = "sha256:04c959a2e616489894bd2eeebb99500d6f1c17d588c3da0ddc75ebd3eb7451ee", size = 6301, upload-time = "2025-05-30T23:00:27.872Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/80/d6ac8afafdd38115d61214891c36876e64f429809abff873660fe30862fe/llama_index_program_openai-0.3.2-py3-none-any.whl", hash = "sha256:451829ae53e074e7b47dcc60a9dd155fcf9d1dcbc1754074bdadd6aab4ceb9aa", size = 6129, upload-time = "2025-05-30T23:00:26.64Z" }, +] + +[[package]] +name = "llama-index-question-gen-openai" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llama-index-core" }, + { name = "llama-index-llms-openai" }, + { name = "llama-index-program-openai" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/6e/19c5051c81ef5fca597d13c6d41b863535521565b1414ab5ab0e5e8c1297/llama_index_question_gen_openai-0.3.1.tar.gz", hash = "sha256:5e9311b433cc2581ff8a531fa19fb3aa21815baff75aaacdef11760ac9522aa9", size = 4107, upload-time = "2025-05-30T23:00:31.016Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/2a/652593d0bd24f901776db0d1778a42363ea2656530da18215f413ce4f981/llama_index_question_gen_openai-0.3.1-py3-none-any.whl", hash = "sha256:1ce266f6c8373fc8d884ff83a44dfbacecde2301785db7144872db51b8b99429", size = 3733, upload-time = "2025-05-30T23:00:29.965Z" }, +] + +[[package]] +name = "llama-index-readers-file" +version = "0.4.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "llama-index-core" }, + { name = "pandas" }, + { name = "pypdf" }, + { name = "striprtf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/55/de3392c59f8f44d20e48b72faa91393061425d68cf7035f167db7732f025/llama_index_readers_file-0.4.8.tar.gz", hash = "sha256:d2821b164c11453b7993456355a6c544631a0ae8656a646df9fc009f16cfb99f", size = 22668, upload-time = "2025-05-27T19:53:18.479Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/24/62226cbf55e91d97d7b7d659d3dc05a42f15c2413589056ace8a4813fb76/llama_index_readers_file-0.4.8-py3-none-any.whl", hash = "sha256:6cc64c70c3a9f5b146200a68c15b742e6f0d13b308a1c0dc6bb919e3b5fd4275", size = 40972, upload-time = "2025-05-27T19:53:17.047Z" }, +] + +[[package]] +name = "llama-index-readers-llama-parse" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llama-index-core" }, + { name = "llama-parse" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/35/30/4611821286f82ba7b5842295607baa876262db86f88b87d83595eed172bf/llama_index_readers_llama_parse-0.4.0.tar.gz", hash = "sha256:e99ec56f4f8546d7fda1a7c1ae26162fb9acb7ebcac343b5abdb4234b4644e0f", size = 2472, upload-time = "2024-11-18T00:00:08.893Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/4f/e30d4257fe9e4224f5612b77fe99aaceddae411b2e74ca30534491de3e6f/llama_index_readers_llama_parse-0.4.0-py3-none-any.whl", hash = "sha256:574e48386f28d2c86c3f961ca4a4906910312f3400dd0c53014465bfbc6b32bf", size = 2472, upload-time = "2024-11-18T00:00:07.293Z" }, +] + +[[package]] +name = "llama-parse" +version = "0.6.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llama-cloud-services" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/27/8014c38cab1e9664153157d3c8693af726c0f7ae0c93adaebace5da688d7/llama_parse-0.6.12.tar.gz", hash = "sha256:c99593fb955c338a69e64a2ec449e09753afe6dcff239ab050989fda74839867", size = 3673, upload-time = "2025-04-11T17:27:49.525Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/ca/71c9367d3e89d61da2462f535dea1a3a09d4a4085b96f2c9ef5c38864820/llama_parse-0.6.12-py3-none-any.whl", hash = "sha256:2dd1c74b0cba1a2bc300286f6b91a650f6ddc396acfce3497ba3d72d43c53fac", size = 4853, upload-time = "2025-04-11T17:27:48.223Z" }, +] + +[[package]] +name = "mako" +version = "1.3.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/38/bd5b78a920a64d708fe6bc8e0a2c075e1389d53bef8413725c63ba041535/mako-1.3.10.tar.gz", hash = "sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28", size = 392474, upload-time = "2025-04-10T12:44:31.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59", size = 78509, upload-time = "2025-04-10T12:50:53.297Z" }, +] + +[[package]] +name = "marisa-trie" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/df/504ca06cfcc6d67ec034f35b863d6719c26970407c94dc638c1994d78684/marisa_trie-1.3.0.tar.gz", hash = "sha256:39af3060b4ab41a3cce18b1808338db8bf50b6ec4b81be3cc452558aaad95581", size = 212383, upload-time = "2025-08-16T10:05:20.9Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/f2/c98483b63c0cf9f54e283e1689980baca66d348ef214f643688b30a4b28c/marisa_trie-1.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5d72ffde56fb1515bcb03539803d42d0a119f6782c5812bf2b7313eddc691735", size = 176084, upload-time = "2025-08-16T10:04:04.575Z" }, + { url = "https://files.pythonhosted.org/packages/d5/29/51ff243488d753f62e7fc5111af20cc3437bf85d44bb901a9bf4c89a0e6d/marisa_trie-1.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6a1f0781bccd854184a9c59b095ed09adf16627460eb8df4a91dc3f87e882352", size = 158864, upload-time = "2025-08-16T10:04:06.016Z" }, + { url = "https://files.pythonhosted.org/packages/69/bd/a21c11c15c0e11071e9b4fc983ecc84c9575ae63abc2dd57586aa7bddebf/marisa_trie-1.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:608d965d47f40b8cd402215b95d85db899268d277ae5b8ebe87b7acdd3e2a0bb", size = 1257508, upload-time = "2025-08-16T10:04:07.683Z" }, + { url = "https://files.pythonhosted.org/packages/7c/4b/02e542a8e22779b35bca354cc66a736fe8ca2f20aa93203c93c16cfc4470/marisa_trie-1.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8b39a7314f6ad141c9c24acff0a71f4fdae1eab5ea827468c40afafc0662cab3", size = 1275539, upload-time = "2025-08-16T10:04:09.397Z" }, + { url = "https://files.pythonhosted.org/packages/87/a2/308ab72fb6e9a7e40a95f56478ab0dc5bfeb17195ed97e11ed46a7c5790d/marisa_trie-1.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6e8e2f1394eecfb780a25950849d64a799b79f538d17945e42b1652da4e0cae4", size = 2199801, upload-time = "2025-08-16T10:04:10.638Z" }, + { url = "https://files.pythonhosted.org/packages/cb/0a/b8e6627f6fb60d37dabd731d0639816c8c2e2f87ea9ecb34fd2d67863d7e/marisa_trie-1.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1a56cc700b1405cc75fde9197f9d2fed66ecbbaee7bdf1f28728494f119dc7f3", size = 2283801, upload-time = "2025-08-16T10:04:11.988Z" }, + { url = "https://files.pythonhosted.org/packages/af/32/fba57bcb72ea8d4331e248fa50ebeb3e9c1889eb7f2b8025974a760c624c/marisa_trie-1.3.0-cp311-cp311-win32.whl", hash = "sha256:58f1b70501c2462583bce5639a65af5516e9785ae6b3158533ddeecde70f0675", size = 117295, upload-time = "2025-08-16T10:04:13.465Z" }, + { url = "https://files.pythonhosted.org/packages/fc/28/f8099b17fc1f0301f02556ef7c40253571b0bda2a405d3416fdf0cba2383/marisa_trie-1.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:938f618d2cece8358899c688591d94db6652d9e1076c15a7efdfcfdc64a96cdb", size = 143966, upload-time = "2025-08-16T10:04:14.873Z" }, + { url = "https://files.pythonhosted.org/packages/2d/ac/db61a1c950f23b876380928098e69c9a24f2810ef1a68eb7d5bf6732fa47/marisa_trie-1.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:28bfd6fada6c87cb31d300bbed5de1bfd338f8c98d1b834cf810a06ce019a020", size = 174778, upload-time = "2025-08-16T10:04:16.312Z" }, + { url = "https://files.pythonhosted.org/packages/e2/de/0b80b66f8fe09b36150e7a1a1e1c9761d136ccb0887186404c3d2c447b25/marisa_trie-1.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:034e483bd35ab6d136d8a91f43088dc78549394cf3787fdeebca144e2e4c82df", size = 155737, upload-time = "2025-08-16T10:04:17.371Z" }, + { url = "https://files.pythonhosted.org/packages/5e/58/5da536b940fa6145744b06714fc25ca0b0b0360d0499e88678d560a0dd68/marisa_trie-1.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b71462677dc6c119589755394086cffbcf4d4d42f906fefb325c982c679406d6", size = 1244807, upload-time = "2025-08-16T10:04:18.58Z" }, + { url = "https://files.pythonhosted.org/packages/45/cd/05bed6d02213da7f2fda63e689300b186a7f16f6b982ea19bb7284ecb1ee/marisa_trie-1.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:31c891ebce899f35936d4ab9f332b69ab762513d5944b0f43f61427e53671d42", size = 1265122, upload-time = "2025-08-16T10:04:19.901Z" }, + { url = "https://files.pythonhosted.org/packages/70/ff/06f718d62f78bfa6d7082533cd187b6425755d77a6ea901228b4085f61b3/marisa_trie-1.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4570850d9b6e6a099797f731652dbe764dfd6dd7eff2934318a7018ba1a82cf1", size = 2172695, upload-time = "2025-08-16T10:04:21.258Z" }, + { url = "https://files.pythonhosted.org/packages/3c/47/a3a50e293f87f3a11082fbb80fdd504e2d8d1a92372476f37b08a0b765dd/marisa_trie-1.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d85a0484f8ecd3a6c843c1b10b42953f14278b35ce30d94bc7cb6305604a6109", size = 2256076, upload-time = "2025-08-16T10:04:23.043Z" }, + { url = "https://files.pythonhosted.org/packages/9a/32/b58f7b0c378c05eb3df632ffc2dadb93611266b0f42997e72eb9597dc25c/marisa_trie-1.3.0-cp312-cp312-win32.whl", hash = "sha256:714dabb0ddd4be72841c962d0559d5a80613964dc2a5db72651ae3b2ae3408fc", size = 115589, upload-time = "2025-08-16T10:04:24.243Z" }, + { url = "https://files.pythonhosted.org/packages/1e/90/57830e0403ef2ae9067ec07ccb2fee8297a5c42f518528c8c2e7401cd4b5/marisa_trie-1.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:bd53e6b99008ff3dab6455791800af405351d98fbf01c4f474642afb1499236d", size = 138547, upload-time = "2025-08-16T10:04:25.637Z" }, +] + +[[package]] +name = "markdown" +version = "3.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/c2/4ab49206c17f75cb08d6311171f2d65798988db4360c4d1485bd0eedd67c/markdown-3.8.2.tar.gz", hash = "sha256:247b9a70dd12e27f67431ce62523e675b866d254f900c4fe75ce3dda62237c45", size = 362071, upload-time = "2025-06-19T17:12:44.483Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/2b/34cc11786bc00d0f04d0f5fdc3a2b1ae0b6239eef72d3d345805f9ad92a1/markdown-3.8.2-py3-none-any.whl", hash = "sha256:5c83764dbd4e00bdd94d85a19b8d55ccca20fe35b2e678a1422b380324dd5f24", size = 106827, upload-time = "2025-06-19T17:12:42.994Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/97/5d42485e71dfc078108a86d6de8fa46db44a1a9295e89c5d6d4a06e23a62/markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", size = 20537, upload-time = "2024-10-18T15:21:54.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/28/bbf83e3f76936960b850435576dd5e67034e200469571be53f69174a2dfd/MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d", size = 14353, upload-time = "2024-10-18T15:21:02.187Z" }, + { url = "https://files.pythonhosted.org/packages/6c/30/316d194b093cde57d448a4c3209f22e3046c5bb2fb0820b118292b334be7/MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93", size = 12392, upload-time = "2024-10-18T15:21:02.941Z" }, + { url = "https://files.pythonhosted.org/packages/f2/96/9cdafba8445d3a53cae530aaf83c38ec64c4d5427d975c974084af5bc5d2/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832", size = 23984, upload-time = "2024-10-18T15:21:03.953Z" }, + { url = "https://files.pythonhosted.org/packages/f1/a4/aefb044a2cd8d7334c8a47d3fb2c9f328ac48cb349468cc31c20b539305f/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84", size = 23120, upload-time = "2024-10-18T15:21:06.495Z" }, + { url = "https://files.pythonhosted.org/packages/8d/21/5e4851379f88f3fad1de30361db501300d4f07bcad047d3cb0449fc51f8c/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca", size = 23032, upload-time = "2024-10-18T15:21:07.295Z" }, + { url = "https://files.pythonhosted.org/packages/00/7b/e92c64e079b2d0d7ddf69899c98842f3f9a60a1ae72657c89ce2655c999d/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798", size = 24057, upload-time = "2024-10-18T15:21:08.073Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ac/46f960ca323037caa0a10662ef97d0a4728e890334fc156b9f9e52bcc4ca/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e", size = 23359, upload-time = "2024-10-18T15:21:09.318Z" }, + { url = "https://files.pythonhosted.org/packages/69/84/83439e16197337b8b14b6a5b9c2105fff81d42c2a7c5b58ac7b62ee2c3b1/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4", size = 23306, upload-time = "2024-10-18T15:21:10.185Z" }, + { url = "https://files.pythonhosted.org/packages/9a/34/a15aa69f01e2181ed8d2b685c0d2f6655d5cca2c4db0ddea775e631918cd/MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d", size = 15094, upload-time = "2024-10-18T15:21:11.005Z" }, + { url = "https://files.pythonhosted.org/packages/da/b8/3a3bd761922d416f3dc5d00bfbed11f66b1ab89a0c2b6e887240a30b0f6b/MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b", size = 15521, upload-time = "2024-10-18T15:21:12.911Z" }, + { url = "https://files.pythonhosted.org/packages/22/09/d1f21434c97fc42f09d290cbb6350d44eb12f09cc62c9476effdb33a18aa/MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf", size = 14274, upload-time = "2024-10-18T15:21:13.777Z" }, + { url = "https://files.pythonhosted.org/packages/6b/b0/18f76bba336fa5aecf79d45dcd6c806c280ec44538b3c13671d49099fdd0/MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225", size = 12348, upload-time = "2024-10-18T15:21:14.822Z" }, + { url = "https://files.pythonhosted.org/packages/e0/25/dd5c0f6ac1311e9b40f4af06c78efde0f3b5cbf02502f8ef9501294c425b/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028", size = 24149, upload-time = "2024-10-18T15:21:15.642Z" }, + { url = "https://files.pythonhosted.org/packages/f3/f0/89e7aadfb3749d0f52234a0c8c7867877876e0a20b60e2188e9850794c17/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8", size = 23118, upload-time = "2024-10-18T15:21:17.133Z" }, + { url = "https://files.pythonhosted.org/packages/d5/da/f2eeb64c723f5e3777bc081da884b414671982008c47dcc1873d81f625b6/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c", size = 22993, upload-time = "2024-10-18T15:21:18.064Z" }, + { url = "https://files.pythonhosted.org/packages/da/0e/1f32af846df486dce7c227fe0f2398dc7e2e51d4a370508281f3c1c5cddc/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557", size = 24178, upload-time = "2024-10-18T15:21:18.859Z" }, + { url = "https://files.pythonhosted.org/packages/c4/f6/bb3ca0532de8086cbff5f06d137064c8410d10779c4c127e0e47d17c0b71/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22", size = 23319, upload-time = "2024-10-18T15:21:19.671Z" }, + { url = "https://files.pythonhosted.org/packages/a2/82/8be4c96ffee03c5b4a034e60a31294daf481e12c7c43ab8e34a1453ee48b/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48", size = 23352, upload-time = "2024-10-18T15:21:20.971Z" }, + { url = "https://files.pythonhosted.org/packages/51/ae/97827349d3fcffee7e184bdf7f41cd6b88d9919c80f0263ba7acd1bbcb18/MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30", size = 15097, upload-time = "2024-10-18T15:21:22.646Z" }, + { url = "https://files.pythonhosted.org/packages/c1/80/a61f99dc3a936413c3ee4e1eecac96c0da5ed07ad56fd975f1a9da5bc630/MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87", size = 15601, upload-time = "2024-10-18T15:21:23.499Z" }, +] + +[[package]] +name = "marshmallow" +version = "3.26.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ab/5e/5e53d26b42ab75491cda89b871dab9e97c840bf12c63ec58a1919710cd06/marshmallow-3.26.1.tar.gz", hash = "sha256:e6d8affb6cb61d39d26402096dc0aee12d5a26d490a121f118d2e81dc0719dc6", size = 221825, upload-time = "2025-02-03T15:32:25.093Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/75/51952c7b2d3873b44a0028b1bd26a25078c18f92f256608e8d1dc61b39fd/marshmallow-3.26.1-py3-none-any.whl", hash = "sha256:3350409f20a70a7e4e11a27661187b77cdcaeb20abca41c1454fe33636bea09c", size = 50878, upload-time = "2025-02-03T15:32:22.295Z" }, +] + +[[package]] +name = "matplotlib" +version = "3.10.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "contourpy" }, + { name = "cycler" }, + { name = "fonttools" }, + { name = "kiwisolver" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "pyparsing" }, + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/91/f2939bb60b7ebf12478b030e0d7f340247390f402b3b189616aad790c366/matplotlib-3.10.5.tar.gz", hash = "sha256:352ed6ccfb7998a00881692f38b4ca083c691d3e275b4145423704c34c909076", size = 34804044, upload-time = "2025-07-31T18:09:33.805Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/c7/1f2db90a1d43710478bb1e9b57b162852f79234d28e4f48a28cc415aa583/matplotlib-3.10.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:dcfc39c452c6a9f9028d3e44d2d721484f665304857188124b505b2c95e1eecf", size = 8239216, upload-time = "2025-07-31T18:07:51.947Z" }, + { url = "https://files.pythonhosted.org/packages/82/6d/ca6844c77a4f89b1c9e4d481c412e1d1dbabf2aae2cbc5aa2da4a1d6683e/matplotlib-3.10.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:903352681b59f3efbf4546985142a9686ea1d616bb054b09a537a06e4b892ccf", size = 8102130, upload-time = "2025-07-31T18:07:53.65Z" }, + { url = "https://files.pythonhosted.org/packages/1d/1e/5e187a30cc673a3e384f3723e5f3c416033c1d8d5da414f82e4e731128ea/matplotlib-3.10.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:080c3676a56b8ee1c762bcf8fca3fe709daa1ee23e6ef06ad9f3fc17332f2d2a", size = 8666471, upload-time = "2025-07-31T18:07:55.304Z" }, + { url = "https://files.pythonhosted.org/packages/03/c0/95540d584d7d645324db99a845ac194e915ef75011a0d5e19e1b5cee7e69/matplotlib-3.10.5-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b4984d5064a35b6f66d2c11d668565f4389b1119cc64db7a4c1725bc11adffc", size = 9500518, upload-time = "2025-07-31T18:07:57.199Z" }, + { url = "https://files.pythonhosted.org/packages/ba/2e/e019352099ea58b4169adb9c6e1a2ad0c568c6377c2b677ee1f06de2adc7/matplotlib-3.10.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3967424121d3a46705c9fa9bdb0931de3228f13f73d7bb03c999c88343a89d89", size = 9552372, upload-time = "2025-07-31T18:07:59.41Z" }, + { url = "https://files.pythonhosted.org/packages/b7/81/3200b792a5e8b354f31f4101ad7834743ad07b6d620259f2059317b25e4d/matplotlib-3.10.5-cp311-cp311-win_amd64.whl", hash = "sha256:33775bbeb75528555a15ac29396940128ef5613cf9a2d31fb1bfd18b3c0c0903", size = 8100634, upload-time = "2025-07-31T18:08:01.801Z" }, + { url = "https://files.pythonhosted.org/packages/52/46/a944f6f0c1f5476a0adfa501969d229ce5ae60cf9a663be0e70361381f89/matplotlib-3.10.5-cp311-cp311-win_arm64.whl", hash = "sha256:c61333a8e5e6240e73769d5826b9a31d8b22df76c0778f8480baf1b4b01c9420", size = 7978880, upload-time = "2025-07-31T18:08:03.407Z" }, + { url = "https://files.pythonhosted.org/packages/66/1e/c6f6bcd882d589410b475ca1fc22e34e34c82adff519caf18f3e6dd9d682/matplotlib-3.10.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:00b6feadc28a08bd3c65b2894f56cf3c94fc8f7adcbc6ab4516ae1e8ed8f62e2", size = 8253056, upload-time = "2025-07-31T18:08:05.385Z" }, + { url = "https://files.pythonhosted.org/packages/53/e6/d6f7d1b59413f233793dda14419776f5f443bcccb2dfc84b09f09fe05dbe/matplotlib-3.10.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ee98a5c5344dc7f48dc261b6ba5d9900c008fc12beb3fa6ebda81273602cc389", size = 8110131, upload-time = "2025-07-31T18:08:07.293Z" }, + { url = "https://files.pythonhosted.org/packages/66/2b/bed8a45e74957549197a2ac2e1259671cd80b55ed9e1fe2b5c94d88a9202/matplotlib-3.10.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a17e57e33de901d221a07af32c08870ed4528db0b6059dce7d7e65c1122d4bea", size = 8669603, upload-time = "2025-07-31T18:08:09.064Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a7/315e9435b10d057f5e52dfc603cd353167ae28bb1a4e033d41540c0067a4/matplotlib-3.10.5-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97b9d6443419085950ee4a5b1ee08c363e5c43d7176e55513479e53669e88468", size = 9508127, upload-time = "2025-07-31T18:08:10.845Z" }, + { url = "https://files.pythonhosted.org/packages/7f/d9/edcbb1f02ca99165365d2768d517898c22c6040187e2ae2ce7294437c413/matplotlib-3.10.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ceefe5d40807d29a66ae916c6a3915d60ef9f028ce1927b84e727be91d884369", size = 9566926, upload-time = "2025-07-31T18:08:13.186Z" }, + { url = "https://files.pythonhosted.org/packages/3b/d9/6dd924ad5616c97b7308e6320cf392c466237a82a2040381163b7500510a/matplotlib-3.10.5-cp312-cp312-win_amd64.whl", hash = "sha256:c04cba0f93d40e45b3c187c6c52c17f24535b27d545f757a2fffebc06c12b98b", size = 8107599, upload-time = "2025-07-31T18:08:15.116Z" }, + { url = "https://files.pythonhosted.org/packages/0e/f3/522dc319a50f7b0279fbe74f86f7a3506ce414bc23172098e8d2bdf21894/matplotlib-3.10.5-cp312-cp312-win_arm64.whl", hash = "sha256:a41bcb6e2c8e79dc99c5511ae6f7787d2fb52efd3d805fff06d5d4f667db16b2", size = 7978173, upload-time = "2025-07-31T18:08:21.518Z" }, + { url = "https://files.pythonhosted.org/packages/dc/d6/e921be4e1a5f7aca5194e1f016cb67ec294548e530013251f630713e456d/matplotlib-3.10.5-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:160e125da27a749481eaddc0627962990f6029811dbeae23881833a011a0907f", size = 8233224, upload-time = "2025-07-31T18:09:27.512Z" }, + { url = "https://files.pythonhosted.org/packages/ec/74/a2b9b04824b9c349c8f1b2d21d5af43fa7010039427f2b133a034cb09e59/matplotlib-3.10.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ac3d50760394d78a3c9be6b28318fe22b494c4fcf6407e8fd4794b538251899b", size = 8098539, upload-time = "2025-07-31T18:09:29.629Z" }, + { url = "https://files.pythonhosted.org/packages/fc/66/cd29ebc7f6c0d2a15d216fb572573e8fc38bd5d6dec3bd9d7d904c0949f7/matplotlib-3.10.5-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c49465bf689c4d59d174d0c7795fb42a21d4244d11d70e52b8011987367ac61", size = 8672192, upload-time = "2025-07-31T18:09:31.407Z" }, +] + +[[package]] +name = "matplotlib-inline" +version = "0.1.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/99/5b/a36a337438a14116b16480db471ad061c36c3694df7c2084a0da7ba538b7/matplotlib_inline-0.1.7.tar.gz", hash = "sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90", size = 8159, upload-time = "2024-04-15T13:44:44.803Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/8e/9ad090d3553c280a8060fbf6e24dc1c0c29704ee7d1c372f0c174aa59285/matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca", size = 9899, upload-time = "2024-04-15T13:44:43.265Z" }, +] + +[[package]] +name = "mccabe" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658, upload-time = "2022-01-24T01:14:51.113Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, +] + +[[package]] +name = "mcp" +version = "1.12.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpx" }, + { name = "httpx-sse" }, + { name = "jsonschema" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "python-multipart" }, + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "sse-starlette" }, + { name = "starlette" }, + { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/31/88/f6cb7e7c260cd4b4ce375f2b1614b33ce401f63af0f49f7141a2e9bf0a45/mcp-1.12.4.tar.gz", hash = "sha256:0765585e9a3a5916a3c3ab8659330e493adc7bd8b2ca6120c2d7a0c43e034ca5", size = 431148, upload-time = "2025-08-07T20:31:18.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/68/316cbc54b7163fa22571dcf42c9cc46562aae0a021b974e0a8141e897200/mcp-1.12.4-py3-none-any.whl", hash = "sha256:7aa884648969fab8e78b89399d59a683202972e12e6bc9a1c88ce7eda7743789", size = 160145, upload-time = "2025-08-07T20:31:15.69Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] name = "milvus-lite" version = "2.5.1" source = { registry = "https://pypi.org/simple" } @@ -1776,186 +2624,577 @@ wheels = [ [[package]] name = "multidict" -version = "6.6.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3d/2c/5dad12e82fbdf7470f29bff2171484bf07cb3b16ada60a6589af8f376440/multidict-6.6.3.tar.gz", hash = "sha256:798a9eb12dab0a6c2e29c1de6f3468af5cb2da6053a20dfa3344907eed0937cc", size = 101006, upload-time = "2025-06-30T15:53:46.929Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/08/f0/1a39863ced51f639c81a5463fbfa9eb4df59c20d1a8769ab9ef4ca57ae04/multidict-6.6.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:18f4eba0cbac3546b8ae31e0bbc55b02c801ae3cbaf80c247fcdd89b456ff58c", size = 76445, upload-time = "2025-06-30T15:51:24.01Z" }, - { url = "https://files.pythonhosted.org/packages/c9/0e/a7cfa451c7b0365cd844e90b41e21fab32edaa1e42fc0c9f68461ce44ed7/multidict-6.6.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ef43b5dd842382329e4797c46f10748d8c2b6e0614f46b4afe4aee9ac33159df", size = 44610, upload-time = "2025-06-30T15:51:25.158Z" }, - { url = "https://files.pythonhosted.org/packages/c6/bb/a14a4efc5ee748cc1904b0748be278c31b9295ce5f4d2ef66526f410b94d/multidict-6.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bf9bd1fd5eec01494e0f2e8e446a74a85d5e49afb63d75a9934e4a5423dba21d", size = 44267, upload-time = "2025-06-30T15:51:26.326Z" }, - { url = "https://files.pythonhosted.org/packages/c2/f8/410677d563c2d55e063ef74fe578f9d53fe6b0a51649597a5861f83ffa15/multidict-6.6.3-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:5bd8d6f793a787153956cd35e24f60485bf0651c238e207b9a54f7458b16d539", size = 230004, upload-time = "2025-06-30T15:51:27.491Z" }, - { url = "https://files.pythonhosted.org/packages/fd/df/2b787f80059314a98e1ec6a4cc7576244986df3e56b3c755e6fc7c99e038/multidict-6.6.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1bf99b4daf908c73856bd87ee0a2499c3c9a3d19bb04b9c6025e66af3fd07462", size = 247196, upload-time = "2025-06-30T15:51:28.762Z" }, - { url = "https://files.pythonhosted.org/packages/05/f2/f9117089151b9a8ab39f9019620d10d9718eec2ac89e7ca9d30f3ec78e96/multidict-6.6.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b9e59946b49dafaf990fd9c17ceafa62976e8471a14952163d10a7a630413a9", size = 225337, upload-time = "2025-06-30T15:51:30.025Z" }, - { url = "https://files.pythonhosted.org/packages/93/2d/7115300ec5b699faa152c56799b089a53ed69e399c3c2d528251f0aeda1a/multidict-6.6.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e2db616467070d0533832d204c54eea6836a5e628f2cb1e6dfd8cd6ba7277cb7", size = 257079, upload-time = "2025-06-30T15:51:31.716Z" }, - { url = "https://files.pythonhosted.org/packages/15/ea/ff4bab367623e39c20d3b07637225c7688d79e4f3cc1f3b9f89867677f9a/multidict-6.6.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7394888236621f61dcdd25189b2768ae5cc280f041029a5bcf1122ac63df79f9", size = 255461, upload-time = "2025-06-30T15:51:33.029Z" }, - { url = "https://files.pythonhosted.org/packages/74/07/2c9246cda322dfe08be85f1b8739646f2c4c5113a1422d7a407763422ec4/multidict-6.6.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f114d8478733ca7388e7c7e0ab34b72547476b97009d643644ac33d4d3fe1821", size = 246611, upload-time = "2025-06-30T15:51:34.47Z" }, - { url = "https://files.pythonhosted.org/packages/a8/62/279c13d584207d5697a752a66ffc9bb19355a95f7659140cb1b3cf82180e/multidict-6.6.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cdf22e4db76d323bcdc733514bf732e9fb349707c98d341d40ebcc6e9318ef3d", size = 243102, upload-time = "2025-06-30T15:51:36.525Z" }, - { url = "https://files.pythonhosted.org/packages/69/cc/e06636f48c6d51e724a8bc8d9e1db5f136fe1df066d7cafe37ef4000f86a/multidict-6.6.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:e995a34c3d44ab511bfc11aa26869b9d66c2d8c799fa0e74b28a473a692532d6", size = 238693, upload-time = "2025-06-30T15:51:38.278Z" }, - { url = "https://files.pythonhosted.org/packages/89/a4/66c9d8fb9acf3b226cdd468ed009537ac65b520aebdc1703dd6908b19d33/multidict-6.6.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:766a4a5996f54361d8d5a9050140aa5362fe48ce51c755a50c0bc3706460c430", size = 246582, upload-time = "2025-06-30T15:51:39.709Z" }, - { url = "https://files.pythonhosted.org/packages/cf/01/c69e0317be556e46257826d5449feb4e6aa0d18573e567a48a2c14156f1f/multidict-6.6.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:3893a0d7d28a7fe6ca7a1f760593bc13038d1d35daf52199d431b61d2660602b", size = 253355, upload-time = "2025-06-30T15:51:41.013Z" }, - { url = "https://files.pythonhosted.org/packages/c0/da/9cc1da0299762d20e626fe0042e71b5694f9f72d7d3f9678397cbaa71b2b/multidict-6.6.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:934796c81ea996e61914ba58064920d6cad5d99140ac3167901eb932150e2e56", size = 247774, upload-time = "2025-06-30T15:51:42.291Z" }, - { url = "https://files.pythonhosted.org/packages/e6/91/b22756afec99cc31105ddd4a52f95ab32b1a4a58f4d417979c570c4a922e/multidict-6.6.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9ed948328aec2072bc00f05d961ceadfd3e9bfc2966c1319aeaf7b7c21219183", size = 242275, upload-time = "2025-06-30T15:51:43.642Z" }, - { url = "https://files.pythonhosted.org/packages/be/f1/adcc185b878036a20399d5be5228f3cbe7f823d78985d101d425af35c800/multidict-6.6.3-cp311-cp311-win32.whl", hash = "sha256:9f5b28c074c76afc3e4c610c488e3493976fe0e596dd3db6c8ddfbb0134dcac5", size = 41290, upload-time = "2025-06-30T15:51:45.264Z" }, - { url = "https://files.pythonhosted.org/packages/e0/d4/27652c1c6526ea6b4f5ddd397e93f4232ff5de42bea71d339bc6a6cc497f/multidict-6.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:bc7f6fbc61b1c16050a389c630da0b32fc6d4a3d191394ab78972bf5edc568c2", size = 45942, upload-time = "2025-06-30T15:51:46.377Z" }, - { url = "https://files.pythonhosted.org/packages/16/18/23f4932019804e56d3c2413e237f866444b774b0263bcb81df2fdecaf593/multidict-6.6.3-cp311-cp311-win_arm64.whl", hash = "sha256:d4e47d8faffaae822fb5cba20937c048d4f734f43572e7079298a6c39fb172cb", size = 42880, upload-time = "2025-06-30T15:51:47.561Z" }, - { url = "https://files.pythonhosted.org/packages/0e/a0/6b57988ea102da0623ea814160ed78d45a2645e4bbb499c2896d12833a70/multidict-6.6.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:056bebbeda16b2e38642d75e9e5310c484b7c24e3841dc0fb943206a72ec89d6", size = 76514, upload-time = "2025-06-30T15:51:48.728Z" }, - { url = "https://files.pythonhosted.org/packages/07/7a/d1e92665b0850c6c0508f101f9cf0410c1afa24973e1115fe9c6a185ebf7/multidict-6.6.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e5f481cccb3c5c5e5de5d00b5141dc589c1047e60d07e85bbd7dea3d4580d63f", size = 45394, upload-time = "2025-06-30T15:51:49.986Z" }, - { url = "https://files.pythonhosted.org/packages/52/6f/dd104490e01be6ef8bf9573705d8572f8c2d2c561f06e3826b081d9e6591/multidict-6.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:10bea2ee839a759ee368b5a6e47787f399b41e70cf0c20d90dfaf4158dfb4e55", size = 43590, upload-time = "2025-06-30T15:51:51.331Z" }, - { url = "https://files.pythonhosted.org/packages/44/fe/06e0e01b1b0611e6581b7fd5a85b43dacc08b6cea3034f902f383b0873e5/multidict-6.6.3-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:2334cfb0fa9549d6ce2c21af2bfbcd3ac4ec3646b1b1581c88e3e2b1779ec92b", size = 237292, upload-time = "2025-06-30T15:51:52.584Z" }, - { url = "https://files.pythonhosted.org/packages/ce/71/4f0e558fb77696b89c233c1ee2d92f3e1d5459070a0e89153c9e9e804186/multidict-6.6.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b8fee016722550a2276ca2cb5bb624480e0ed2bd49125b2b73b7010b9090e888", size = 258385, upload-time = "2025-06-30T15:51:53.913Z" }, - { url = "https://files.pythonhosted.org/packages/e3/25/cca0e68228addad24903801ed1ab42e21307a1b4b6dd2cf63da5d3ae082a/multidict-6.6.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5511cb35f5c50a2db21047c875eb42f308c5583edf96bd8ebf7d770a9d68f6d", size = 242328, upload-time = "2025-06-30T15:51:55.672Z" }, - { url = "https://files.pythonhosted.org/packages/6e/a3/46f2d420d86bbcb8fe660b26a10a219871a0fbf4d43cb846a4031533f3e0/multidict-6.6.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:712b348f7f449948e0a6c4564a21c7db965af900973a67db432d724619b3c680", size = 268057, upload-time = "2025-06-30T15:51:57.037Z" }, - { url = "https://files.pythonhosted.org/packages/9e/73/1c743542fe00794a2ec7466abd3f312ccb8fad8dff9f36d42e18fb1ec33e/multidict-6.6.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e4e15d2138ee2694e038e33b7c3da70e6b0ad8868b9f8094a72e1414aeda9c1a", size = 269341, upload-time = "2025-06-30T15:51:59.111Z" }, - { url = "https://files.pythonhosted.org/packages/a4/11/6ec9dcbe2264b92778eeb85407d1df18812248bf3506a5a1754bc035db0c/multidict-6.6.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8df25594989aebff8a130f7899fa03cbfcc5d2b5f4a461cf2518236fe6f15961", size = 256081, upload-time = "2025-06-30T15:52:00.533Z" }, - { url = "https://files.pythonhosted.org/packages/9b/2b/631b1e2afeb5f1696846d747d36cda075bfdc0bc7245d6ba5c319278d6c4/multidict-6.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:159ca68bfd284a8860f8d8112cf0521113bffd9c17568579e4d13d1f1dc76b65", size = 253581, upload-time = "2025-06-30T15:52:02.43Z" }, - { url = "https://files.pythonhosted.org/packages/bf/0e/7e3b93f79efeb6111d3bf9a1a69e555ba1d07ad1c11bceb56b7310d0d7ee/multidict-6.6.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:e098c17856a8c9ade81b4810888c5ad1914099657226283cab3062c0540b0643", size = 250750, upload-time = "2025-06-30T15:52:04.26Z" }, - { url = "https://files.pythonhosted.org/packages/ad/9e/086846c1d6601948e7de556ee464a2d4c85e33883e749f46b9547d7b0704/multidict-6.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:67c92ed673049dec52d7ed39f8cf9ebbadf5032c774058b4406d18c8f8fe7063", size = 251548, upload-time = "2025-06-30T15:52:06.002Z" }, - { url = "https://files.pythonhosted.org/packages/8c/7b/86ec260118e522f1a31550e87b23542294880c97cfbf6fb18cc67b044c66/multidict-6.6.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:bd0578596e3a835ef451784053cfd327d607fc39ea1a14812139339a18a0dbc3", size = 262718, upload-time = "2025-06-30T15:52:07.707Z" }, - { url = "https://files.pythonhosted.org/packages/8c/bd/22ce8f47abb0be04692c9fc4638508b8340987b18691aa7775d927b73f72/multidict-6.6.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:346055630a2df2115cd23ae271910b4cae40f4e336773550dca4889b12916e75", size = 259603, upload-time = "2025-06-30T15:52:09.58Z" }, - { url = "https://files.pythonhosted.org/packages/07/9c/91b7ac1691be95cd1f4a26e36a74b97cda6aa9820632d31aab4410f46ebd/multidict-6.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:555ff55a359302b79de97e0468e9ee80637b0de1fce77721639f7cd9440b3a10", size = 251351, upload-time = "2025-06-30T15:52:10.947Z" }, - { url = "https://files.pythonhosted.org/packages/6f/5c/4d7adc739884f7a9fbe00d1eac8c034023ef8bad71f2ebe12823ca2e3649/multidict-6.6.3-cp312-cp312-win32.whl", hash = "sha256:73ab034fb8d58ff85c2bcbadc470efc3fafeea8affcf8722855fb94557f14cc5", size = 41860, upload-time = "2025-06-30T15:52:12.334Z" }, - { url = "https://files.pythonhosted.org/packages/6a/a3/0fbc7afdf7cb1aa12a086b02959307848eb6bcc8f66fcb66c0cb57e2a2c1/multidict-6.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:04cbcce84f63b9af41bad04a54d4cc4e60e90c35b9e6ccb130be2d75b71f8c17", size = 45982, upload-time = "2025-06-30T15:52:13.6Z" }, - { url = "https://files.pythonhosted.org/packages/b8/95/8c825bd70ff9b02462dc18d1295dd08d3e9e4eb66856d292ffa62cfe1920/multidict-6.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:0f1130b896ecb52d2a1e615260f3ea2af55fa7dc3d7c3003ba0c3121a759b18b", size = 43210, upload-time = "2025-06-30T15:52:14.893Z" }, - { url = "https://files.pythonhosted.org/packages/d8/30/9aec301e9772b098c1f5c0ca0279237c9766d94b97802e9888010c64b0ed/multidict-6.6.3-py3-none-any.whl", hash = "sha256:8db10f29c7541fc5da4defd8cd697e1ca429db743fa716325f236079b96f775a", size = 12313, upload-time = "2025-06-30T15:53:45.437Z" }, +version = "6.6.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/7f/0652e6ed47ab288e3756ea9c0df8b14950781184d4bd7883f4d87dd41245/multidict-6.6.4.tar.gz", hash = "sha256:d2d4e4787672911b48350df02ed3fa3fffdc2f2e8ca06dd6afdf34189b76a9dd", size = 101843, upload-time = "2025-08-11T12:08:48.217Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/7f/90a7f01e2d005d6653c689039977f6856718c75c5579445effb7e60923d1/multidict-6.6.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c7a0e9b561e6460484318a7612e725df1145d46b0ef57c6b9866441bf6e27e0c", size = 76472, upload-time = "2025-08-11T12:06:29.006Z" }, + { url = "https://files.pythonhosted.org/packages/54/a3/bed07bc9e2bb302ce752f1dabc69e884cd6a676da44fb0e501b246031fdd/multidict-6.6.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6bf2f10f70acc7a2446965ffbc726e5fc0b272c97a90b485857e5c70022213eb", size = 44634, upload-time = "2025-08-11T12:06:30.374Z" }, + { url = "https://files.pythonhosted.org/packages/a7/4b/ceeb4f8f33cf81277da464307afeaf164fb0297947642585884f5cad4f28/multidict-6.6.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:66247d72ed62d5dd29752ffc1d3b88f135c6a8de8b5f63b7c14e973ef5bda19e", size = 44282, upload-time = "2025-08-11T12:06:31.958Z" }, + { url = "https://files.pythonhosted.org/packages/03/35/436a5da8702b06866189b69f655ffdb8f70796252a8772a77815f1812679/multidict-6.6.4-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:105245cc6b76f51e408451a844a54e6823bbd5a490ebfe5bdfc79798511ceded", size = 229696, upload-time = "2025-08-11T12:06:33.087Z" }, + { url = "https://files.pythonhosted.org/packages/b6/0e/915160be8fecf1fca35f790c08fb74ca684d752fcba62c11daaf3d92c216/multidict-6.6.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbbc54e58b34c3bae389ef00046be0961f30fef7cb0dd9c7756aee376a4f7683", size = 246665, upload-time = "2025-08-11T12:06:34.448Z" }, + { url = "https://files.pythonhosted.org/packages/08/ee/2f464330acd83f77dcc346f0b1a0eaae10230291450887f96b204b8ac4d3/multidict-6.6.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:56c6b3652f945c9bc3ac6c8178cd93132b8d82dd581fcbc3a00676c51302bc1a", size = 225485, upload-time = "2025-08-11T12:06:35.672Z" }, + { url = "https://files.pythonhosted.org/packages/71/cc/9a117f828b4d7fbaec6adeed2204f211e9caf0a012692a1ee32169f846ae/multidict-6.6.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b95494daf857602eccf4c18ca33337dd2be705bccdb6dddbfc9d513e6addb9d9", size = 257318, upload-time = "2025-08-11T12:06:36.98Z" }, + { url = "https://files.pythonhosted.org/packages/25/77/62752d3dbd70e27fdd68e86626c1ae6bccfebe2bb1f84ae226363e112f5a/multidict-6.6.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e5b1413361cef15340ab9dc61523e653d25723e82d488ef7d60a12878227ed50", size = 254689, upload-time = "2025-08-11T12:06:38.233Z" }, + { url = "https://files.pythonhosted.org/packages/00/6e/fac58b1072a6fc59af5e7acb245e8754d3e1f97f4f808a6559951f72a0d4/multidict-6.6.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e167bf899c3d724f9662ef00b4f7fef87a19c22b2fead198a6f68b263618df52", size = 246709, upload-time = "2025-08-11T12:06:39.517Z" }, + { url = "https://files.pythonhosted.org/packages/01/ef/4698d6842ef5e797c6db7744b0081e36fb5de3d00002cc4c58071097fac3/multidict-6.6.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:aaea28ba20a9026dfa77f4b80369e51cb767c61e33a2d4043399c67bd95fb7c6", size = 243185, upload-time = "2025-08-11T12:06:40.796Z" }, + { url = "https://files.pythonhosted.org/packages/aa/c9/d82e95ae1d6e4ef396934e9b0e942dfc428775f9554acf04393cce66b157/multidict-6.6.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:8c91cdb30809a96d9ecf442ec9bc45e8cfaa0f7f8bdf534e082c2443a196727e", size = 237838, upload-time = "2025-08-11T12:06:42.595Z" }, + { url = "https://files.pythonhosted.org/packages/57/cf/f94af5c36baaa75d44fab9f02e2a6bcfa0cd90acb44d4976a80960759dbc/multidict-6.6.4-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1a0ccbfe93ca114c5d65a2471d52d8829e56d467c97b0e341cf5ee45410033b3", size = 246368, upload-time = "2025-08-11T12:06:44.304Z" }, + { url = "https://files.pythonhosted.org/packages/4a/fe/29f23460c3d995f6a4b678cb2e9730e7277231b981f0b234702f0177818a/multidict-6.6.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:55624b3f321d84c403cb7d8e6e982f41ae233d85f85db54ba6286f7295dc8a9c", size = 253339, upload-time = "2025-08-11T12:06:45.597Z" }, + { url = "https://files.pythonhosted.org/packages/29/b6/fd59449204426187b82bf8a75f629310f68c6adc9559dc922d5abe34797b/multidict-6.6.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:4a1fb393a2c9d202cb766c76208bd7945bc194eba8ac920ce98c6e458f0b524b", size = 246933, upload-time = "2025-08-11T12:06:46.841Z" }, + { url = "https://files.pythonhosted.org/packages/19/52/d5d6b344f176a5ac3606f7a61fb44dc746e04550e1a13834dff722b8d7d6/multidict-6.6.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:43868297a5759a845fa3a483fb4392973a95fb1de891605a3728130c52b8f40f", size = 242225, upload-time = "2025-08-11T12:06:48.588Z" }, + { url = "https://files.pythonhosted.org/packages/ec/d3/5b2281ed89ff4d5318d82478a2a2450fcdfc3300da48ff15c1778280ad26/multidict-6.6.4-cp311-cp311-win32.whl", hash = "sha256:ed3b94c5e362a8a84d69642dbeac615452e8af9b8eb825b7bc9f31a53a1051e2", size = 41306, upload-time = "2025-08-11T12:06:49.95Z" }, + { url = "https://files.pythonhosted.org/packages/74/7d/36b045c23a1ab98507aefd44fd8b264ee1dd5e5010543c6fccf82141ccef/multidict-6.6.4-cp311-cp311-win_amd64.whl", hash = "sha256:d8c112f7a90d8ca5d20213aa41eac690bb50a76da153e3afb3886418e61cb22e", size = 46029, upload-time = "2025-08-11T12:06:51.082Z" }, + { url = "https://files.pythonhosted.org/packages/0f/5e/553d67d24432c5cd52b49047f2d248821843743ee6d29a704594f656d182/multidict-6.6.4-cp311-cp311-win_arm64.whl", hash = "sha256:3bb0eae408fa1996d87247ca0d6a57b7fc1dcf83e8a5c47ab82c558c250d4adf", size = 43017, upload-time = "2025-08-11T12:06:52.243Z" }, + { url = "https://files.pythonhosted.org/packages/05/f6/512ffd8fd8b37fb2680e5ac35d788f1d71bbaf37789d21a820bdc441e565/multidict-6.6.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0ffb87be160942d56d7b87b0fdf098e81ed565add09eaa1294268c7f3caac4c8", size = 76516, upload-time = "2025-08-11T12:06:53.393Z" }, + { url = "https://files.pythonhosted.org/packages/99/58/45c3e75deb8855c36bd66cc1658007589662ba584dbf423d01df478dd1c5/multidict-6.6.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d191de6cbab2aff5de6c5723101705fd044b3e4c7cfd587a1929b5028b9714b3", size = 45394, upload-time = "2025-08-11T12:06:54.555Z" }, + { url = "https://files.pythonhosted.org/packages/fd/ca/e8c4472a93a26e4507c0b8e1f0762c0d8a32de1328ef72fd704ef9cc5447/multidict-6.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38a0956dd92d918ad5feff3db8fcb4a5eb7dba114da917e1a88475619781b57b", size = 43591, upload-time = "2025-08-11T12:06:55.672Z" }, + { url = "https://files.pythonhosted.org/packages/05/51/edf414f4df058574a7265034d04c935aa84a89e79ce90fcf4df211f47b16/multidict-6.6.4-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6865f6d3b7900ae020b495d599fcf3765653bc927951c1abb959017f81ae8287", size = 237215, upload-time = "2025-08-11T12:06:57.213Z" }, + { url = "https://files.pythonhosted.org/packages/c8/45/8b3d6dbad8cf3252553cc41abea09ad527b33ce47a5e199072620b296902/multidict-6.6.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a2088c126b6f72db6c9212ad827d0ba088c01d951cee25e758c450da732c138", size = 258299, upload-time = "2025-08-11T12:06:58.946Z" }, + { url = "https://files.pythonhosted.org/packages/3c/e8/8ca2e9a9f5a435fc6db40438a55730a4bf4956b554e487fa1b9ae920f825/multidict-6.6.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0f37bed7319b848097085d7d48116f545985db988e2256b2e6f00563a3416ee6", size = 242357, upload-time = "2025-08-11T12:07:00.301Z" }, + { url = "https://files.pythonhosted.org/packages/0f/84/80c77c99df05a75c28490b2af8f7cba2a12621186e0a8b0865d8e745c104/multidict-6.6.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:01368e3c94032ba6ca0b78e7ccb099643466cf24f8dc8eefcfdc0571d56e58f9", size = 268369, upload-time = "2025-08-11T12:07:01.638Z" }, + { url = "https://files.pythonhosted.org/packages/0d/e9/920bfa46c27b05fb3e1ad85121fd49f441492dca2449c5bcfe42e4565d8a/multidict-6.6.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fe323540c255db0bffee79ad7f048c909f2ab0edb87a597e1c17da6a54e493c", size = 269341, upload-time = "2025-08-11T12:07:02.943Z" }, + { url = "https://files.pythonhosted.org/packages/af/65/753a2d8b05daf496f4a9c367fe844e90a1b2cac78e2be2c844200d10cc4c/multidict-6.6.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8eb3025f17b0a4c3cd08cda49acf312a19ad6e8a4edd9dbd591e6506d999402", size = 256100, upload-time = "2025-08-11T12:07:04.564Z" }, + { url = "https://files.pythonhosted.org/packages/09/54/655be13ae324212bf0bc15d665a4e34844f34c206f78801be42f7a0a8aaa/multidict-6.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bbc14f0365534d35a06970d6a83478b249752e922d662dc24d489af1aa0d1be7", size = 253584, upload-time = "2025-08-11T12:07:05.914Z" }, + { url = "https://files.pythonhosted.org/packages/5c/74/ab2039ecc05264b5cec73eb018ce417af3ebb384ae9c0e9ed42cb33f8151/multidict-6.6.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:75aa52fba2d96bf972e85451b99d8e19cc37ce26fd016f6d4aa60da9ab2b005f", size = 251018, upload-time = "2025-08-11T12:07:08.301Z" }, + { url = "https://files.pythonhosted.org/packages/af/0a/ccbb244ac848e56c6427f2392741c06302bbfba49c0042f1eb3c5b606497/multidict-6.6.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4fefd4a815e362d4f011919d97d7b4a1e566f1dde83dc4ad8cfb5b41de1df68d", size = 251477, upload-time = "2025-08-11T12:07:10.248Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b0/0ed49bba775b135937f52fe13922bc64a7eaf0a3ead84a36e8e4e446e096/multidict-6.6.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:db9801fe021f59a5b375ab778973127ca0ac52429a26e2fd86aa9508f4d26eb7", size = 263575, upload-time = "2025-08-11T12:07:11.928Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d9/7fb85a85e14de2e44dfb6a24f03c41e2af8697a6df83daddb0e9b7569f73/multidict-6.6.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a650629970fa21ac1fb06ba25dabfc5b8a2054fcbf6ae97c758aa956b8dba802", size = 259649, upload-time = "2025-08-11T12:07:13.244Z" }, + { url = "https://files.pythonhosted.org/packages/03/9e/b3a459bcf9b6e74fa461a5222a10ff9b544cb1cd52fd482fb1b75ecda2a2/multidict-6.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:452ff5da78d4720d7516a3a2abd804957532dd69296cb77319c193e3ffb87e24", size = 251505, upload-time = "2025-08-11T12:07:14.57Z" }, + { url = "https://files.pythonhosted.org/packages/86/a2/8022f78f041dfe6d71e364001a5cf987c30edfc83c8a5fb7a3f0974cff39/multidict-6.6.4-cp312-cp312-win32.whl", hash = "sha256:8c2fcb12136530ed19572bbba61b407f655e3953ba669b96a35036a11a485793", size = 41888, upload-time = "2025-08-11T12:07:15.904Z" }, + { url = "https://files.pythonhosted.org/packages/c7/eb/d88b1780d43a56db2cba24289fa744a9d216c1a8546a0dc3956563fd53ea/multidict-6.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:047d9425860a8c9544fed1b9584f0c8bcd31bcde9568b047c5e567a1025ecd6e", size = 46072, upload-time = "2025-08-11T12:07:17.045Z" }, + { url = "https://files.pythonhosted.org/packages/9f/16/b929320bf5750e2d9d4931835a4c638a19d2494a5b519caaaa7492ebe105/multidict-6.6.4-cp312-cp312-win_arm64.whl", hash = "sha256:14754eb72feaa1e8ae528468f24250dd997b8e2188c3d2f593f9eba259e4b364", size = 43222, upload-time = "2025-08-11T12:07:18.328Z" }, + { url = "https://files.pythonhosted.org/packages/fd/69/b547032297c7e63ba2af494edba695d781af8a0c6e89e4d06cf848b21d80/multidict-6.6.4-py3-none-any.whl", hash = "sha256:27d8f8e125c07cb954e54d75d04905a9bba8a439c1d84aca94949d4d03d8601c", size = 12313, upload-time = "2025-08-11T12:08:46.891Z" }, ] [[package]] name = "multiprocess" -version = "0.70.18" +version = "0.70.16" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dill" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/72/fd/2ae3826f5be24c6ed87266bc4e59c46ea5b059a103f3d7e7eb76a52aeecb/multiprocess-0.70.18.tar.gz", hash = "sha256:f9597128e6b3e67b23956da07cf3d2e5cba79e2f4e0fba8d7903636663ec6d0d", size = 1798503, upload-time = "2025-04-17T03:11:27.742Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/ae/04f39c5d0d0def03247c2893d6f2b83c136bf3320a2154d7b8858f2ba72d/multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1", size = 1772603, upload-time = "2024-01-28T18:52:34.85Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/f7/7ec7fddc92e50714ea3745631f79bd9c96424cb2702632521028e57d3a36/multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02", size = 134824, upload-time = "2024-01-28T18:52:26.062Z" }, + { url = "https://files.pythonhosted.org/packages/50/15/b56e50e8debaf439f44befec5b2af11db85f6e0f344c3113ae0be0593a91/multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a", size = 143519, upload-time = "2024-01-28T18:52:28.115Z" }, + { url = "https://files.pythonhosted.org/packages/0a/7d/a988f258104dcd2ccf1ed40fdc97e26c4ac351eeaf81d76e266c52d84e2f/multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e", size = 146741, upload-time = "2024-01-28T18:52:29.395Z" }, + { url = "https://files.pythonhosted.org/packages/ea/89/38df130f2c799090c978b366cfdf5b96d08de5b29a4a293df7f7429fa50b/multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435", size = 132628, upload-time = "2024-01-28T18:52:30.853Z" }, + { url = "https://files.pythonhosted.org/packages/da/d9/f7f9379981e39b8c2511c9e0326d212accacb82f12fbfdc1aa2ce2a7b2b6/multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3", size = 133351, upload-time = "2024-01-28T18:52:31.981Z" }, +] + +[[package]] +name = "murmurhash" +version = "1.0.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/e9/02efbc6dfc2dd2085da3daacf9a8c17e8356019eceaedbfa21555e32d2af/murmurhash-1.0.13.tar.gz", hash = "sha256:737246d41ee00ff74b07b0bd1f0888be304d203ce668e642c86aa64ede30f8b7", size = 13258, upload-time = "2025-05-22T12:35:57.019Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/d1/9d13a02d9c8bfff10b1f68d19df206eaf2a8011defeccf7eb05ea0b8c54e/murmurhash-1.0.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b20d168370bc3ce82920121b78ab35ae244070a9b18798f4a2e8678fa03bd7e0", size = 26410, upload-time = "2025-05-22T12:35:20.786Z" }, + { url = "https://files.pythonhosted.org/packages/14/b0/3ee762e98cf9a8c2df9c8b377c326f3dd4495066d4eace9066fca46eba7a/murmurhash-1.0.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cef667d2e83bdceea3bc20c586c491fa442662ace1aea66ff5e3a18bb38268d8", size = 26679, upload-time = "2025-05-22T12:35:21.808Z" }, + { url = "https://files.pythonhosted.org/packages/39/06/24618f79cd5aac48490932e50263bddfd1ea90f7123d49bfe806a5982675/murmurhash-1.0.13-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:507148e50929ba1fce36898808573b9f81c763d5676f3fc6e4e832ff56b66992", size = 125970, upload-time = "2025-05-22T12:35:23.222Z" }, + { url = "https://files.pythonhosted.org/packages/e8/09/0e7afce0a422692506c85474a26fb3a03c1971b2b5f7e7745276c4b3de7f/murmurhash-1.0.13-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64d50f6173d266ad165beb8bca6101d824217fc9279f9e9981f4c0245c1e7ee6", size = 123390, upload-time = "2025-05-22T12:35:24.303Z" }, + { url = "https://files.pythonhosted.org/packages/22/4c/c98f579b1a951b2bcc722a35270a2eec105c1e21585c9b314a02079e3c4d/murmurhash-1.0.13-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0f272e15a84a8ae5f8b4bc0a68f9f47be38518ddffc72405791178058e9d019a", size = 124007, upload-time = "2025-05-22T12:35:25.446Z" }, + { url = "https://files.pythonhosted.org/packages/df/f8/1b0dcebc8df8e091341617102b5b3b97deb6435f345b84f75382c290ec2c/murmurhash-1.0.13-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9423e0b0964ed1013a06c970199538c7ef9ca28c0be54798c0f1473a6591761", size = 123705, upload-time = "2025-05-22T12:35:26.709Z" }, + { url = "https://files.pythonhosted.org/packages/79/17/f2a38558e150a0669d843f75e128afb83c1a67af41885ea2acb940e18e2a/murmurhash-1.0.13-cp311-cp311-win_amd64.whl", hash = "sha256:83b81e7084b696df3d853f2c78e0c9bda6b285d643f923f1a6fa9ab145d705c5", size = 24572, upload-time = "2025-05-22T12:35:30.38Z" }, + { url = "https://files.pythonhosted.org/packages/e1/53/56ce2d8d4b9ab89557cb1d00ffce346b80a2eb2d8c7944015e5c83eacdec/murmurhash-1.0.13-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bbe882e46cb3f86e092d8a1dd7a5a1c992da1ae3b39f7dd4507b6ce33dae7f92", size = 26859, upload-time = "2025-05-22T12:35:31.815Z" }, + { url = "https://files.pythonhosted.org/packages/f8/85/3a0ad54a61257c31496545ae6861515d640316f93681d1dd917e7be06634/murmurhash-1.0.13-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:52a33a12ecedc432493692c207c784b06b6427ffaa897fc90b7a76e65846478d", size = 26900, upload-time = "2025-05-22T12:35:34.267Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cd/6651de26744b50ff11c79f0c0d41244db039625de53c0467a7a52876b2d8/murmurhash-1.0.13-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:950403a7f0dc2d9c8d0710f07c296f2daab66299d9677d6c65d6b6fa2cb30aaa", size = 131367, upload-time = "2025-05-22T12:35:35.258Z" }, + { url = "https://files.pythonhosted.org/packages/50/6c/01ded95ddce33811c9766cae4ce32e0a54288da1d909ee2bcaa6ed13b9f1/murmurhash-1.0.13-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fde9fb5d2c106d86ff3ef2e4a9a69c2a8d23ba46e28c6b30034dc58421bc107b", size = 128943, upload-time = "2025-05-22T12:35:36.358Z" }, + { url = "https://files.pythonhosted.org/packages/ab/27/e539a9622d7bea3ae22706c1eb80d4af80f9dddd93b54d151955c2ae4011/murmurhash-1.0.13-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3aa55d62773745616e1ab19345dece122f6e6d09224f7be939cc5b4c513c8473", size = 129108, upload-time = "2025-05-22T12:35:37.864Z" }, + { url = "https://files.pythonhosted.org/packages/7a/84/18af5662e07d06839ad4db18ce026e6f8ef850d7b0ba92817b28dad28ba6/murmurhash-1.0.13-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:060dfef1b405cf02c450f182fb629f76ebe7f79657cced2db5054bc29b34938b", size = 129175, upload-time = "2025-05-22T12:35:38.928Z" }, + { url = "https://files.pythonhosted.org/packages/fe/8d/b01d3ee1f1cf3957250223b7c6ce35454f38fbf4abe236bf04a3f769341d/murmurhash-1.0.13-cp312-cp312-win_amd64.whl", hash = "sha256:a8e79627d44a6e20a6487effc30bfe1c74754c13d179106e68cc6d07941b022c", size = 24869, upload-time = "2025-05-22T12:35:40.035Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "nbformat" +version = "5.10.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fastjsonschema" }, + { name = "jsonschema" }, + { name = "jupyter-core" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/fd/91545e604bc3dad7dca9ed03284086039b294c6b3d75c0d2fa45f9e9caf3/nbformat-5.10.4.tar.gz", hash = "sha256:322168b14f937a5d11362988ecac2a4952d3d8e3a2cbeb2319584631226d5b3a", size = 142749, upload-time = "2024-04-04T11:20:37.371Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/55/4d/9af0d1279c84618bcd35bf5fd7e371657358c7b0a523e54a9cffb87461f8/multiprocess-0.70.18-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:8b8940ae30139e04b076da6c5b83e9398585ebdf0f2ad3250673fef5b2ff06d6", size = 144695, upload-time = "2025-04-17T03:11:09.161Z" }, - { url = "https://files.pythonhosted.org/packages/17/bf/87323e79dd0562474fad3373c21c66bc6c3c9963b68eb2a209deb4c8575e/multiprocess-0.70.18-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:0929ba95831adb938edbd5fb801ac45e705ecad9d100b3e653946b7716cb6bd3", size = 144742, upload-time = "2025-04-17T03:11:10.072Z" }, - { url = "https://files.pythonhosted.org/packages/dd/74/cb8c831e58dc6d5cf450b17c7db87f14294a1df52eb391da948b5e0a0b94/multiprocess-0.70.18-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4d77f8e4bfe6c6e2e661925bbf9aed4d5ade9a1c6502d5dfc10129b9d1141797", size = 144745, upload-time = "2025-04-17T03:11:11.453Z" }, - { url = "https://files.pythonhosted.org/packages/ba/d8/0cba6cf51a1a31f20471fbc823a716170c73012ddc4fb85d706630ed6e8f/multiprocess-0.70.18-py310-none-any.whl", hash = "sha256:60c194974c31784019c1f459d984e8f33ee48f10fcf42c309ba97b30d9bd53ea", size = 134948, upload-time = "2025-04-17T03:11:20.223Z" }, - { url = "https://files.pythonhosted.org/packages/4b/88/9039f2fed1012ef584751d4ceff9ab4a51e5ae264898f0b7cbf44340a859/multiprocess-0.70.18-py311-none-any.whl", hash = "sha256:5aa6eef98e691281b3ad923be2832bf1c55dd2c859acd73e5ec53a66aae06a1d", size = 144462, upload-time = "2025-04-17T03:11:21.657Z" }, - { url = "https://files.pythonhosted.org/packages/bf/b6/5f922792be93b82ec6b5f270bbb1ef031fd0622847070bbcf9da816502cc/multiprocess-0.70.18-py312-none-any.whl", hash = "sha256:9b78f8e5024b573730bfb654783a13800c2c0f2dfc0c25e70b40d184d64adaa2", size = 150287, upload-time = "2025-04-17T03:11:22.69Z" }, - { url = "https://files.pythonhosted.org/packages/3b/c3/ca84c19bd14cdfc21c388fdcebf08b86a7a470ebc9f5c3c084fc2dbc50f7/multiprocess-0.70.18-py38-none-any.whl", hash = "sha256:dbf705e52a154fe5e90fb17b38f02556169557c2dd8bb084f2e06c2784d8279b", size = 132636, upload-time = "2025-04-17T03:11:24.936Z" }, - { url = "https://files.pythonhosted.org/packages/6c/28/dd72947e59a6a8c856448a5e74da6201cb5502ddff644fbc790e4bd40b9a/multiprocess-0.70.18-py39-none-any.whl", hash = "sha256:e78ca805a72b1b810c690b6b4cc32579eba34f403094bbbae962b7b5bf9dfcb8", size = 133478, upload-time = "2025-04-17T03:11:26.253Z" }, + { url = "https://files.pythonhosted.org/packages/a9/82/0340caa499416c78e5d8f5f05947ae4bc3cba53c9f038ab6e9ed964e22f1/nbformat-5.10.4-py3-none-any.whl", hash = "sha256:3b48d6c8fbca4b299bf3982ea7db1af21580e4fec269ad087b9e81588891200b", size = 78454, upload-time = "2024-04-04T11:20:34.895Z" }, +] + +[[package]] +name = "nemollm" +version = "0.3.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, + { name = "requests-futures" }, + { name = "requests-toolbelt" }, + { name = "tqdm" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/c1/0975d3e7b3293fc4480fe6521ddfcba2a02da131f7751c295430c797e5f8/nemollm-0.3.5-py3-none-any.whl", hash = "sha256:de3962d82d557ce5f3d773144361b897a181028b8e1bbdf94804eff865eead88", size = 11573, upload-time = "2024-01-05T19:46:57.009Z" }, +] + +[[package]] +name = "nest-asyncio" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, +] + +[[package]] +name = "networkx" +version = "3.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/4f/ccdb8ad3a38e583f214547fd2f7ff1fc160c43a75af88e6aec213404b96a/networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037", size = 2471065, upload-time = "2025-05-29T11:35:07.804Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", size = 2034406, upload-time = "2025-05-29T11:35:04.961Z" }, +] + +[[package]] +name = "nltk" +version = "3.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "joblib" }, + { name = "regex" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3c/87/db8be88ad32c2d042420b6fd9ffd4a149f9a0d7f0e86b3f543be2eeeedd2/nltk-3.9.1.tar.gz", hash = "sha256:87d127bd3de4bd89a4f81265e5fa59cb1b199b27440175370f7417d2bc7ae868", size = 2904691, upload-time = "2024-08-18T19:48:37.769Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/66/7d9e26593edda06e8cb531874633f7c2372279c3b0f46235539fe546df8b/nltk-3.9.1-py3-none-any.whl", hash = "sha256:4fa26829c5b00715afe3061398a8989dc643b92ce7dd93fb4585a70930d168a1", size = 1505442, upload-time = "2024-08-18T19:48:21.909Z" }, +] + +[[package]] +name = "numpy" +version = "1.26.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/6e/09db70a523a96d25e115e71cc56a6f9031e7b8cd166c1ac8438307c14058/numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", size = 15786129, upload-time = "2024-02-06T00:26:44.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/57/baae43d14fe163fa0e4c47f307b6b2511ab8d7d30177c491960504252053/numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71", size = 20630554, upload-time = "2024-02-05T23:51:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/1a/2e/151484f49fd03944c4a3ad9c418ed193cfd02724e138ac8a9505d056c582/numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef", size = 13997127, upload-time = "2024-02-05T23:52:15.314Z" }, + { url = "https://files.pythonhosted.org/packages/79/ae/7e5b85136806f9dadf4878bf73cf223fe5c2636818ba3ab1c585d0403164/numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e", size = 14222994, upload-time = "2024-02-05T23:52:47.569Z" }, + { url = "https://files.pythonhosted.org/packages/3a/d0/edc009c27b406c4f9cbc79274d6e46d634d139075492ad055e3d68445925/numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5", size = 18252005, upload-time = "2024-02-05T23:53:15.637Z" }, + { url = "https://files.pythonhosted.org/packages/09/bf/2b1aaf8f525f2923ff6cfcf134ae5e750e279ac65ebf386c75a0cf6da06a/numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a", size = 13885297, upload-time = "2024-02-05T23:53:42.16Z" }, + { url = "https://files.pythonhosted.org/packages/df/a0/4e0f14d847cfc2a633a1c8621d00724f3206cfeddeb66d35698c4e2cf3d2/numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a", size = 18093567, upload-time = "2024-02-05T23:54:11.696Z" }, + { url = "https://files.pythonhosted.org/packages/d2/b7/a734c733286e10a7f1a8ad1ae8c90f2d33bf604a96548e0a4a3a6739b468/numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20", size = 5968812, upload-time = "2024-02-05T23:54:26.453Z" }, + { url = "https://files.pythonhosted.org/packages/3f/6b/5610004206cf7f8e7ad91c5a85a8c71b2f2f8051a0c0c4d5916b76d6cbb2/numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2", size = 15811913, upload-time = "2024-02-05T23:54:53.933Z" }, + { url = "https://files.pythonhosted.org/packages/95/12/8f2020a8e8b8383ac0177dc9570aad031a3beb12e38847f7129bacd96228/numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218", size = 20335901, upload-time = "2024-02-05T23:55:32.801Z" }, + { url = "https://files.pythonhosted.org/packages/75/5b/ca6c8bd14007e5ca171c7c03102d17b4f4e0ceb53957e8c44343a9546dcc/numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b", size = 13685868, upload-time = "2024-02-05T23:55:56.28Z" }, + { url = "https://files.pythonhosted.org/packages/79/f8/97f10e6755e2a7d027ca783f63044d5b1bc1ae7acb12afe6a9b4286eac17/numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b", size = 13925109, upload-time = "2024-02-05T23:56:20.368Z" }, + { url = "https://files.pythonhosted.org/packages/0f/50/de23fde84e45f5c4fda2488c759b69990fd4512387a8632860f3ac9cd225/numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed", size = 17950613, upload-time = "2024-02-05T23:56:56.054Z" }, + { url = "https://files.pythonhosted.org/packages/4c/0c/9c603826b6465e82591e05ca230dfc13376da512b25ccd0894709b054ed0/numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a", size = 13572172, upload-time = "2024-02-05T23:57:21.56Z" }, + { url = "https://files.pythonhosted.org/packages/76/8c/2ba3902e1a0fc1c74962ea9bb33a534bb05984ad7ff9515bf8d07527cadd/numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0", size = 17786643, upload-time = "2024-02-05T23:57:56.585Z" }, + { url = "https://files.pythonhosted.org/packages/28/4a/46d9e65106879492374999e76eb85f87b15328e06bd1550668f79f7b18c6/numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110", size = 5677803, upload-time = "2024-02-05T23:58:08.963Z" }, + { url = "https://files.pythonhosted.org/packages/16/2e/86f24451c2d530c88daf997cb8d6ac622c1d40d19f5a031ed68a4b73a374/numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818", size = 15517754, upload-time = "2024-02-05T23:58:36.364Z" }, +] + +[[package]] +name = "nvidia-nat" +version = "1.2.0rc8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aioboto3" }, + { name = "authlib" }, + { name = "click" }, + { name = "colorama" }, + { name = "datasets" }, + { name = "expandvars" }, + { name = "fastapi" }, + { name = "httpx" }, + { name = "jinja2" }, + { name = "jsonpath-ng" }, + { name = "mcp" }, + { name = "networkx" }, + { name = "numpy" }, + { name = "openinference-semantic-conventions" }, + { name = "openpyxl" }, + { name = "pkce" }, + { name = "pkginfo" }, + { name = "platformdirs" }, + { name = "pydantic" }, + { name = "pymilvus" }, + { name = "pyyaml" }, + { name = "ragas" }, + { name = "rich" }, + { name = "tabulate" }, + { name = "uvicorn", extra = ["standard"] }, + { name = "wikipedia" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/95/a487db754505fa0d66fbd13fbd85a5aa3cf0bc81d5fec1a036cd1032be10/nvidia_nat-1.2.0rc8-py3-none-any.whl", hash = "sha256:30c23e798a567b791f2f82a95fd185c0f58597d85c5d9975e78c772b0f063bf4", size = 713049, upload-time = "2025-08-15T01:20:49.358Z" }, +] + +[package.optional-dependencies] +langchain = [ + { name = "nvidia-nat-langchain" }, +] +opentelemetry = [ + { name = "nvidia-nat-opentelemetry" }, +] +profiling = [ + { name = "matplotlib" }, + { name = "prefixspan" }, + { name = "scikit-learn" }, +] +telemetry = [ + { name = "nvidia-nat-opentelemetry" }, + { name = "nvidia-nat-phoenix" }, + { name = "nvidia-nat-ragaai" }, + { name = "nvidia-nat-weave" }, +] + +[[package]] +name = "nvidia-nat-langchain" +version = "1.2.0rc8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-aws" }, + { name = "langchain-core" }, + { name = "langchain-milvus" }, + { name = "langchain-nvidia-ai-endpoints" }, + { name = "langchain-openai" }, + { name = "langgraph" }, + { name = "nvidia-nat" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/ab/90768e1cd4b46d7f19ea6fc27ac1c9b649a8b5566159fe9758764122010a/nvidia_nat_langchain-1.2.0rc8-py3-none-any.whl", hash = "sha256:4d0f142ceb201e3ce041e4b067d6c6fdc217a8a841289542bdc6ea936b3c2958", size = 13669, upload-time = "2025-08-15T01:21:07.928Z" }, +] + +[[package]] +name = "nvidia-nat-opentelemetry" +version = "1.2.0rc8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nat" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp" }, + { name = "opentelemetry-sdk" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/91/1c53e70ca026af1422aff99336b2be992d770ab1c1ae930a28b7e5edf75d/nvidia_nat_opentelemetry-1.2.0rc8-py3-none-any.whl", hash = "sha256:cde60971c49683675657ac554bbdd17ee0872da2b8c637b1dec248406cb6af40", size = 18699, upload-time = "2025-08-15T01:19:57.304Z" }, +] + +[[package]] +name = "nvidia-nat-phoenix" +version = "1.2.0rc8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "arize-phoenix" }, + { name = "nvidia-nat", extra = ["opentelemetry"] }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/8b/e1a55c848d66c143865da6f54f29089b01357610a3f13aa87a1500c19c6f/nvidia_nat_phoenix-1.2.0rc8-py3-none-any.whl", hash = "sha256:999b957c4515fda7bb2c2a2ca5ca8bf42297e5a88c52fe1fc32dfa500e1d8ab3", size = 8035, upload-time = "2025-08-15T01:19:22.418Z" }, +] + +[[package]] +name = "nvidia-nat-ragaai" +version = "1.2.0rc8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nat", extra = ["opentelemetry"] }, + { name = "ragaai-catalyst" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/b5/1f64bca65c619a0b83b8f745eb6c06ed5b763578e4cc577852db8b23a262/nvidia_nat_ragaai-1.2.0rc8-py3-none-any.whl", hash = "sha256:12fff01606239603477bbcfd424b0559209d27e0d32cdcf3b6f1bf1fa7f63620", size = 10224, upload-time = "2025-08-15T01:20:14.599Z" }, +] + +[[package]] +name = "nvidia-nat-weave" +version = "1.2.0rc8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nat" }, + { name = "presidio-analyzer" }, + { name = "presidio-anonymizer" }, + { name = "weave" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/76/b8b74f1d6e39b0462ef70dfacc1cffd9383a0229463c734fe0e7619d3c2f/nvidia_nat_weave-1.2.0rc8-py3-none-any.whl", hash = "sha256:b88e6437f9caa8116ee1331f4a7f90c1434d4f58f3d15715b88d0f102bd85172", size = 7518, upload-time = "2025-08-15T01:21:25.292Z" }, +] + +[[package]] +name = "openai" +version = "1.100.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/71/efa1ddeedcba1a385034b66e6f34c7b0ace4bf99ffbfc2859c025daa147a/openai-1.100.1.tar.gz", hash = "sha256:3e9ae652903e5120514e544af2426334141404657cdcdb6dc6845fc243d66e66", size = 508419, upload-time = "2025-08-18T21:57:44.702Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/b4/686944f0903c65202e86311ec0f42171e697e4f7324caeee0c318046b738/openai-1.100.1-py3-none-any.whl", hash = "sha256:2e8224caaf3136c58e30e6b3984fd7a8e6da0931d2c36fbbb7d668e5c11db914", size = 788286, upload-time = "2025-08-18T21:57:42.754Z" }, +] + +[[package]] +name = "openinference-instrumentation" +version = "0.1.37" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-sdk" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/e0/9d3fe148d27602f794840ba7e2353ba1f25ff6f43ad32fe4390fba393ba4/openinference_instrumentation-0.1.37.tar.gz", hash = "sha256:67fe1c83a864c0cb38a19165b63f28b7287f5c0d7924c47dad599e006a934fd1", size = 23012, upload-time = "2025-08-06T00:29:46.227Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d8/68/acb8517ab0b1238114c9470dfb3825d30ab75c871f0f26dc0b28325a8f5c/openinference_instrumentation-0.1.37-py3-none-any.whl", hash = "sha256:4165642efbcad3b59b1fbf22914f8e600af51845aa5290c9b2683022795b4dfa", size = 28829, upload-time = "2025-08-06T00:29:45.037Z" }, +] + +[[package]] +name = "openinference-instrumentation-anthropic" +version = "0.1.19" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0c/11/a3f1c8603fc9ed75dc52d6df119e5c59b22ac000cf5d777d4e175dff4875/openinference_instrumentation_anthropic-0.1.19.tar.gz", hash = "sha256:974ea6aaba5bc7441d0f2227f355a781ce860bf9c2996db45a14442c4402ed8e", size = 14262, upload-time = "2025-08-18T18:13:02.274Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/cc/e7d5ee816b7ef287d76168bb61dcef47c39f9b9ebd7cd86487ff1ee9688d/openinference_instrumentation_anthropic-0.1.19-py3-none-any.whl", hash = "sha256:fa513ed575e245953a2124c3d0cf8cc562b263dace00d6a717e002a3dde3ce39", size = 17389, upload-time = "2025-08-18T18:12:58.796Z" }, +] + +[[package]] +name = "openinference-instrumentation-bedrock" +version = "0.1.26" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dacite" }, + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d4/b6/5c0167e364da21bd0b2aea3b88fbde0a8d77e14fccfe7948d3608c438b22/openinference_instrumentation_bedrock-0.1.26.tar.gz", hash = "sha256:2191f664ec9791f8f42d15fb6af43f3937640249c1aeeb7edbd6a719b3cfd77b", size = 170038, upload-time = "2025-06-19T16:54:19.165Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/d3/6d52f61360abe0cda5cdd7be71fb64f3953718fe74f03be82cd1087af631/openinference_instrumentation_bedrock-0.1.26-py3-none-any.whl", hash = "sha256:953d1bc8b958d56b03263a5752b2e9a8c6a33069d05e1233527a21a6dd1505ce", size = 50677, upload-time = "2025-06-19T16:54:17.805Z" }, +] + +[[package]] +name = "openinference-instrumentation-crewai" +version = "0.1.11" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7e/30/95276c074a257d8c6a92e7f92637eaf78c1f89fb6281e82f76c17c3d27d3/openinference_instrumentation_crewai-0.1.11.tar.gz", hash = "sha256:61234cce6aead3debd18a2b0eab6b8fb70776a268e14ae71f1f570d707e861aa", size = 10323, upload-time = "2025-07-16T21:17:37.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/7c/7a2ae73f88ea6ab1a6a6242a83aa3070859aafb3a9e5746250b0f6c4f275/openinference_instrumentation_crewai-0.1.11-py3-none-any.whl", hash = "sha256:096975809acc2326c409b5d3fb12cc264f1012f9d207b1b820b172e1b83a147a", size = 11845, upload-time = "2025-07-16T21:17:36.768Z" }, +] + +[[package]] +name = "openinference-instrumentation-google-adk" +version = "0.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4f/0e/78e1613d97f6d197bf0b59c4e6c975b2360b417e036316736d71d78833e1/openinference_instrumentation_google_adk-0.1.3.tar.gz", hash = "sha256:6cf913d83d08010bd638e02be08a9d74b4d0f76e804b6d9a369cabd18cb3c551", size = 11852, upload-time = "2025-08-04T20:53:47.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/3f/561981f1df408a8dfe6131d904f1a6ad65a3fe3d6ad79c01ee060f7e86b5/openinference_instrumentation_google_adk-0.1.3-py3-none-any.whl", hash = "sha256:b4b27c4e3891513f7c056f55ae8bae50ab0f85a2e8cfd918e55fa57e1518fa6e", size = 13660, upload-time = "2025-08-04T20:53:46.086Z" }, +] + +[[package]] +name = "openinference-instrumentation-groq" +version = "0.1.11" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4f/77/4bafc9b2307325e97afc9857e18eaa2d952ebeedb4063f4f4ae82fa9624f/openinference_instrumentation_groq-0.1.11.tar.gz", hash = "sha256:d23d640822dd66682f779c490b7d9cebbf26895cc06ff75d6ae8e1df8210f1e8", size = 11817, upload-time = "2025-04-28T23:14:06.058Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/1a/073e8bea94af702bf554f0a1c53a403783ef942fdd07a5ec286a4d642d0a/openinference_instrumentation_groq-0.1.11-py3-none-any.whl", hash = "sha256:b5303469870ba474dc3461558730c18cf5ef787b490386f72d20a250ede9cf36", size = 15717, upload-time = "2025-04-28T23:13:56.832Z" }, +] + +[[package]] +name = "openinference-instrumentation-haystack" +version = "0.1.24" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/15/d750f65dff58524fb7c1d679bae91948447e1a3547bc32e662e6d3ac7fcc/openinference_instrumentation_haystack-0.1.24.tar.gz", hash = "sha256:14aea1e46d16415e373dd3f65d4e2ae94a1aa705b7b6967e606ab6e5dbb2cc18", size = 13001, upload-time = "2025-05-31T00:10:31.585Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/2f/0bad4a4840de04defa9f82fc6ffaa48b02dd19aa085fdf08701ba349cb59/openinference_instrumentation_haystack-0.1.24-py3-none-any.whl", hash = "sha256:55628441fdccb13904c7bc90e3621d18d346acf9bf3884b9fac672a6dba9ad2f", size = 14415, upload-time = "2025-05-31T00:10:29.683Z" }, +] + +[[package]] +name = "openinference-instrumentation-langchain" +version = "0.1.50" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "wrapt" }, ] - -[[package]] -name = "mypy-extensions" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +sdist = { url = "https://files.pythonhosted.org/packages/88/7b/8b0b8ecb147f352363bfbe01202af8fbd69d3cbb4e5c5c40905688ab940f/openinference_instrumentation_langchain-0.1.50.tar.gz", hash = "sha256:9ef7ca19f72d6dd66dabcf84a0d0c5c4ec6fe659a846b6211beb486129a21e4e", size = 66966, upload-time = "2025-07-30T16:50:38.091Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, + { url = "https://files.pythonhosted.org/packages/5a/91/4853bbe5c6df3606f3d660865ca14058cbd65971962f14581be6fcaa5242/openinference_instrumentation_langchain-0.1.50-py3-none-any.whl", hash = "sha256:5fe2f206c6ebd61a6a815042d763ac7dc2576a6033c99e6a54fe0b467192a022", size = 21312, upload-time = "2025-07-30T16:50:36.6Z" }, ] [[package]] -name = "nbformat" -version = "5.10.4" +name = "openinference-instrumentation-litellm" +version = "0.1.25" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "fastjsonschema" }, - { name = "jsonschema" }, - { name = "jupyter-core" }, - { name = "traitlets" }, + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-sdk" }, + { name = "setuptools" }, + { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6d/fd/91545e604bc3dad7dca9ed03284086039b294c6b3d75c0d2fa45f9e9caf3/nbformat-5.10.4.tar.gz", hash = "sha256:322168b14f937a5d11362988ecac2a4952d3d8e3a2cbeb2319584631226d5b3a", size = 142749, upload-time = "2024-04-04T11:20:37.371Z" } +sdist = { url = "https://files.pythonhosted.org/packages/14/c3/1b980a8d7142a9087278e8dafce94c80976d5da7ec753d469dacf6ccd722/openinference_instrumentation_litellm-0.1.25.tar.gz", hash = "sha256:d289f670f38c407542220ffdb3f89759134f02926f15ed1b57e975ca12f19ad5", size = 19292, upload-time = "2025-08-18T18:13:05.196Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/82/0340caa499416c78e5d8f5f05947ae4bc3cba53c9f038ab6e9ed964e22f1/nbformat-5.10.4-py3-none-any.whl", hash = "sha256:3b48d6c8fbca4b299bf3982ea7db1af21580e4fec269ad087b9e81588891200b", size = 78454, upload-time = "2024-04-04T11:20:34.895Z" }, + { url = "https://files.pythonhosted.org/packages/11/2a/bb35cbb951edd511fb311c4bd9c934fe07da4748018541b687c968a635b8/openinference_instrumentation_litellm-0.1.25-py3-none-any.whl", hash = "sha256:a09de0fdff8b2141fea128535be2676df452f5faa774694055723b3fbd3fca5c", size = 13137, upload-time = "2025-08-18T18:13:01.256Z" }, ] [[package]] -name = "nemollm" -version = "0.3.5" +name = "openinference-instrumentation-llama-index" +version = "4.3.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "python-dateutil" }, - { name = "requests-futures" }, - { name = "requests-toolbelt" }, - { name = "tqdm" }, + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, { name = "typing-extensions" }, - { name = "urllib3" }, + { name = "wrapt" }, ] +sdist = { url = "https://files.pythonhosted.org/packages/89/df/6622fb09eacfad140067c5c6f649c6a3d835f081b5cfba6c8fa5252ab944/openinference_instrumentation_llama_index-4.3.4.tar.gz", hash = "sha256:2e03347bf7d9d6f7ff4b62101239c7408c2a6c69065f7d00f9a3b1e2145ac626", size = 60693, upload-time = "2025-08-01T22:33:33.532Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/c1/0975d3e7b3293fc4480fe6521ddfcba2a02da131f7751c295430c797e5f8/nemollm-0.3.5-py3-none-any.whl", hash = "sha256:de3962d82d557ce5f3d773144361b897a181028b8e1bbdf94804eff865eead88", size = 11573, upload-time = "2024-01-05T19:46:57.009Z" }, + { url = "https://files.pythonhosted.org/packages/be/a7/e63f05f49ea66d416c770b92f0670723811c285e2f10c13ce91c0558eb4b/openinference_instrumentation_llama_index-4.3.4-py3-none-any.whl", hash = "sha256:aa94297ede190cd55f62e04dd050fbb7975fec55a63752df8f4faec6b403bac0", size = 28815, upload-time = "2025-08-01T22:33:32.285Z" }, ] [[package]] -name = "nest-asyncio" -version = "1.6.0" +name = "openinference-instrumentation-mistralai" +version = "1.3.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, +dependencies = [ + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "wrapt" }, ] - -[[package]] -name = "networkx" -version = "3.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6c/4f/ccdb8ad3a38e583f214547fd2f7ff1fc160c43a75af88e6aec213404b96a/networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037", size = 2471065, upload-time = "2025-05-29T11:35:07.804Z" } +sdist = { url = "https://files.pythonhosted.org/packages/12/93/40e5f7b951fe309893f0fc4ff615264e021a3ebfc63cbe3815fd6b0f8cb3/openinference_instrumentation_mistralai-1.3.3.tar.gz", hash = "sha256:a55472a52dfa179058b851f2e502346a3cc1acadb023dd5b66b77c542369480a", size = 21307, upload-time = "2025-04-28T23:14:06.082Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", size = 2034406, upload-time = "2025-05-29T11:35:04.961Z" }, + { url = "https://files.pythonhosted.org/packages/c0/f0/8e5678d39ff0cac31f2093f194a2510c8cccf443442df01bbab8042aaff9/openinference_instrumentation_mistralai-1.3.3-py3-none-any.whl", hash = "sha256:f7323f4de35dc4aacf80be40d105a2f800f3ff7f9e0404b052d51c657e0d33e0", size = 20386, upload-time = "2025-04-28T23:13:55.994Z" }, ] [[package]] -name = "numpy" -version = "1.26.4" +name = "openinference-instrumentation-openai" +version = "0.1.31" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/65/6e/09db70a523a96d25e115e71cc56a6f9031e7b8cd166c1ac8438307c14058/numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", size = 15786129, upload-time = "2024-02-06T00:26:44.495Z" } +dependencies = [ + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f9/50/a7c250db10743b10a667d9fd70fee3ac0a56b4934a60a9d7a348f93a8b94/openinference_instrumentation_openai-0.1.31.tar.gz", hash = "sha256:9e52248bdce499fa4eff3a5567cfd55eb916a7b2f533f51b2ce459b03d905bad", size = 21450, upload-time = "2025-08-18T18:12:59.128Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/11/57/baae43d14fe163fa0e4c47f307b6b2511ab8d7d30177c491960504252053/numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71", size = 20630554, upload-time = "2024-02-05T23:51:50.149Z" }, - { url = "https://files.pythonhosted.org/packages/1a/2e/151484f49fd03944c4a3ad9c418ed193cfd02724e138ac8a9505d056c582/numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef", size = 13997127, upload-time = "2024-02-05T23:52:15.314Z" }, - { url = "https://files.pythonhosted.org/packages/79/ae/7e5b85136806f9dadf4878bf73cf223fe5c2636818ba3ab1c585d0403164/numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e", size = 14222994, upload-time = "2024-02-05T23:52:47.569Z" }, - { url = "https://files.pythonhosted.org/packages/3a/d0/edc009c27b406c4f9cbc79274d6e46d634d139075492ad055e3d68445925/numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5", size = 18252005, upload-time = "2024-02-05T23:53:15.637Z" }, - { url = "https://files.pythonhosted.org/packages/09/bf/2b1aaf8f525f2923ff6cfcf134ae5e750e279ac65ebf386c75a0cf6da06a/numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a", size = 13885297, upload-time = "2024-02-05T23:53:42.16Z" }, - { url = "https://files.pythonhosted.org/packages/df/a0/4e0f14d847cfc2a633a1c8621d00724f3206cfeddeb66d35698c4e2cf3d2/numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a", size = 18093567, upload-time = "2024-02-05T23:54:11.696Z" }, - { url = "https://files.pythonhosted.org/packages/d2/b7/a734c733286e10a7f1a8ad1ae8c90f2d33bf604a96548e0a4a3a6739b468/numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20", size = 5968812, upload-time = "2024-02-05T23:54:26.453Z" }, - { url = "https://files.pythonhosted.org/packages/3f/6b/5610004206cf7f8e7ad91c5a85a8c71b2f2f8051a0c0c4d5916b76d6cbb2/numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2", size = 15811913, upload-time = "2024-02-05T23:54:53.933Z" }, - { url = "https://files.pythonhosted.org/packages/95/12/8f2020a8e8b8383ac0177dc9570aad031a3beb12e38847f7129bacd96228/numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218", size = 20335901, upload-time = "2024-02-05T23:55:32.801Z" }, - { url = "https://files.pythonhosted.org/packages/75/5b/ca6c8bd14007e5ca171c7c03102d17b4f4e0ceb53957e8c44343a9546dcc/numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b", size = 13685868, upload-time = "2024-02-05T23:55:56.28Z" }, - { url = "https://files.pythonhosted.org/packages/79/f8/97f10e6755e2a7d027ca783f63044d5b1bc1ae7acb12afe6a9b4286eac17/numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b", size = 13925109, upload-time = "2024-02-05T23:56:20.368Z" }, - { url = "https://files.pythonhosted.org/packages/0f/50/de23fde84e45f5c4fda2488c759b69990fd4512387a8632860f3ac9cd225/numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed", size = 17950613, upload-time = "2024-02-05T23:56:56.054Z" }, - { url = "https://files.pythonhosted.org/packages/4c/0c/9c603826b6465e82591e05ca230dfc13376da512b25ccd0894709b054ed0/numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a", size = 13572172, upload-time = "2024-02-05T23:57:21.56Z" }, - { url = "https://files.pythonhosted.org/packages/76/8c/2ba3902e1a0fc1c74962ea9bb33a534bb05984ad7ff9515bf8d07527cadd/numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0", size = 17786643, upload-time = "2024-02-05T23:57:56.585Z" }, - { url = "https://files.pythonhosted.org/packages/28/4a/46d9e65106879492374999e76eb85f87b15328e06bd1550668f79f7b18c6/numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110", size = 5677803, upload-time = "2024-02-05T23:58:08.963Z" }, - { url = "https://files.pythonhosted.org/packages/16/2e/86f24451c2d530c88daf997cb8d6ac622c1d40d19f5a031ed68a4b73a374/numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818", size = 15517754, upload-time = "2024-02-05T23:58:36.364Z" }, + { url = "https://files.pythonhosted.org/packages/d5/50/99b1562a6d9093d48548cf8df1820579e4f7239f8ef10f03f01af014355d/openinference_instrumentation_openai-0.1.31-py3-none-any.whl", hash = "sha256:225873a9cebf3a5d1bd258675d55caa88adb91ed057283fbbad904660aec996c", size = 28492, upload-time = "2025-08-18T18:12:57.687Z" }, ] [[package]] -name = "openai" -version = "1.96.1" +name = "openinference-instrumentation-openai-agents" +version = "1.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "httpx" }, - { name = "jiter" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "tqdm" }, + { name = "openinference-instrumentation" }, + { name = "openinference-semantic-conventions" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, { name = "typing-extensions" }, + { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2f/b5/18fd5e1b6b6c7dca52d60307b3637f9e9e3206a8041a9c8028985dbc6260/openai-1.96.1.tar.gz", hash = "sha256:6d505b5cc550e036bfa3fe99d6cff565b11491d12378d4c353f92ef72b0a408a", size = 489065, upload-time = "2025-07-15T21:39:37.215Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/d1/965d8b2ef7aa782264ede8c4fc60749f4641004e44aa4e98861253fc1d1d/openinference_instrumentation_openai_agents-1.2.0.tar.gz", hash = "sha256:8c042870fbe4775f8d59d0a72f56bbb4acf555d7b15432042773271645d72e4d", size = 12414, upload-time = "2025-08-18T18:12:58.189Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/57/325bbdbdc27b47309be35cb4e0eb8980b0c1bc997194c797c3691d88ae41/openai-1.96.1-py3-none-any.whl", hash = "sha256:0afaab2019bae8e145e7a1baf6953167084f019dd15042c65edd117398c1eb1c", size = 757454, upload-time = "2025-07-15T21:39:34.517Z" }, + { url = "https://files.pythonhosted.org/packages/ac/90/746a165fa5878beb6cf2b312c1c100c98de0b3e8dc9a800dae7b179af2b3/openinference_instrumentation_openai_agents-1.2.0-py3-none-any.whl", hash = "sha256:df4de2cb3ceb57a519d7649c5250b44b45cf563da7b758cd207ba16d313ffbeb", size = 14235, upload-time = "2025-08-18T18:12:57.274Z" }, ] [[package]] -name = "openinference-instrumentation" -version = "0.1.35" +name = "openinference-instrumentation-smolagents" +version = "0.1.14" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "openinference-instrumentation" }, { name = "openinference-semantic-conventions" }, { name = "opentelemetry-api" }, - { name = "opentelemetry-sdk" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, + { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cf/ef/1147c17a23c8a832787fe42a485613097383ec583f3de908b59fbe3ae9e7/openinference_instrumentation-0.1.35.tar.gz", hash = "sha256:cdb302e2007a41a52e2b6a66de3a9fa8c0bc737edefd370703455b50d1ae23e9", size = 22426, upload-time = "2025-07-16T05:12:19.602Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/3f/7acd78b90ce7a77c7beb32f4af86c1efcf669d45019160ad1ee2af3ccdff/openinference_instrumentation_smolagents-0.1.14.tar.gz", hash = "sha256:9a24a778e5f34c3837576fcf92bf98a4af34fa213249a04d6776e82c51ccdde6", size = 11107, upload-time = "2025-07-10T20:20:23.012Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/8d/ab5fca6e442ce1a3827aecc3fc00d3773ac662a5c756e8ebc37ee94146fc/openinference_instrumentation-0.1.35-py3-none-any.whl", hash = "sha256:baf5a49a7b017ad3c3e5eb606a7abea66eec8a3a44093d590e3dcd694dd879f8", size = 28194, upload-time = "2025-07-16T05:12:18.735Z" }, + { url = "https://files.pythonhosted.org/packages/ee/b8/43652d8452e8faaed85535b9652412e8393ba1e288a1a4cc267e6c6aff04/openinference_instrumentation_smolagents-0.1.14-py3-none-any.whl", hash = "sha256:8f818ac8df18e57588b267dc033ef76065aa7c2af930864309931b7b1f3a8966", size = 12719, upload-time = "2025-07-10T20:20:22.055Z" }, ] [[package]] -name = "openinference-instrumentation-langchain" -version = "0.1.46" +name = "openinference-instrumentation-vertexai" +version = "0.1.11" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "openinference-instrumentation" }, @@ -1965,9 +3204,9 @@ dependencies = [ { name = "opentelemetry-semantic-conventions" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/25/d9/2936c480d6f6ccbd2f045ef8a0838a663e17f1513bbf535267c0d28cfffa/openinference_instrumentation_langchain-0.1.46.tar.gz", hash = "sha256:0d3d41723bc985b3804cf916d25890f1ba5245e8e51aba7b4730556d4332de5a", size = 53166, upload-time = "2025-07-03T16:37:56.021Z" } +sdist = { url = "https://files.pythonhosted.org/packages/06/3e/a1cd041ed1237e80178c70e5d71d95417225f84065e3aae854b70173f01c/openinference_instrumentation_vertexai-0.1.11.tar.gz", hash = "sha256:600d9be988af6e1371e17187cf6aeed3ebb893d1ca0ef567cc00f67b3242aba6", size = 21061, upload-time = "2025-05-05T16:38:46.18Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/47/46b4b25febbc362b4f4d9406d66ce42421c2e0dcc4ee403d1c1771c0306e/openinference_instrumentation_langchain-0.1.46-py3-none-any.whl", hash = "sha256:e6ee9b6b26e44467bcec846a6b31ee0f0df686a7c887fc7c7f5ad2be64122033", size = 19605, upload-time = "2025-07-03T16:37:54.549Z" }, + { url = "https://files.pythonhosted.org/packages/8f/c9/e2ea586fec1f7650d0f8c6e630417859851995ed5ba693315d1ef282de57/openinference_instrumentation_vertexai-0.1.11-py3-none-any.whl", hash = "sha256:5fceb31a89aea0d8b7c93b7b89f8095888fedf8f8944518b2d803b02f39810dc", size = 17169, upload-time = "2025-05-05T16:38:45.151Z" }, ] [[package]] @@ -1993,45 +3232,45 @@ wheels = [ [[package]] name = "opentelemetry-api" -version = "1.35.0" +version = "1.36.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "importlib-metadata" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/99/c9/4509bfca6bb43220ce7f863c9f791e0d5001c2ec2b5867d48586008b3d96/opentelemetry_api-1.35.0.tar.gz", hash = "sha256:a111b959bcfa5b4d7dffc2fbd6a241aa72dd78dd8e79b5b1662bda896c5d2ffe", size = 64778, upload-time = "2025-07-11T12:23:28.804Z" } +sdist = { url = "https://files.pythonhosted.org/packages/27/d2/c782c88b8afbf961d6972428821c302bd1e9e7bc361352172f0ca31296e2/opentelemetry_api-1.36.0.tar.gz", hash = "sha256:9a72572b9c416d004d492cbc6e61962c0501eaf945ece9b5a0f56597d8348aa0", size = 64780, upload-time = "2025-07-29T15:12:06.02Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/5a/3f8d078dbf55d18442f6a2ecedf6786d81d7245844b2b20ce2b8ad6f0307/opentelemetry_api-1.35.0-py3-none-any.whl", hash = "sha256:c4ea7e258a244858daf18474625e9cc0149b8ee354f37843415771a40c25ee06", size = 65566, upload-time = "2025-07-11T12:23:07.944Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ee/6b08dde0a022c463b88f55ae81149584b125a42183407dc1045c486cc870/opentelemetry_api-1.36.0-py3-none-any.whl", hash = "sha256:02f20bcacf666e1333b6b1f04e647dc1d5111f86b8e510238fcc56d7762cda8c", size = 65564, upload-time = "2025-07-29T15:11:47.998Z" }, ] [[package]] name = "opentelemetry-exporter-otlp" -version = "1.35.0" +version = "1.36.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-exporter-otlp-proto-grpc" }, { name = "opentelemetry-exporter-otlp-proto-http" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2e/2e/63718faa67b17f449a7fb7efdc7125a408cbe5d8c0bb35f423f2776d60b5/opentelemetry_exporter_otlp-1.35.0.tar.gz", hash = "sha256:f94feff09b3524df867c7876b79c96cef20068106cb5efe55340e8d08192c8a4", size = 6142, upload-time = "2025-07-11T12:23:30.128Z" } +sdist = { url = "https://files.pythonhosted.org/packages/95/7f/d31294ac28d567a14aefd855756bab79fed69c5a75df712f228f10c47e04/opentelemetry_exporter_otlp-1.36.0.tar.gz", hash = "sha256:72f166ea5a8923ac42889337f903e93af57db8893de200369b07401e98e4e06b", size = 6144, upload-time = "2025-07-29T15:12:07.153Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/db/2da28358d3101ca936c1643becbb4ebd69e9e48acf27f153d735a4813c6b/opentelemetry_exporter_otlp-1.35.0-py3-none-any.whl", hash = "sha256:8e6bb9025f6238db7d69bba7ee37c77e4858d0a1ff22a9e126f7c9e017e83afe", size = 7016, upload-time = "2025-07-11T12:23:10.679Z" }, + { url = "https://files.pythonhosted.org/packages/a0/a2/8966111a285124f3d6156a663ddf2aeddd52843c1a3d6b56cbd9b6c3fd0e/opentelemetry_exporter_otlp-1.36.0-py3-none-any.whl", hash = "sha256:de93b7c45bcc78296998775d52add7c63729e83ef2cd6560730a6b336d7f6494", size = 7018, upload-time = "2025-07-29T15:11:50.498Z" }, ] [[package]] name = "opentelemetry-exporter-otlp-proto-common" -version = "1.35.0" +version = "1.36.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-proto" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/56/d1/887f860529cba7fc3aba2f6a3597fefec010a17bd1b126810724707d9b51/opentelemetry_exporter_otlp_proto_common-1.35.0.tar.gz", hash = "sha256:6f6d8c39f629b9fa5c79ce19a2829dbd93034f8ac51243cdf40ed2196f00d7eb", size = 20299, upload-time = "2025-07-11T12:23:31.046Z" } +sdist = { url = "https://files.pythonhosted.org/packages/34/da/7747e57eb341c59886052d733072bc878424bf20f1d8cf203d508bbece5b/opentelemetry_exporter_otlp_proto_common-1.36.0.tar.gz", hash = "sha256:6c496ccbcbe26b04653cecadd92f73659b814c6e3579af157d8716e5f9f25cbf", size = 20302, upload-time = "2025-07-29T15:12:07.71Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/2c/e31dd3c719bff87fa77391eb7f38b1430d22868c52312cba8aad60f280e5/opentelemetry_exporter_otlp_proto_common-1.35.0-py3-none-any.whl", hash = "sha256:863465de697ae81279ede660f3918680b4480ef5f69dcdac04f30722ed7b74cc", size = 18349, upload-time = "2025-07-11T12:23:11.713Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ed/22290dca7db78eb32e0101738366b5bbda00d0407f00feffb9bf8c3fdf87/opentelemetry_exporter_otlp_proto_common-1.36.0-py3-none-any.whl", hash = "sha256:0fc002a6ed63eac235ada9aa7056e5492e9a71728214a61745f6ad04b923f840", size = 18349, upload-time = "2025-07-29T15:11:51.327Z" }, ] [[package]] name = "opentelemetry-exporter-otlp-proto-grpc" -version = "1.35.0" +version = "1.36.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "googleapis-common-protos" }, @@ -2042,14 +3281,14 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/20/de/222e4f2f8cd39250991f84d76b661534aef457cafc6a3eb3fcd513627698/opentelemetry_exporter_otlp_proto_grpc-1.35.0.tar.gz", hash = "sha256:ac4c2c3aa5674642db0df0091ab43ec08bbd91a9be469c8d9b18923eb742b9cc", size = 23794, upload-time = "2025-07-11T12:23:31.662Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/6f/6c1b0bdd0446e5532294d1d41bf11fbaea39c8a2423a4cdfe4fe6b708127/opentelemetry_exporter_otlp_proto_grpc-1.36.0.tar.gz", hash = "sha256:b281afbf7036b325b3588b5b6c8bb175069e3978d1bd24071f4a59d04c1e5bbf", size = 23822, upload-time = "2025-07-29T15:12:08.292Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/a6/3f60a77279e6a3dc21fc076dcb51be159a633b0bba5cba9fb804062a9332/opentelemetry_exporter_otlp_proto_grpc-1.35.0-py3-none-any.whl", hash = "sha256:ee31203eb3e50c7967b8fa71db366cc355099aca4e3726e489b248cdb2fd5a62", size = 18846, upload-time = "2025-07-11T12:23:12.957Z" }, + { url = "https://files.pythonhosted.org/packages/0c/67/5f6bd188d66d0fd8e81e681bbf5822e53eb150034e2611dd2b935d3ab61a/opentelemetry_exporter_otlp_proto_grpc-1.36.0-py3-none-any.whl", hash = "sha256:734e841fc6a5d6f30e7be4d8053adb703c70ca80c562ae24e8083a28fadef211", size = 18828, upload-time = "2025-07-29T15:11:52.235Z" }, ] [[package]] name = "opentelemetry-exporter-otlp-proto-http" -version = "1.35.0" +version = "1.36.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "googleapis-common-protos" }, @@ -2060,14 +3299,14 @@ dependencies = [ { name = "requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/88/7f/7bdc06e84266a5b4b0fefd9790b3859804bf7682ce2daabcba2e22fdb3b2/opentelemetry_exporter_otlp_proto_http-1.35.0.tar.gz", hash = "sha256:cf940147f91b450ef5f66e9980d40eb187582eed399fa851f4a7a45bb880de79", size = 15908, upload-time = "2025-07-11T12:23:32.335Z" } +sdist = { url = "https://files.pythonhosted.org/packages/25/85/6632e7e5700ba1ce5b8a065315f92c1e6d787ccc4fb2bdab15139eaefc82/opentelemetry_exporter_otlp_proto_http-1.36.0.tar.gz", hash = "sha256:dd3637f72f774b9fc9608ab1ac479f8b44d09b6fb5b2f3df68a24ad1da7d356e", size = 16213, upload-time = "2025-07-29T15:12:08.932Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/71/f118cd90dc26797077931dd598bde5e0cc652519db166593f962f8fcd022/opentelemetry_exporter_otlp_proto_http-1.35.0-py3-none-any.whl", hash = "sha256:9a001e3df3c7f160fb31056a28ed7faa2de7df68877ae909516102ae36a54e1d", size = 18589, upload-time = "2025-07-11T12:23:13.906Z" }, + { url = "https://files.pythonhosted.org/packages/7f/41/a680d38b34f8f5ddbd78ed9f0042e1cc712d58ec7531924d71cb1e6c629d/opentelemetry_exporter_otlp_proto_http-1.36.0-py3-none-any.whl", hash = "sha256:3d769f68e2267e7abe4527f70deb6f598f40be3ea34c6adc35789bea94a32902", size = 18752, upload-time = "2025-07-29T15:11:53.164Z" }, ] [[package]] name = "opentelemetry-instrumentation" -version = "0.56b0" +version = "0.57b0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-api" }, @@ -2075,48 +3314,48 @@ dependencies = [ { name = "packaging" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bb/14/964e90f524655aed5c699190dad8dd9a05ed0f5fa334b4b33532237c2b51/opentelemetry_instrumentation-0.56b0.tar.gz", hash = "sha256:d2dbb3021188ca0ec8c5606349ee9a2919239627e8341d4d37f1d21ec3291d11", size = 28551, upload-time = "2025-07-11T12:26:19.305Z" } +sdist = { url = "https://files.pythonhosted.org/packages/12/37/cf17cf28f945a3aca5a038cfbb45ee01317d4f7f3a0e5209920883fe9b08/opentelemetry_instrumentation-0.57b0.tar.gz", hash = "sha256:f2a30135ba77cdea2b0e1df272f4163c154e978f57214795d72f40befd4fcf05", size = 30807, upload-time = "2025-07-29T15:42:44.746Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/26/aa/2328f27200b8e51640d4d7ff5343ba6a81ab7d2650a9f574db016aae4adf/opentelemetry_instrumentation-0.56b0-py3-none-any.whl", hash = "sha256:948967f7c8f5bdc6e43512ba74c9ae14acb48eb72a35b61afe8db9909f743be3", size = 31105, upload-time = "2025-07-11T12:25:22.788Z" }, + { url = "https://files.pythonhosted.org/packages/d0/6f/f20cd1542959f43fb26a5bf9bb18cd81a1ea0700e8870c8f369bd07f5c65/opentelemetry_instrumentation-0.57b0-py3-none-any.whl", hash = "sha256:9109280f44882e07cec2850db28210b90600ae9110b42824d196de357cbddf7e", size = 32460, upload-time = "2025-07-29T15:41:40.883Z" }, ] [[package]] name = "opentelemetry-proto" -version = "1.35.0" +version = "1.36.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/dc/a2/7366e32d9a2bccbb8614942dbea2cf93c209610385ea966cb050334f8df7/opentelemetry_proto-1.35.0.tar.gz", hash = "sha256:532497341bd3e1c074def7c5b00172601b28bb83b48afc41a4b779f26eb4ee05", size = 46151, upload-time = "2025-07-11T12:23:38.797Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/02/f6556142301d136e3b7e95ab8ea6a5d9dc28d879a99f3dd673b5f97dca06/opentelemetry_proto-1.36.0.tar.gz", hash = "sha256:0f10b3c72f74c91e0764a5ec88fd8f1c368ea5d9c64639fb455e2854ef87dd2f", size = 46152, upload-time = "2025-07-29T15:12:15.717Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/00/a7/3f05de580da7e8a8b8dff041d3d07a20bf3bb62d3bcc027f8fd669a73ff4/opentelemetry_proto-1.35.0-py3-none-any.whl", hash = "sha256:98fffa803164499f562718384e703be8d7dfbe680192279a0429cb150a2f8809", size = 72536, upload-time = "2025-07-11T12:23:23.247Z" }, + { url = "https://files.pythonhosted.org/packages/b3/57/3361e06136225be8180e879199caea520f38026f8071366241ac458beb8d/opentelemetry_proto-1.36.0-py3-none-any.whl", hash = "sha256:151b3bf73a09f94afc658497cf77d45a565606f62ce0c17acb08cd9937ca206e", size = 72537, upload-time = "2025-07-29T15:12:02.243Z" }, ] [[package]] name = "opentelemetry-sdk" -version = "1.35.0" +version = "1.36.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-api" }, { name = "opentelemetry-semantic-conventions" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9a/cf/1eb2ed2ce55e0a9aa95b3007f26f55c7943aeef0a783bb006bdd92b3299e/opentelemetry_sdk-1.35.0.tar.gz", hash = "sha256:2a400b415ab68aaa6f04e8a6a9f6552908fb3090ae2ff78d6ae0c597ac581954", size = 160871, upload-time = "2025-07-11T12:23:39.566Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/85/8567a966b85a2d3f971c4d42f781c305b2b91c043724fa08fd37d158e9dc/opentelemetry_sdk-1.36.0.tar.gz", hash = "sha256:19c8c81599f51b71670661ff7495c905d8fdf6976e41622d5245b791b06fa581", size = 162557, upload-time = "2025-07-29T15:12:16.76Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/01/4f/8e32b757ef3b660511b638ab52d1ed9259b666bdeeceba51a082ce3aea95/opentelemetry_sdk-1.35.0-py3-none-any.whl", hash = "sha256:223d9e5f5678518f4842311bb73966e0b6db5d1e0b74e35074c052cd2487f800", size = 119379, upload-time = "2025-07-11T12:23:24.521Z" }, + { url = "https://files.pythonhosted.org/packages/0b/59/7bed362ad1137ba5886dac8439e84cd2df6d087be7c09574ece47ae9b22c/opentelemetry_sdk-1.36.0-py3-none-any.whl", hash = "sha256:19fe048b42e98c5c1ffe85b569b7073576ad4ce0bcb6e9b4c6a39e890a6c45fb", size = 119995, upload-time = "2025-07-29T15:12:03.181Z" }, ] [[package]] name = "opentelemetry-semantic-conventions" -version = "0.56b0" +version = "0.57b0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-api" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/32/8e/214fa817f63b9f068519463d8ab46afd5d03b98930c39394a37ae3e741d0/opentelemetry_semantic_conventions-0.56b0.tar.gz", hash = "sha256:c114c2eacc8ff6d3908cb328c811eaf64e6d68623840be9224dc829c4fd6c2ea", size = 124221, upload-time = "2025-07-11T12:23:40.71Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/31/67dfa252ee88476a29200b0255bda8dfc2cf07b56ad66dc9a6221f7dc787/opentelemetry_semantic_conventions-0.57b0.tar.gz", hash = "sha256:609a4a79c7891b4620d64c7aac6898f872d790d75f22019913a660756f27ff32", size = 124225, upload-time = "2025-07-29T15:12:17.873Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/3f/e80c1b017066a9d999efffe88d1cce66116dcf5cb7f80c41040a83b6e03b/opentelemetry_semantic_conventions-0.56b0-py3-none-any.whl", hash = "sha256:df44492868fd6b482511cc43a942e7194be64e94945f572db24df2e279a001a2", size = 201625, upload-time = "2025-07-11T12:23:25.63Z" }, + { url = "https://files.pythonhosted.org/packages/05/75/7d591371c6c39c73de5ce5da5a2cc7b72d1d1cd3f8f4638f553c01c37b11/opentelemetry_semantic_conventions-0.57b0-py3-none-any.whl", hash = "sha256:757f7e76293294f124c827e514c2a3144f191ef175b069ce8d1211e1e38e9e78", size = 201627, upload-time = "2025-07-29T15:12:04.174Z" }, ] [[package]] @@ -2130,40 +3369,40 @@ wheels = [ [[package]] name = "orjson" -version = "3.11.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/29/87/03ababa86d984952304ac8ce9fbd3a317afb4a225b9a81f9b606ac60c873/orjson-3.11.0.tar.gz", hash = "sha256:2e4c129da624f291bcc607016a99e7f04a353f6874f3bd8d9b47b88597d5f700", size = 5318246, upload-time = "2025-07-15T16:08:29.194Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/2c/0b71a763f0f5130aa2631ef79e2cd84d361294665acccbb12b7a9813194e/orjson-3.11.0-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:1785df7ada75c18411ff7e20ac822af904a40161ea9dfe8c55b3f6b66939add6", size = 240007, upload-time = "2025-07-15T16:06:45.411Z" }, - { url = "https://files.pythonhosted.org/packages/f4/5a/f79ccd63d378b9c7c771d7a54c203d261b4c618fe3034ae95cd30f934f34/orjson-3.11.0-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:a57899bebbcea146616a2426d20b51b3562b4bc9f8039a3bd14fae361c23053d", size = 129320, upload-time = "2025-07-15T16:06:47.249Z" }, - { url = "https://files.pythonhosted.org/packages/7b/8a/63dafc147fa5ba945ad809c374b8f4ee692bb6b18aa6e161c3e6b69b594e/orjson-3.11.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b6fbc2fc825aff1456dd358c11a0ad7912a4cb4537d3db92e5334af7463a967", size = 132254, upload-time = "2025-07-15T16:06:48.597Z" }, - { url = "https://files.pythonhosted.org/packages/3c/11/4d1eb230483cc689a2f039c531bb2c980029c40ca5a9b5f64dce9786e955/orjson-3.11.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4305a638f4cf9bed3746ca3b7c242f14e05177d5baec2527026e0f9ee6c24fb7", size = 127003, upload-time = "2025-07-15T16:06:50.34Z" }, - { url = "https://files.pythonhosted.org/packages/4f/39/b6e96072946d908684e0f4b3de1639062fd5b32016b2929c035bd8e5c847/orjson-3.11.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1235fe7bbc37164f69302199d46f29cfb874018738714dccc5a5a44042c79c77", size = 128674, upload-time = "2025-07-15T16:06:51.659Z" }, - { url = "https://files.pythonhosted.org/packages/1e/dd/c77e3013f35b202ec2cc1f78a95fadf86b8c5a320d56eb1a0bbb965a87bb/orjson-3.11.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a640e3954e7b4fcb160097551e54cafbde9966be3991932155b71071077881aa", size = 131846, upload-time = "2025-07-15T16:06:53.359Z" }, - { url = "https://files.pythonhosted.org/packages/3f/7d/d83f0f96c2b142f9cdcf12df19052ea3767970989dc757598dc108db208f/orjson-3.11.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6d750b97d22d5566955e50b02c622f3a1d32744d7a578c878b29a873190ccb7a", size = 134016, upload-time = "2025-07-15T16:06:54.691Z" }, - { url = "https://files.pythonhosted.org/packages/67/4f/d22f79a3c56dde563c4fbc12eebf9224a1b87af5e4ec61beb11f9b3eb499/orjson-3.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4bfcfe498484161e011f8190a400591c52b026de96b3b3cbd3f21e8999b9dc0e", size = 127930, upload-time = "2025-07-15T16:06:56.001Z" }, - { url = "https://files.pythonhosted.org/packages/07/1e/26aede257db2163d974139fd4571f1e80f565216ccbd2c44ee1d43a63dcc/orjson-3.11.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:feaed3ed43a1d2df75c039798eb5ec92c350c7d86be53369bafc4f3700ce7df2", size = 130569, upload-time = "2025-07-15T16:06:57.275Z" }, - { url = "https://files.pythonhosted.org/packages/b4/bf/2cb57eac8d6054b555cba27203490489a7d3f5dca8c34382f22f2f0f17ba/orjson-3.11.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:aa1120607ec8fc98acf8c54aac6fb0b7b003ba883401fa2d261833111e2fa071", size = 403844, upload-time = "2025-07-15T16:06:59.107Z" }, - { url = "https://files.pythonhosted.org/packages/76/34/36e859ccfc45464df7b35c438c0ecc7751c930b3ebbefb50db7e3a641eb7/orjson-3.11.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c4b48d9775b0cf1f0aca734f4c6b272cbfacfac38e6a455e6520662f9434afb7", size = 144613, upload-time = "2025-07-15T16:07:00.48Z" }, - { url = "https://files.pythonhosted.org/packages/31/c5/5aeb84cdd0b44dc3972668944a1312f7983c2a45fb6b0e5e32b2f9408540/orjson-3.11.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f018ed1986d79434ac712ff19f951cd00b4dfcb767444410fbb834ebec160abf", size = 132419, upload-time = "2025-07-15T16:07:01.927Z" }, - { url = "https://files.pythonhosted.org/packages/59/0c/95ee1e61a067ad24c4921609156b3beeca8b102f6f36dca62b08e1a7c7a8/orjson-3.11.0-cp311-cp311-win32.whl", hash = "sha256:08e191f8a55ac2c00be48e98a5d10dca004cbe8abe73392c55951bfda60fc123", size = 134620, upload-time = "2025-07-15T16:07:03.304Z" }, - { url = "https://files.pythonhosted.org/packages/94/3e/afd5e284db9387023803553061ea05c785c36fe7845e4fe25912424b343f/orjson-3.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:b5a4214ea59c8a3b56f8d484b28114af74e9fba0956f9be5c3ce388ae143bf1f", size = 129333, upload-time = "2025-07-15T16:07:04.973Z" }, - { url = "https://files.pythonhosted.org/packages/8b/a4/d29e9995d73f23f2444b4db299a99477a4f7e6f5bf8923b775ef43a4e660/orjson-3.11.0-cp311-cp311-win_arm64.whl", hash = "sha256:57e8e7198a679ab21241ab3f355a7990c7447559e35940595e628c107ef23736", size = 126656, upload-time = "2025-07-15T16:07:06.288Z" }, - { url = "https://files.pythonhosted.org/packages/92/c9/241e304fb1e58ea70b720f1a9e5349c6bb7735ffac401ef1b94f422edd6d/orjson-3.11.0-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b4089f940c638bb1947d54e46c1cd58f4259072fcc97bc833ea9c78903150ac9", size = 240269, upload-time = "2025-07-15T16:07:08.173Z" }, - { url = "https://files.pythonhosted.org/packages/26/7c/289457cdf40be992b43f1d90ae213ebc03a31a8e2850271ecd79e79a3135/orjson-3.11.0-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:8335a0ba1c26359fb5c82d643b4c1abbee2bc62875e0f2b5bde6c8e9e25eb68c", size = 129276, upload-time = "2025-07-15T16:07:10.128Z" }, - { url = "https://files.pythonhosted.org/packages/66/de/5c0528d46ded965939b6b7f75b1fe93af42b9906b0039096fc92c9001c12/orjson-3.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63c1c9772dafc811d16d6a7efa3369a739da15d1720d6e58ebe7562f54d6f4a2", size = 131966, upload-time = "2025-07-15T16:07:11.509Z" }, - { url = "https://files.pythonhosted.org/packages/ad/74/39822f267b5935fb6fc961ccc443f4968a74d34fc9270b83caa44e37d907/orjson-3.11.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9457ccbd8b241fb4ba516417a4c5b95ba0059df4ac801309bcb4ec3870f45ad9", size = 127028, upload-time = "2025-07-15T16:07:13.023Z" }, - { url = "https://files.pythonhosted.org/packages/7c/e3/28f6ed7f03db69bddb3ef48621b2b05b394125188f5909ee0a43fcf4820e/orjson-3.11.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0846e13abe79daece94a00b92574f294acad1d362be766c04245b9b4dd0e47e1", size = 129105, upload-time = "2025-07-15T16:07:14.367Z" }, - { url = "https://files.pythonhosted.org/packages/cb/50/8867fd2fc92c0ab1c3e14673ec5d9d0191202e4ab8ba6256d7a1d6943ad3/orjson-3.11.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5587c85ae02f608a3f377b6af9eb04829606f518257cbffa8f5081c1aacf2e2f", size = 131902, upload-time = "2025-07-15T16:07:16.176Z" }, - { url = "https://files.pythonhosted.org/packages/13/65/c189deea10342afee08006331082ff67d11b98c2394989998b3ea060354a/orjson-3.11.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c7a1964a71c1567b4570c932a0084ac24ad52c8cf6253d1881400936565ed438", size = 134042, upload-time = "2025-07-15T16:07:17.937Z" }, - { url = "https://files.pythonhosted.org/packages/2b/e4/cf23c3f4231d2a9a043940ab045f799f84a6df1b4fb6c9b4412cdc3ebf8c/orjson-3.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5a8243e73690cc6e9151c9e1dd046a8f21778d775f7d478fa1eb4daa4897c61", size = 128260, upload-time = "2025-07-15T16:07:19.651Z" }, - { url = "https://files.pythonhosted.org/packages/de/b9/2cb94d3a67edb918d19bad4a831af99cd96c3657a23daa239611bcf335d7/orjson-3.11.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:51646f6d995df37b6e1b628f092f41c0feccf1d47e3452c6e95e2474b547d842", size = 130282, upload-time = "2025-07-15T16:07:21.022Z" }, - { url = "https://files.pythonhosted.org/packages/0b/96/df963cc973e689d4c56398647917b4ee95f47e5b6d2779338c09c015b23b/orjson-3.11.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:2fb8ca8f0b4e31b8aaec674c7540649b64ef02809410506a44dc68d31bd5647b", size = 403765, upload-time = "2025-07-15T16:07:25.469Z" }, - { url = "https://files.pythonhosted.org/packages/fb/92/71429ee1badb69f53281602dbb270fa84fc2e51c83193a814d0208bb63b0/orjson-3.11.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:64a6a3e94a44856c3f6557e6aa56a6686544fed9816ae0afa8df9077f5759791", size = 144779, upload-time = "2025-07-15T16:07:27.339Z" }, - { url = "https://files.pythonhosted.org/packages/c8/ab/3678b2e5ff0c622a974cb8664ed7cdda5ed26ae2b9d71ba66ec36f32d6cf/orjson-3.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d69f95d484938d8fab5963e09131bcf9fbbb81fa4ec132e316eb2fb9adb8ce78", size = 132797, upload-time = "2025-07-15T16:07:28.717Z" }, - { url = "https://files.pythonhosted.org/packages/9d/8c/74509f715ff189d2aca90ebb0bd5af6658e0f9aa2512abbe6feca4c78208/orjson-3.11.0-cp312-cp312-win32.whl", hash = "sha256:8514f9f9c667ce7d7ef709ab1a73e7fcab78c297270e90b1963df7126d2b0e23", size = 134695, upload-time = "2025-07-15T16:07:30.034Z" }, - { url = "https://files.pythonhosted.org/packages/82/ba/ef25e3e223f452a01eac6a5b38d05c152d037508dcbf87ad2858cbb7d82e/orjson-3.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:41b38a894520b8cb5344a35ffafdf6ae8042f56d16771b2c5eb107798cee85ee", size = 129446, upload-time = "2025-07-15T16:07:31.412Z" }, - { url = "https://files.pythonhosted.org/packages/e3/cd/6f4d93867c5d81bb4ab2d4ac870d3d6e9ba34fa580a03b8d04bf1ce1d8ad/orjson-3.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:5579acd235dd134467340b2f8a670c1c36023b5a69c6a3174c4792af7502bd92", size = 126400, upload-time = "2025-07-15T16:07:34.143Z" }, +version = "3.11.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/1d/5e0ae38788bdf0721326695e65fdf41405ed535f633eb0df0f06f57552fa/orjson-3.11.2.tar.gz", hash = "sha256:91bdcf5e69a8fd8e8bdb3de32b31ff01d2bd60c1e8d5fe7d5afabdcf19920309", size = 5470739, upload-time = "2025-08-12T15:12:28.626Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/7d/e295df1ac9920cbb19fb4c1afa800e86f175cb657143aa422337270a4782/orjson-3.11.2-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:888b64ef7eaeeff63f773881929434a5834a6a140a63ad45183d59287f07fc6a", size = 226502, upload-time = "2025-08-12T15:10:42.284Z" }, + { url = "https://files.pythonhosted.org/packages/65/21/ffb0f10ea04caf418fb4e7ad1fda4b9ab3179df9d7a33b69420f191aadd5/orjson-3.11.2-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:83387cc8b26c9fa0ae34d1ea8861a7ae6cff8fb3e346ab53e987d085315a728e", size = 115999, upload-time = "2025-08-12T15:10:43.738Z" }, + { url = "https://files.pythonhosted.org/packages/90/d5/8da1e252ac3353d92e6f754ee0c85027c8a2cda90b6899da2be0df3ef83d/orjson-3.11.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7e35f003692c216d7ee901b6b916b5734d6fc4180fcaa44c52081f974c08e17", size = 111563, upload-time = "2025-08-12T15:10:45.301Z" }, + { url = "https://files.pythonhosted.org/packages/4f/81/baabc32e52c570b0e4e1044b1bd2ccbec965e0de3ba2c13082255efa2006/orjson-3.11.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4a0a4c29ae90b11d0c00bcc31533854d89f77bde2649ec602f512a7e16e00640", size = 116222, upload-time = "2025-08-12T15:10:46.92Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b7/da2ad55ad80b49b560dce894c961477d0e76811ee6e614b301de9f2f8728/orjson-3.11.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:585d712b1880f68370108bc5534a257b561672d1592fae54938738fe7f6f1e33", size = 118594, upload-time = "2025-08-12T15:10:48.488Z" }, + { url = "https://files.pythonhosted.org/packages/61/be/014f7eab51449f3c894aa9bbda2707b5340c85650cb7d0db4ec9ae280501/orjson-3.11.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d08e342a7143f8a7c11f1c4033efe81acbd3c98c68ba1b26b96080396019701f", size = 120700, upload-time = "2025-08-12T15:10:49.811Z" }, + { url = "https://files.pythonhosted.org/packages/cf/ae/c217903a30c51341868e2d8c318c59a8413baa35af54d7845071c8ccd6fe/orjson-3.11.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:29c0f84fc50398773a702732c87cd622737bf11c0721e6db3041ac7802a686fb", size = 123433, upload-time = "2025-08-12T15:10:51.06Z" }, + { url = "https://files.pythonhosted.org/packages/57/c2/b3c346f78b1ff2da310dd300cb0f5d32167f872b4d3bb1ad122c889d97b0/orjson-3.11.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:140f84e3c8d4c142575898c91e3981000afebf0333df753a90b3435d349a5fe5", size = 121061, upload-time = "2025-08-12T15:10:52.381Z" }, + { url = "https://files.pythonhosted.org/packages/00/c8/c97798f6010327ffc75ad21dd6bca11ea2067d1910777e798c2849f1c68f/orjson-3.11.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:96304a2b7235e0f3f2d9363ddccdbfb027d27338722fe469fe656832a017602e", size = 119410, upload-time = "2025-08-12T15:10:53.692Z" }, + { url = "https://files.pythonhosted.org/packages/37/fd/df720f7c0e35694617b7f95598b11a2cb0374661d8389703bea17217da53/orjson-3.11.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d7612bb227d5d9582f1f50a60bd55c64618fc22c4a32825d233a4f2771a428a", size = 392294, upload-time = "2025-08-12T15:10:55.079Z" }, + { url = "https://files.pythonhosted.org/packages/ba/52/0120d18f60ab0fe47531d520372b528a45c9a25dcab500f450374421881c/orjson-3.11.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:a134587d18fe493befc2defffef2a8d27cfcada5696cb7234de54a21903ae89a", size = 134134, upload-time = "2025-08-12T15:10:56.568Z" }, + { url = "https://files.pythonhosted.org/packages/ec/10/1f967671966598366de42f07e92b0fc694ffc66eafa4b74131aeca84915f/orjson-3.11.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0b84455e60c4bc12c1e4cbaa5cfc1acdc7775a9da9cec040e17232f4b05458bd", size = 123745, upload-time = "2025-08-12T15:10:57.907Z" }, + { url = "https://files.pythonhosted.org/packages/43/eb/76081238671461cfd0f47e0c24f408ffa66184237d56ef18c33e86abb612/orjson-3.11.2-cp311-cp311-win32.whl", hash = "sha256:f0660efeac223f0731a70884e6914a5f04d613b5ae500744c43f7bf7b78f00f9", size = 124393, upload-time = "2025-08-12T15:10:59.267Z" }, + { url = "https://files.pythonhosted.org/packages/26/76/cc598c1811ba9ba935171267b02e377fc9177489efce525d478a2999d9cc/orjson-3.11.2-cp311-cp311-win_amd64.whl", hash = "sha256:955811c8405251d9e09cbe8606ad8fdef49a451bcf5520095a5ed38c669223d8", size = 119561, upload-time = "2025-08-12T15:11:00.559Z" }, + { url = "https://files.pythonhosted.org/packages/d8/17/c48011750f0489006f7617b0a3cebc8230f36d11a34e7e9aca2085f07792/orjson-3.11.2-cp311-cp311-win_arm64.whl", hash = "sha256:2e4d423a6f838552e3a6d9ec734b729f61f88b1124fd697eab82805ea1a2a97d", size = 114186, upload-time = "2025-08-12T15:11:01.931Z" }, + { url = "https://files.pythonhosted.org/packages/40/02/46054ebe7996a8adee9640dcad7d39d76c2000dc0377efa38e55dc5cbf78/orjson-3.11.2-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:901d80d349d8452162b3aa1afb82cec5bee79a10550660bc21311cc61a4c5486", size = 226528, upload-time = "2025-08-12T15:11:03.317Z" }, + { url = "https://files.pythonhosted.org/packages/e2/c6/6b6f0b4d8aea1137436546b990f71be2cd8bd870aa2f5aa14dba0fcc95dc/orjson-3.11.2-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:cf3bd3967a360e87ee14ed82cb258b7f18c710dacf3822fb0042a14313a673a1", size = 115931, upload-time = "2025-08-12T15:11:04.759Z" }, + { url = "https://files.pythonhosted.org/packages/ae/05/4205cc97c30e82a293dd0d149b1a89b138ebe76afeca66fc129fa2aa4e6a/orjson-3.11.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26693dde66910078229a943e80eeb99fdce6cd2c26277dc80ead9f3ab97d2131", size = 111382, upload-time = "2025-08-12T15:11:06.468Z" }, + { url = "https://files.pythonhosted.org/packages/50/c7/b8a951a93caa821f9272a7c917115d825ae2e4e8768f5ddf37968ec9de01/orjson-3.11.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4ad4c8acb50a28211c33fc7ef85ddf5cb18d4636a5205fd3fa2dce0411a0e30c", size = 116271, upload-time = "2025-08-12T15:11:07.845Z" }, + { url = "https://files.pythonhosted.org/packages/17/03/1006c7f8782d5327439e26d9b0ec66500ea7b679d4bbb6b891d2834ab3ee/orjson-3.11.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:994181e7f1725bb5f2d481d7d228738e0743b16bf319ca85c29369c65913df14", size = 119086, upload-time = "2025-08-12T15:11:09.329Z" }, + { url = "https://files.pythonhosted.org/packages/44/61/57d22bc31f36a93878a6f772aea76b2184102c6993dea897656a66d18c74/orjson-3.11.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dbb79a0476393c07656b69c8e763c3cc925fa8e1d9e9b7d1f626901bb5025448", size = 120724, upload-time = "2025-08-12T15:11:10.674Z" }, + { url = "https://files.pythonhosted.org/packages/78/a9/4550e96b4c490c83aea697d5347b8f7eb188152cd7b5a38001055ca5b379/orjson-3.11.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:191ed27a1dddb305083d8716af413d7219f40ec1d4c9b0e977453b4db0d6fb6c", size = 123577, upload-time = "2025-08-12T15:11:12.015Z" }, + { url = "https://files.pythonhosted.org/packages/3a/86/09b8cb3ebd513d708ef0c92d36ac3eebda814c65c72137b0a82d6d688fc4/orjson-3.11.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0afb89f16f07220183fd00f5f297328ed0a68d8722ad1b0c8dcd95b12bc82804", size = 121195, upload-time = "2025-08-12T15:11:13.399Z" }, + { url = "https://files.pythonhosted.org/packages/37/68/7b40b39ac2c1c644d4644e706d0de6c9999764341cd85f2a9393cb387661/orjson-3.11.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6ab6e6b4e93b1573a026b6ec16fca9541354dd58e514b62c558b58554ae04307", size = 119234, upload-time = "2025-08-12T15:11:15.134Z" }, + { url = "https://files.pythonhosted.org/packages/40/7c/bb6e7267cd80c19023d44d8cbc4ea4ed5429fcd4a7eb9950f50305697a28/orjson-3.11.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:9cb23527efb61fb75527df55d20ee47989c4ee34e01a9c98ee9ede232abf6219", size = 392250, upload-time = "2025-08-12T15:11:16.604Z" }, + { url = "https://files.pythonhosted.org/packages/64/f2/6730ace05583dbca7c1b406d59f4266e48cd0d360566e71482420fb849fc/orjson-3.11.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a4dd1268e4035af21b8a09e4adf2e61f87ee7bf63b86d7bb0a237ac03fad5b45", size = 134572, upload-time = "2025-08-12T15:11:18.205Z" }, + { url = "https://files.pythonhosted.org/packages/96/0f/7d3e03a30d5aac0432882b539a65b8c02cb6dd4221ddb893babf09c424cc/orjson-3.11.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ff8b155b145eaf5a9d94d2c476fbe18d6021de93cf36c2ae2c8c5b775763f14e", size = 123869, upload-time = "2025-08-12T15:11:19.554Z" }, + { url = "https://files.pythonhosted.org/packages/45/80/1513265eba6d4a960f078f4b1d2bff94a571ab2d28c6f9835e03dfc65cc6/orjson-3.11.2-cp312-cp312-win32.whl", hash = "sha256:ae3bb10279d57872f9aba68c9931aa71ed3b295fa880f25e68da79e79453f46e", size = 124430, upload-time = "2025-08-12T15:11:20.914Z" }, + { url = "https://files.pythonhosted.org/packages/fb/61/eadf057b68a332351eeb3d89a4cc538d14f31cd8b5ec1b31a280426ccca2/orjson-3.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:d026e1967239ec11a2559b4146a61d13914504b396f74510a1c4d6b19dfd8732", size = 119598, upload-time = "2025-08-12T15:11:22.372Z" }, + { url = "https://files.pythonhosted.org/packages/6b/3f/7f4b783402143d965ab7e9a2fc116fdb887fe53bdce7d3523271cd106098/orjson-3.11.2-cp312-cp312-win_arm64.whl", hash = "sha256:59f8d5ad08602711af9589375be98477d70e1d102645430b5a7985fdbf613b36", size = 114052, upload-time = "2025-08-12T15:11:23.762Z" }, ] [[package]] @@ -2190,6 +3429,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b0/0d/73143ecb94ac4a5dcba223402139240a75dee0cc6ba8a543788a5646407a/ormsgpack-1.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:35fa9f81e5b9a0dab42e09a73f7339ecffdb978d6dbf9deb2ecf1e9fc7808722", size = 121401, upload-time = "2025-05-24T19:07:28.308Z" }, ] +[[package]] +name = "packageurl-python" +version = "0.17.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/d6/3b5a4e3cfaef7a53869a26ceb034d1ff5e5c27c814ce77260a96d50ab7bb/packageurl_python-0.17.6.tar.gz", hash = "sha256:1252ce3a102372ca6f86eb968e16f9014c4ba511c5c37d95a7f023e2ca6e5c25", size = 50618, upload-time = "2025-11-24T15:20:17.998Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/2f/c7277b7615a93f51b5fbc1eacfc1b75e8103370e786fd8ce2abf6e5c04ab/packageurl_python-0.17.6-py3-none-any.whl", hash = "sha256:31a85c2717bc41dd818f3c62908685ff9eebcb68588213745b14a6ee9e7df7c9", size = 36776, upload-time = "2025-11-24T15:20:16.962Z" }, +] + [[package]] name = "packaging" version = "25.0" @@ -2227,6 +3475,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/80/a5/3a92893e7399a691bad7664d977cb5e7c81cf666c81f89ea76ba2bff483d/pandas-2.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:ac942bfd0aca577bef61f2bc8da8147c4ef6879965ef883d8e8d5d2dc3e744b8", size = 10987601, upload-time = "2025-07-07T19:19:09.589Z" }, ] +[[package]] +name = "parso" +version = "0.8.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/94/68e2e17afaa9169cf6412ab0f28623903be73d1b32e208d9e8e541bb086d/parso-0.8.4.tar.gz", hash = "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d", size = 400609, upload-time = "2024-04-05T09:43:55.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/ac/dac4a63f978e4dcb3c6d3a78c4d8e0192a113d288502a1216950c41b1027/parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18", size = 103650, upload-time = "2024-04-05T09:43:53.299Z" }, +] + +[[package]] +name = "pdfminer-six" +version = "20240706" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "charset-normalizer" }, + { name = "cryptography" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e3/37/63cb918ffa21412dd5d54e32e190e69bfc340f3d6aa072ad740bec9386bb/pdfminer.six-20240706.tar.gz", hash = "sha256:c631a46d5da957a9ffe4460c5dce21e8431dabb615fee5f9f4400603a58d95a6", size = 7363505, upload-time = "2024-07-06T13:48:50.795Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/7d/44d6b90e5a293d3a975cefdc4e12a932ebba814995b2a07e37e599dd27c6/pdfminer.six-20240706-py3-none-any.whl", hash = "sha256:f4f70e74174b4b3542fcb8406a210b6e2e27cd0f0b5fd04534a8cc0d8951e38c", size = 5615414, upload-time = "2024-07-06T13:48:48.408Z" }, +] + +[[package]] +name = "pexpect" +version = "4.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ptyprocess" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" }, +] + [[package]] name = "pgpy13" version = "0.6.1rc1" @@ -2240,6 +3522,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4f/de/8e43636f1244989fb733513fa7c1252a3f82feda4ee861ec1b6377f2e6e7/pgpy13-0.6.1rc1-py3-none-any.whl", hash = "sha256:c8d6cf3dea0cc23373ff1c4ed79a6a4bf39216bf54970bae7cca656003d1aa4f", size = 84589, upload-time = "2025-03-01T22:27:17.242Z" }, ] +[[package]] +name = "phonenumbers" +version = "9.0.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9f/07/f76fb718ad78d0cd0926ef750b58928e5b13f0f94cc6c516d310f11228fb/phonenumbers-9.0.12.tar.gz", hash = "sha256:ccadff6b949494bd606836d8c9678bee5b55cb1cbad1e98bf7adae108e6fd0be", size = 2297813, upload-time = "2025-08-15T11:29:03.489Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/f1/e94b3504078d67d69d024ffb1798c31df9ddfb09ec2ec1680f3e7776414a/phonenumbers-9.0.12-py2.py3-none-any.whl", hash = "sha256:900633afc3e12191458d710262df5efc117838bd1e2e613b64fa254a86bb20a1", size = 2583620, upload-time = "2025-08-15T11:28:59.161Z" }, +] + [[package]] name = "pillow" version = "11.3.0" @@ -2279,11 +3570,20 @@ wheels = [ [[package]] name = "pip" -version = "25.1.1" +version = "25.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/16/650289cd3f43d5a2fadfd98c68bd1e1e7f2550a1a5326768cddfbcedb2c5/pip-25.2.tar.gz", hash = "sha256:578283f006390f85bb6282dffb876454593d637f5d1be494b5202ce4877e71f2", size = 1840021, upload-time = "2025-07-30T21:50:15.401Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/3f/945ef7ab14dc4f9d7f40288d2df998d1837ee0888ec3659c813487572faa/pip-25.2-py3-none-any.whl", hash = "sha256:6d67a2b4e7f14d8b31b8b52648866fa717f45a1eb70e83002f4331d07e953717", size = 1752557, upload-time = "2025-07-30T21:50:13.323Z" }, +] + +[[package]] +name = "pkce" +version = "1.0.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/59/de/241caa0ca606f2ec5fe0c1f4261b0465df78d786a38da693864a116c37f4/pip-25.1.1.tar.gz", hash = "sha256:3de45d411d308d5054c2168185d8da7f9a2cd753dbac8acbfa88a8909ecd9077", size = 1940155, upload-time = "2025-05-02T15:14:02.057Z" } +sdist = { url = "https://files.pythonhosted.org/packages/29/ea/ddd845c2ec21bf1e8555c782b32dc39b82f0b12764feb9f73ccbb2470f13/pkce-1.0.3.tar.gz", hash = "sha256:9775fd76d8a743d39b87df38af1cd04a58c9b5a5242d5a6350ef343d06814ab6", size = 2757, upload-time = "2021-02-08T18:29:07.07Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/29/a2/d40fb2460e883eca5199c62cfc2463fd261f760556ae6290f88488c362c0/pip-25.1.1-py3-none-any.whl", hash = "sha256:2913a38a2abf4ea6b64ab507bd9e967f3b53dc1ede74b01b0931e1ce548751af", size = 1825227, upload-time = "2025-05-02T15:13:59.102Z" }, + { url = "https://files.pythonhosted.org/packages/15/51/52c22ec0812d25f5bf297a01153604bfa7bfa59ed66f6cd8345beb3c2b2a/pkce-1.0.3-py3-none-any.whl", hash = "sha256:55927e24c7d403b2491ebe182b95d9dcb1807643243d47e3879fbda5aad4471d", size = 3200, upload-time = "2021-02-08T18:29:05.678Z" }, ] [[package]] @@ -2322,6 +3622,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/58/35da89ee790598a0700ea49b2a66594140f44dec458c07e8e3d4979137fc/ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce", size = 49567, upload-time = "2018-02-15T19:01:27.172Z" }, ] +[[package]] +name = "polyfile-weave" +version = "0.5.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "abnf" }, + { name = "chardet" }, + { name = "cint" }, + { name = "fickling" }, + { name = "graphviz" }, + { name = "intervaltree" }, + { name = "jinja2" }, + { name = "kaitaistruct" }, + { name = "networkx" }, + { name = "pdfminer-six" }, + { name = "pillow" }, + { name = "pyreadline3", marker = "sys_platform == 'win32'" }, + { name = "pyyaml" }, + { name = "setuptools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/16/11/7e0b3908a4f5436197b1fc11713c628cd7f9136dc7c1fb00ac8879991f87/polyfile_weave-0.5.6.tar.gz", hash = "sha256:a9fc41b456272c95a3788a2cab791e052acc24890c512fc5a6f9f4e221d24ed1", size = 5987173, upload-time = "2025-07-28T20:26:32.092Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/63/04c5c7c2093cf69c9eeea338f4757522a5d048703a35b3ac8a5580ed2369/polyfile_weave-0.5.6-py3-none-any.whl", hash = "sha256:658e5b6ed040a973279a0cd7f54f4566249c85b977dee556788fa6f903c1d30b", size = 1655007, upload-time = "2025-07-28T20:26:30.132Z" }, +] + [[package]] name = "prefixspan" version = "0.5.2" @@ -2332,6 +3657,71 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/c5/b0/e66e9f6e07a0b37aa0f5703c46f54bafbdf65dfba63994247676b19076c4/prefixspan-0.5.2.tar.gz", hash = "sha256:8fbd2f94b3a7f4399d04f9bd6aa214b830fb7828799c472cd43dda10b03f671c", size = 10404, upload-time = "2018-09-29T06:12:27.708Z" } +[[package]] +name = "preshed" +version = "3.0.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cymem" }, + { name = "murmurhash" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/3a/db814f67a05b6d7f9c15d38edef5ec9b21415710705b393883de92aee5ef/preshed-3.0.10.tar.gz", hash = "sha256:5a5c8e685e941f4ffec97f1fbf32694b8107858891a4bc34107fac981d8296ff", size = 15039, upload-time = "2025-05-26T15:18:33.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/99/c3709638f687da339504d1daeca48604cadb338bf3556a1484d1f0cd95e6/preshed-3.0.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d96c4fe2b41c1cdcc8c4fc1fdb10f922a6095c0430a3ebe361fe62c78902d068", size = 131486, upload-time = "2025-05-26T15:17:52.231Z" }, + { url = "https://files.pythonhosted.org/packages/e0/27/0fd36b63caa8bbf57b31a121d9565d385bbd7521771d4eb93e17d326873d/preshed-3.0.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cb01ea930b96f3301526a2ab26f41347d07555e4378c4144c6b7645074f2ebb0", size = 127938, upload-time = "2025-05-26T15:17:54.19Z" }, + { url = "https://files.pythonhosted.org/packages/90/54/6a876d9cc8d401a9c1fb6bb8ca5a31b3664d0bcb888a9016258a1ae17344/preshed-3.0.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dd1f0a7b7d150e229d073fd4fe94f72610cae992e907cee74687c4695873a98", size = 842263, upload-time = "2025-05-26T15:17:55.398Z" }, + { url = "https://files.pythonhosted.org/packages/1c/7d/ff19f74d15ee587905bafa3582883cfe2f72b574e6d691ee64dc690dc276/preshed-3.0.10-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fd7b350c280137f324cd447afbf6ba9a849af0e8898850046ac6f34010e08bd", size = 842913, upload-time = "2025-05-26T15:17:56.687Z" }, + { url = "https://files.pythonhosted.org/packages/f1/3a/1c345a26463345557705b61965e1e0a732cc0e9c6dfd4787845dbfa50b4a/preshed-3.0.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cf6a5fdc89ad06079aa6ee63621e417d4f4cf2a3d8b63c72728baad35a9ff641", size = 820548, upload-time = "2025-05-26T15:17:58.057Z" }, + { url = "https://files.pythonhosted.org/packages/7f/6b/71f25e2b7a23dba168f43edfae0bb508552dbef89114ce65c73f2ea7172f/preshed-3.0.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b4c29a7bd66985808ad181c9ad05205a6aa7400cd0f98426acd7bc86588b93f8", size = 840379, upload-time = "2025-05-26T15:17:59.565Z" }, + { url = "https://files.pythonhosted.org/packages/3a/86/d8f32b0b31a36ee8770a9b1a95321430e364cd0ba4bfebb7348aed2f198d/preshed-3.0.10-cp311-cp311-win_amd64.whl", hash = "sha256:1367c1fd6f44296305315d4e1c3fe3171787d4d01c1008a76bc9466bd79c3249", size = 117655, upload-time = "2025-05-26T15:18:00.836Z" }, + { url = "https://files.pythonhosted.org/packages/c3/14/322a4f58bc25991a87f216acb1351800739b0794185d27508ee86c35f382/preshed-3.0.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6e9c46933d55c8898c8f7a6019a8062cd87ef257b075ada2dd5d1e57810189ea", size = 131367, upload-time = "2025-05-26T15:18:02.408Z" }, + { url = "https://files.pythonhosted.org/packages/38/80/67507653c35620cace913f617df6d6f658b87e8da83087b851557d65dd86/preshed-3.0.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5c4ebc4f8ef0114d55f2ffdce4965378129c7453d0203664aeeb03055572d9e4", size = 126535, upload-time = "2025-05-26T15:18:03.589Z" }, + { url = "https://files.pythonhosted.org/packages/db/b1/ab4f811aeaf20af0fa47148c1c54b62d7e8120d59025bd0a3f773bb67725/preshed-3.0.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ab5ab4c6dfd3746fb4328e7fbeb2a0544416b872db02903bfac18e6f5cd412f", size = 864907, upload-time = "2025-05-26T15:18:04.794Z" }, + { url = "https://files.pythonhosted.org/packages/fb/db/fe37c1f99cfb26805dd89381ddd54901307feceb267332eaaca228e9f9c1/preshed-3.0.10-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40586fd96ae3974c552a7cd78781b6844ecb1559ee7556586f487058cf13dd96", size = 869329, upload-time = "2025-05-26T15:18:06.353Z" }, + { url = "https://files.pythonhosted.org/packages/a7/fd/efb6a6233d1cd969966f3f65bdd8e662579c3d83114e5c356cec1927b1f7/preshed-3.0.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a606c24cda931306b98e0edfafed3309bffcf8d6ecfe07804db26024c4f03cd6", size = 846829, upload-time = "2025-05-26T15:18:07.716Z" }, + { url = "https://files.pythonhosted.org/packages/14/49/0e4ce5db3bf86b081abb08a404fb37b7c2dbfd7a73ec6c0bc71b650307eb/preshed-3.0.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:394015566f9354738be903447039e8dbc6d93ba5adf091af694eb03c4e726b1e", size = 874008, upload-time = "2025-05-26T15:18:09.364Z" }, + { url = "https://files.pythonhosted.org/packages/6f/17/76d6593fc2d055d4e413b68a8c87b70aa9b7697d4972cb8062559edcf6e9/preshed-3.0.10-cp312-cp312-win_amd64.whl", hash = "sha256:fd7e38225937e580420c84d1996dde9b4f726aacd9405093455c3a2fa60fede5", size = 116701, upload-time = "2025-05-26T15:18:11.905Z" }, +] + +[[package]] +name = "presidio-analyzer" +version = "2.2.359" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "phonenumbers" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "spacy" }, + { name = "tldextract" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/09/276238e475262a184e4bb9e0c37394442b19d486a2ecd3664d51a91a487b/presidio_analyzer-2.2.359-py3-none-any.whl", hash = "sha256:5f9a71ce5e484b1d9fd10a3f40ba37cb311deeb7cc25c3a87c0ba36b468ee26d", size = 116505, upload-time = "2025-07-13T07:22:35.248Z" }, +] + +[[package]] +name = "presidio-anonymizer" +version = "2.2.357" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "azure-core" }, + { name = "pycryptodome" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/5a/74d4f11e7b111c7570235c2ce4dac923f30c7c85fcc5d8523c8c146b9cdf/presidio_anonymizer-2.2.357-py3-none-any.whl", hash = "sha256:0b3e5e0526f5950bb9b27941e5b1b01b6761295d178a8ba4cedd2771aa2aee52", size = 31209, upload-time = "2025-01-13T13:01:43.477Z" }, +] + +[[package]] +name = "prompt-toolkit" +version = "3.0.51" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/6e/9d084c929dfe9e3bfe0c6a47e31f78a25c54627d64a66e884a8bf5474f1c/prompt_toolkit-3.0.51.tar.gz", hash = "sha256:931a162e3b27fc90c86f1b48bb1fb2c528c2761475e57c9c06de13311c7b54ed", size = 428940, upload-time = "2025-04-15T09:18:47.731Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/4f/5249960887b1fbe561d9ff265496d170b55a735b76724f10ef19f9e40716/prompt_toolkit-3.0.51-py3-none-any.whl", hash = "sha256:52742911fde84e2d423e2f9a4cf1de7d7ac4e51958f648d9540e0fb8db077b07", size = 387810, upload-time = "2025-04-15T09:18:44.753Z" }, +] + [[package]] name = "propcache" version = "0.3.2" @@ -2389,43 +3779,66 @@ wheels = [ [[package]] name = "psutil" -version = "7.0.0" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/c7/8c6872f7372eb6a6b2e4708b88419fb46b857f7a2e1892966b851cc79fc9/psutil-6.0.0.tar.gz", hash = "sha256:8faae4f310b6d969fa26ca0545338b21f73c6b15db7c4a8d934a5482faa818f2", size = 508067, upload-time = "2024-06-18T21:40:10.559Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/37/f8da2fbd29690b3557cca414c1949f92162981920699cd62095a984983bf/psutil-6.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c588a7e9b1173b6e866756dde596fd4cad94f9399daf99ad8c3258b3cb2b47a0", size = 250961, upload-time = "2024-06-18T21:41:11.662Z" }, + { url = "https://files.pythonhosted.org/packages/35/56/72f86175e81c656a01c4401cd3b1c923f891b31fbcebe98985894176d7c9/psutil-6.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ed2440ada7ef7d0d608f20ad89a04ec47d2d3ab7190896cd62ca5fc4fe08bf0", size = 287478, upload-time = "2024-06-18T21:41:16.18Z" }, + { url = "https://files.pythonhosted.org/packages/19/74/f59e7e0d392bc1070e9a70e2f9190d652487ac115bb16e2eff6b22ad1d24/psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fd9a97c8e94059b0ef54a7d4baf13b405011176c3b6ff257c247cae0d560ecd", size = 290455, upload-time = "2024-06-18T21:41:29.048Z" }, + { url = "https://files.pythonhosted.org/packages/cd/5f/60038e277ff0a9cc8f0c9ea3d0c5eb6ee1d2470ea3f9389d776432888e47/psutil-6.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2e8d0054fc88153ca0544f5c4d554d42e33df2e009c4ff42284ac9ebdef4132", size = 292046, upload-time = "2024-06-18T21:41:33.53Z" }, + { url = "https://files.pythonhosted.org/packages/8b/20/2ff69ad9c35c3df1858ac4e094f20bd2374d33c8643cf41da8fd7cdcb78b/psutil-6.0.0-cp37-abi3-win32.whl", hash = "sha256:a495580d6bae27291324fe60cea0b5a7c23fa36a7cd35035a16d93bdcf076b9d", size = 253560, upload-time = "2024-06-18T21:41:46.067Z" }, + { url = "https://files.pythonhosted.org/packages/73/44/561092313ae925f3acfaace6f9ddc4f6a9c748704317bad9c8c8f8a36a79/psutil-6.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:33ea5e1c975250a720b3a6609c490db40dae5d83a4eb315170c4fe0d8b1f34b3", size = 257399, upload-time = "2024-06-18T21:41:52.1Z" }, + { url = "https://files.pythonhosted.org/packages/7c/06/63872a64c312a24fb9b4af123ee7007a306617da63ff13bcc1432386ead7/psutil-6.0.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:ffe7fc9b6b36beadc8c322f84e1caff51e8703b88eee1da46d1e3a6ae11b4fd0", size = 251988, upload-time = "2024-06-18T21:41:57.337Z" }, +] + +[[package]] +name = "ptyprocess" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/e5/16ff212c1e452235a90aeb09066144d0c5a6a8c0834397e03f5224495c4e/ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220", size = 70762, upload-time = "2020-12-28T15:15:30.155Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", size = 13993, upload-time = "2020-12-28T15:15:28.35Z" }, +] + +[[package]] +name = "pure-eval" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/05/0a34433a064256a578f1783a10da6df098ceaa4a57bbeaa96a6c0352786b/pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42", size = 19752, upload-time = "2024-07-21T12:58:21.801Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, +] + +[[package]] +name = "py-cpuinfo" +version = "9.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2a/80/336820c1ad9286a4ded7e845b2eccfcb27851ab8ac6abece774a6ff4d3de/psutil-7.0.0.tar.gz", hash = "sha256:7be9c3eba38beccb6495ea33afd982a44074b78f28c434a1f51cc07fd315c456", size = 497003, upload-time = "2025-02-13T21:54:07.946Z" } +sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/e6/2d26234410f8b8abdbf891c9da62bee396583f713fb9f3325a4760875d22/psutil-7.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25", size = 238051, upload-time = "2025-02-13T21:54:12.36Z" }, - { url = "https://files.pythonhosted.org/packages/04/8b/30f930733afe425e3cbfc0e1468a30a18942350c1a8816acfade80c005c4/psutil-7.0.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da", size = 239535, upload-time = "2025-02-13T21:54:16.07Z" }, - { url = "https://files.pythonhosted.org/packages/2a/ed/d362e84620dd22876b55389248e522338ed1bf134a5edd3b8231d7207f6d/psutil-7.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fcee592b4c6f146991ca55919ea3d1f8926497a713ed7faaf8225e174581e91", size = 275004, upload-time = "2025-02-13T21:54:18.662Z" }, - { url = "https://files.pythonhosted.org/packages/bf/b9/b0eb3f3cbcb734d930fdf839431606844a825b23eaf9a6ab371edac8162c/psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34", size = 277986, upload-time = "2025-02-13T21:54:21.811Z" }, - { url = "https://files.pythonhosted.org/packages/eb/a2/709e0fe2f093556c17fbafda93ac032257242cabcc7ff3369e2cb76a97aa/psutil-7.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5f098451abc2828f7dc6b58d44b532b22f2088f4999a937557b603ce72b1993", size = 279544, upload-time = "2025-02-13T21:54:24.68Z" }, - { url = "https://files.pythonhosted.org/packages/50/e6/eecf58810b9d12e6427369784efe814a1eec0f492084ce8eb8f4d89d6d61/psutil-7.0.0-cp37-abi3-win32.whl", hash = "sha256:ba3fcef7523064a6c9da440fc4d6bd07da93ac726b5733c29027d7dc95b39d99", size = 241053, upload-time = "2025-02-13T21:54:34.31Z" }, - { url = "https://files.pythonhosted.org/packages/50/1b/6921afe68c74868b4c9fa424dad3be35b095e16687989ebbb50ce4fceb7c/psutil-7.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553", size = 244885, upload-time = "2025-02-13T21:54:37.486Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, ] [[package]] name = "pyarrow" -version = "20.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/ee/a7810cb9f3d6e9238e61d312076a9859bf3668fd21c69744de9532383912/pyarrow-20.0.0.tar.gz", hash = "sha256:febc4a913592573c8d5805091a6c2b5064c8bd6e002131f01061797d91c783c1", size = 1125187, upload-time = "2025-04-27T12:34:23.264Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/47/a2/b7930824181ceadd0c63c1042d01fa4ef63eee233934826a7a2a9af6e463/pyarrow-20.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:24ca380585444cb2a31324c546a9a56abbe87e26069189e14bdba19c86c049f0", size = 30856035, upload-time = "2025-04-27T12:28:40.78Z" }, - { url = "https://files.pythonhosted.org/packages/9b/18/c765770227d7f5bdfa8a69f64b49194352325c66a5c3bb5e332dfd5867d9/pyarrow-20.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:95b330059ddfdc591a3225f2d272123be26c8fa76e8c9ee1a77aad507361cfdb", size = 32309552, upload-time = "2025-04-27T12:28:47.051Z" }, - { url = "https://files.pythonhosted.org/packages/44/fb/dfb2dfdd3e488bb14f822d7335653092dde150cffc2da97de6e7500681f9/pyarrow-20.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f0fb1041267e9968c6d0d2ce3ff92e3928b243e2b6d11eeb84d9ac547308232", size = 41334704, upload-time = "2025-04-27T12:28:55.064Z" }, - { url = "https://files.pythonhosted.org/packages/58/0d/08a95878d38808051a953e887332d4a76bc06c6ee04351918ee1155407eb/pyarrow-20.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8ff87cc837601532cc8242d2f7e09b4e02404de1b797aee747dd4ba4bd6313f", size = 42399836, upload-time = "2025-04-27T12:29:02.13Z" }, - { url = "https://files.pythonhosted.org/packages/f3/cd/efa271234dfe38f0271561086eedcad7bc0f2ddd1efba423916ff0883684/pyarrow-20.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:7a3a5dcf54286e6141d5114522cf31dd67a9e7c9133d150799f30ee302a7a1ab", size = 40711789, upload-time = "2025-04-27T12:29:09.951Z" }, - { url = "https://files.pythonhosted.org/packages/46/1f/7f02009bc7fc8955c391defee5348f510e589a020e4b40ca05edcb847854/pyarrow-20.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a6ad3e7758ecf559900261a4df985662df54fb7fdb55e8e3b3aa99b23d526b62", size = 42301124, upload-time = "2025-04-27T12:29:17.187Z" }, - { url = "https://files.pythonhosted.org/packages/4f/92/692c562be4504c262089e86757a9048739fe1acb4024f92d39615e7bab3f/pyarrow-20.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6bb830757103a6cb300a04610e08d9636f0cd223d32f388418ea893a3e655f1c", size = 42916060, upload-time = "2025-04-27T12:29:24.253Z" }, - { url = "https://files.pythonhosted.org/packages/a4/ec/9f5c7e7c828d8e0a3c7ef50ee62eca38a7de2fa6eb1b8fa43685c9414fef/pyarrow-20.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:96e37f0766ecb4514a899d9a3554fadda770fb57ddf42b63d80f14bc20aa7db3", size = 44547640, upload-time = "2025-04-27T12:29:32.782Z" }, - { url = "https://files.pythonhosted.org/packages/54/96/46613131b4727f10fd2ffa6d0d6f02efcc09a0e7374eff3b5771548aa95b/pyarrow-20.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:3346babb516f4b6fd790da99b98bed9708e3f02e734c84971faccb20736848dc", size = 25781491, upload-time = "2025-04-27T12:29:38.464Z" }, - { url = "https://files.pythonhosted.org/packages/a1/d6/0c10e0d54f6c13eb464ee9b67a68b8c71bcf2f67760ef5b6fbcddd2ab05f/pyarrow-20.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:75a51a5b0eef32727a247707d4755322cb970be7e935172b6a3a9f9ae98404ba", size = 30815067, upload-time = "2025-04-27T12:29:44.384Z" }, - { url = "https://files.pythonhosted.org/packages/7e/e2/04e9874abe4094a06fd8b0cbb0f1312d8dd7d707f144c2ec1e5e8f452ffa/pyarrow-20.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:211d5e84cecc640c7a3ab900f930aaff5cd2702177e0d562d426fb7c4f737781", size = 32297128, upload-time = "2025-04-27T12:29:52.038Z" }, - { url = "https://files.pythonhosted.org/packages/31/fd/c565e5dcc906a3b471a83273039cb75cb79aad4a2d4a12f76cc5ae90a4b8/pyarrow-20.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ba3cf4182828be7a896cbd232aa8dd6a31bd1f9e32776cc3796c012855e1199", size = 41334890, upload-time = "2025-04-27T12:29:59.452Z" }, - { url = "https://files.pythonhosted.org/packages/af/a9/3bdd799e2c9b20c1ea6dc6fa8e83f29480a97711cf806e823f808c2316ac/pyarrow-20.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c3a01f313ffe27ac4126f4c2e5ea0f36a5fc6ab51f8726cf41fee4b256680bd", size = 42421775, upload-time = "2025-04-27T12:30:06.875Z" }, - { url = "https://files.pythonhosted.org/packages/10/f7/da98ccd86354c332f593218101ae56568d5dcedb460e342000bd89c49cc1/pyarrow-20.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:a2791f69ad72addd33510fec7bb14ee06c2a448e06b649e264c094c5b5f7ce28", size = 40687231, upload-time = "2025-04-27T12:30:13.954Z" }, - { url = "https://files.pythonhosted.org/packages/bb/1b/2168d6050e52ff1e6cefc61d600723870bf569cbf41d13db939c8cf97a16/pyarrow-20.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:4250e28a22302ce8692d3a0e8ec9d9dde54ec00d237cff4dfa9c1fbf79e472a8", size = 42295639, upload-time = "2025-04-27T12:30:21.949Z" }, - { url = "https://files.pythonhosted.org/packages/b2/66/2d976c0c7158fd25591c8ca55aee026e6d5745a021915a1835578707feb3/pyarrow-20.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:89e030dc58fc760e4010148e6ff164d2f44441490280ef1e97a542375e41058e", size = 42908549, upload-time = "2025-04-27T12:30:29.551Z" }, - { url = "https://files.pythonhosted.org/packages/31/a9/dfb999c2fc6911201dcbf348247f9cc382a8990f9ab45c12eabfd7243a38/pyarrow-20.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6102b4864d77102dbbb72965618e204e550135a940c2534711d5ffa787df2a5a", size = 44557216, upload-time = "2025-04-27T12:30:36.977Z" }, - { url = "https://files.pythonhosted.org/packages/a0/8e/9adee63dfa3911be2382fb4d92e4b2e7d82610f9d9f668493bebaa2af50f/pyarrow-20.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:96d6a0a37d9c98be08f5ed6a10831d88d52cac7b13f5287f1e0f625a0de8062b", size = 25660496, upload-time = "2025-04-27T12:30:42.809Z" }, +version = "21.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ef/c2/ea068b8f00905c06329a3dfcd40d0fcc2b7d0f2e355bdb25b65e0a0e4cd4/pyarrow-21.0.0.tar.gz", hash = "sha256:5051f2dccf0e283ff56335760cbc8622cf52264d67e359d5569541ac11b6d5bc", size = 1133487, upload-time = "2025-07-18T00:57:31.761Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/dc/80564a3071a57c20b7c32575e4a0120e8a330ef487c319b122942d665960/pyarrow-21.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:c077f48aab61738c237802836fc3844f85409a46015635198761b0d6a688f87b", size = 31243234, upload-time = "2025-07-18T00:55:03.812Z" }, + { url = "https://files.pythonhosted.org/packages/ea/cc/3b51cb2db26fe535d14f74cab4c79b191ed9a8cd4cbba45e2379b5ca2746/pyarrow-21.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:689f448066781856237eca8d1975b98cace19b8dd2ab6145bf49475478bcaa10", size = 32714370, upload-time = "2025-07-18T00:55:07.495Z" }, + { url = "https://files.pythonhosted.org/packages/24/11/a4431f36d5ad7d83b87146f515c063e4d07ef0b7240876ddb885e6b44f2e/pyarrow-21.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:479ee41399fcddc46159a551705b89c05f11e8b8cb8e968f7fec64f62d91985e", size = 41135424, upload-time = "2025-07-18T00:55:11.461Z" }, + { url = "https://files.pythonhosted.org/packages/74/dc/035d54638fc5d2971cbf1e987ccd45f1091c83bcf747281cf6cc25e72c88/pyarrow-21.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:40ebfcb54a4f11bcde86bc586cbd0272bac0d516cfa539c799c2453768477569", size = 42823810, upload-time = "2025-07-18T00:55:16.301Z" }, + { url = "https://files.pythonhosted.org/packages/2e/3b/89fced102448a9e3e0d4dded1f37fa3ce4700f02cdb8665457fcc8015f5b/pyarrow-21.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8d58d8497814274d3d20214fbb24abcad2f7e351474357d552a8d53bce70c70e", size = 43391538, upload-time = "2025-07-18T00:55:23.82Z" }, + { url = "https://files.pythonhosted.org/packages/fb/bb/ea7f1bd08978d39debd3b23611c293f64a642557e8141c80635d501e6d53/pyarrow-21.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:585e7224f21124dd57836b1530ac8f2df2afc43c861d7bf3d58a4870c42ae36c", size = 45120056, upload-time = "2025-07-18T00:55:28.231Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0b/77ea0600009842b30ceebc3337639a7380cd946061b620ac1a2f3cb541e2/pyarrow-21.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:555ca6935b2cbca2c0e932bedd853e9bc523098c39636de9ad4693b5b1df86d6", size = 26220568, upload-time = "2025-07-18T00:55:32.122Z" }, + { url = "https://files.pythonhosted.org/packages/ca/d4/d4f817b21aacc30195cf6a46ba041dd1be827efa4a623cc8bf39a1c2a0c0/pyarrow-21.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:3a302f0e0963db37e0a24a70c56cf91a4faa0bca51c23812279ca2e23481fccd", size = 31160305, upload-time = "2025-07-18T00:55:35.373Z" }, + { url = "https://files.pythonhosted.org/packages/a2/9c/dcd38ce6e4b4d9a19e1d36914cb8e2b1da4e6003dd075474c4cfcdfe0601/pyarrow-21.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:b6b27cf01e243871390474a211a7922bfbe3bda21e39bc9160daf0da3fe48876", size = 32684264, upload-time = "2025-07-18T00:55:39.303Z" }, + { url = "https://files.pythonhosted.org/packages/4f/74/2a2d9f8d7a59b639523454bec12dba35ae3d0a07d8ab529dc0809f74b23c/pyarrow-21.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e72a8ec6b868e258a2cd2672d91f2860ad532d590ce94cdf7d5e7ec674ccf03d", size = 41108099, upload-time = "2025-07-18T00:55:42.889Z" }, + { url = "https://files.pythonhosted.org/packages/ad/90/2660332eeb31303c13b653ea566a9918484b6e4d6b9d2d46879a33ab0622/pyarrow-21.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b7ae0bbdc8c6674259b25bef5d2a1d6af5d39d7200c819cf99e07f7dfef1c51e", size = 42829529, upload-time = "2025-07-18T00:55:47.069Z" }, + { url = "https://files.pythonhosted.org/packages/33/27/1a93a25c92717f6aa0fca06eb4700860577d016cd3ae51aad0e0488ac899/pyarrow-21.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:58c30a1729f82d201627c173d91bd431db88ea74dcaa3885855bc6203e433b82", size = 43367883, upload-time = "2025-07-18T00:55:53.069Z" }, + { url = "https://files.pythonhosted.org/packages/05/d9/4d09d919f35d599bc05c6950095e358c3e15148ead26292dfca1fb659b0c/pyarrow-21.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:072116f65604b822a7f22945a7a6e581cfa28e3454fdcc6939d4ff6090126623", size = 45133802, upload-time = "2025-07-18T00:55:57.714Z" }, + { url = "https://files.pythonhosted.org/packages/71/30/f3795b6e192c3ab881325ffe172e526499eb3780e306a15103a2764916a2/pyarrow-21.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:cf56ec8b0a5c8c9d7021d6fd754e688104f9ebebf1bf4449613c9531f5346a18", size = 26203175, upload-time = "2025-07-18T00:56:01.364Z" }, ] [[package]] @@ -2437,6 +3850,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c8/f1/d6a797abb14f6283c0ddff96bbdd46937f64122b8c925cab503dd37f8214/pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629", size = 83135, upload-time = "2024-09-11T16:00:36.122Z" }, ] +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, +] + [[package]] name = "pycodestyle" version = "2.14.0" @@ -2455,6 +3880,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552, upload-time = "2024-03-30T13:22:20.476Z" }, ] +[[package]] +name = "pycryptodome" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/a6/8452177684d5e906854776276ddd34eca30d1b1e15aa1ee9cefc289a33f5/pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef", size = 4921276, upload-time = "2025-05-17T17:21:45.242Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27", size = 2495627, upload-time = "2025-05-17T17:20:47.139Z" }, + { url = "https://files.pythonhosted.org/packages/6e/4e/a066527e079fc5002390c8acdd3aca431e6ea0a50ffd7201551175b47323/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843", size = 1640362, upload-time = "2025-05-17T17:20:50.392Z" }, + { url = "https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490", size = 2182625, upload-time = "2025-05-17T17:20:52.866Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575", size = 2268954, upload-time = "2025-05-17T17:20:55.027Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c5/ffe6474e0c551d54cab931918127c46d70cab8f114e0c2b5a3c071c2f484/pycryptodome-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b", size = 2308534, upload-time = "2025-05-17T17:20:57.279Z" }, + { url = "https://files.pythonhosted.org/packages/18/28/e199677fc15ecf43010f2463fde4c1a53015d1fe95fb03bca2890836603a/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a", size = 2181853, upload-time = "2025-05-17T17:20:59.322Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ea/4fdb09f2165ce1365c9eaefef36625583371ee514db58dc9b65d3a255c4c/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f", size = 2342465, upload-time = "2025-05-17T17:21:03.83Z" }, + { url = "https://files.pythonhosted.org/packages/22/82/6edc3fc42fe9284aead511394bac167693fb2b0e0395b28b8bedaa07ef04/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa", size = 2267414, upload-time = "2025-05-17T17:21:06.72Z" }, + { url = "https://files.pythonhosted.org/packages/59/fe/aae679b64363eb78326c7fdc9d06ec3de18bac68be4b612fc1fe8902693c/pycryptodome-3.23.0-cp37-abi3-win32.whl", hash = "sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886", size = 1768484, upload-time = "2025-05-17T17:21:08.535Z" }, + { url = "https://files.pythonhosted.org/packages/54/2f/e97a1b8294db0daaa87012c24a7bb714147c7ade7656973fd6c736b484ff/pycryptodome-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2", size = 1799636, upload-time = "2025-05-17T17:21:10.393Z" }, + { url = "https://files.pythonhosted.org/packages/18/3d/f9441a0d798bf2b1e645adc3265e55706aead1255ccdad3856dbdcffec14/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c", size = 1703675, upload-time = "2025-05-17T17:21:13.146Z" }, +] + [[package]] name = "pydantic" version = "2.10.6" @@ -2558,7 +4002,7 @@ wheels = [ [[package]] name = "pylint" -version = "3.3.7" +version = "3.3.8" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "astroid" }, @@ -2569,36 +4013,76 @@ dependencies = [ { name = "platformdirs" }, { name = "tomlkit" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1c/e4/83e487d3ddd64ab27749b66137b26dc0c5b5c161be680e6beffdc99070b3/pylint-3.3.7.tar.gz", hash = "sha256:2b11de8bde49f9c5059452e0c310c079c746a0a8eeaa789e5aa966ecc23e4559", size = 1520709, upload-time = "2025-05-04T17:07:51.089Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/58/1f614a84d3295c542e9f6e2c764533eea3f318f4592dc1ea06c797114767/pylint-3.3.8.tar.gz", hash = "sha256:26698de19941363037e2937d3db9ed94fb3303fdadf7d98847875345a8bb6b05", size = 1523947, upload-time = "2025-08-09T09:12:57.234Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/1a/711e93a7ab6c392e349428ea56e794a3902bb4e0284c1997cff2d7efdbc1/pylint-3.3.8-py3-none-any.whl", hash = "sha256:7ef94aa692a600e82fabdd17102b73fc226758218c97473c7ad67bd4cb905d83", size = 523153, upload-time = "2025-08-09T09:12:54.836Z" }, +] + +[[package]] +name = "pymilvus" +version = "2.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "grpcio" }, + { name = "milvus-lite", marker = "sys_platform != 'win32'" }, + { name = "pandas" }, + { name = "protobuf" }, + { name = "python-dotenv" }, + { name = "setuptools" }, + { name = "ujson" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/86/21/5c25a975299415a5a8f26d4759ddf7852aefdf3595f002b5203c4aaf5c8e/pymilvus-2.6.0.tar.gz", hash = "sha256:2b2ca487e098abc34231755e33af2f5294e9f6a64d92d03551532defbac0a3fb", size = 1292994, upload-time = "2025-08-06T09:09:01.705Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/a2/dfc2a2225aeb90a7dff9443f2d26fe9d04f6f7bcefe537945b5d5220fddd/pymilvus-2.6.0-py3-none-any.whl", hash = "sha256:d743fdd928c9007184d24a52b4f5dfdd18d405a37b4dba66b5ea4bf196fac526", size = 248299, upload-time = "2025-08-06T09:08:58.272Z" }, +] + +[[package]] +name = "pyopenssl" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/8c/cd89ad05804f8e3c17dea8f178c3f40eeab5694c30e0c9f5bcd49f576fc3/pyopenssl-25.1.0.tar.gz", hash = "sha256:8d031884482e0c67ee92bf9a4d8cceb08d92aba7136432ffb0703c5280fc205b", size = 179937, upload-time = "2025-05-17T16:28:31.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/80/28/2659c02301b9500751f8d42f9a6632e1508aa5120de5e43042b8b30f8d5d/pyopenssl-25.1.0-py3-none-any.whl", hash = "sha256:2b11f239acc47ac2e5aca04fd7fa829800aeee22a2eb30d744572a157bd8a1ab", size = 56771, upload-time = "2025-05-17T16:28:29.197Z" }, +] + +[[package]] +name = "pyparsing" +version = "3.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bb/22/f1129e69d94ffff626bdb5c835506b3a5b4f3d070f17ea295e12c2c6f60f/pyparsing-3.2.3.tar.gz", hash = "sha256:b9c13f1ab8b3b542f72e28f634bad4de758ab3ce4546e4301970ad6fa77c38be", size = 1088608, upload-time = "2025-03-25T05:01:28.114Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/83/bff755d09e31b5d25cc7fdc4bf3915d1a404e181f1abf0359af376845c24/pylint-3.3.7-py3-none-any.whl", hash = "sha256:43860aafefce92fca4cf6b61fe199cdc5ae54ea28f9bf4cd49de267b5195803d", size = 522565, upload-time = "2025-05-04T17:07:48.714Z" }, + { url = "https://files.pythonhosted.org/packages/05/e7/df2285f3d08fee213f2d041540fa4fc9ca6c2d44cf36d3a035bf2a8d2bcc/pyparsing-3.2.3-py3-none-any.whl", hash = "sha256:a749938e02d6fd0b59b356ca504a24982314bb090c383e3cf201c95ef7e2bfcf", size = 111120, upload-time = "2025-03-25T05:01:24.908Z" }, ] [[package]] -name = "pymilvus" -version = "2.6.0b0" +name = "pypdf" +version = "5.9.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "grpcio" }, - { name = "milvus-lite", marker = "sys_platform != 'win32'" }, - { name = "pandas" }, - { name = "protobuf" }, - { name = "python-dotenv" }, - { name = "setuptools" }, - { name = "ujson" }, +sdist = { url = "https://files.pythonhosted.org/packages/89/3a/584b97a228950ed85aec97c811c68473d9b8d149e6a8c155668287cf1a28/pypdf-5.9.0.tar.gz", hash = "sha256:30f67a614d558e495e1fbb157ba58c1de91ffc1718f5e0dfeb82a029233890a1", size = 5035118, upload-time = "2025-07-27T14:04:52.364Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/d9/6cff57c80a6963e7dd183bf09e9f21604a77716644b1e580e97b259f7612/pypdf-5.9.0-py3-none-any.whl", hash = "sha256:be10a4c54202f46d9daceaa8788be07aa8cd5ea8c25c529c50dd509206382c35", size = 313193, upload-time = "2025-07-27T14:04:50.53Z" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ef/fb/5e40ef2c4f5736c8c59084289d065df69510d549761941f7599ab80c1c1e/pymilvus-2.6.0b0.tar.gz", hash = "sha256:af907518417c3698112fdb6fe40db70798457d8c5c1961e56c64bf8fb126a721", size = 1281149, upload-time = "2025-06-18T16:11:15.269Z" } + +[[package]] +name = "pyreadline3" +version = "3.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/49/4cea918a08f02817aabae639e3d0ac046fef9f9180518a3ad394e22da148/pyreadline3-3.5.4.tar.gz", hash = "sha256:8d57d53039a1c75adba8e50dd3d992b28143480816187ea5efbd5c78e6c885b7", size = 99839, upload-time = "2024-09-19T02:40:10.062Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/80/fb/7cf9aaf57c0866ea0df851f052366d4a9b65794dda71f6f140168035c219/pymilvus-2.6.0b0-py3-none-any.whl", hash = "sha256:0a585e38b6492aa298f5a8cb4d59cc332a3fead4068cfe49ffe2998029ae3997", size = 240167, upload-time = "2025-06-18T16:11:11.404Z" }, + { url = "https://files.pythonhosted.org/packages/5a/dc/491b7661614ab97483abf2056be1deee4dc2490ecbf7bff9ab5cdbac86e1/pyreadline3-3.5.4-py3-none-any.whl", hash = "sha256:eaf8e6cc3c49bcccf145fc6067ba8643d1df34d604a1ec0eccbf7a18e6d3fae6", size = 83178, upload-time = "2024-09-19T02:40:08.598Z" }, ] [[package]] -name = "pyparsing" -version = "3.2.3" +name = "pystache" +version = "0.6.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bb/22/f1129e69d94ffff626bdb5c835506b3a5b4f3d070f17ea295e12c2c6f60f/pyparsing-3.2.3.tar.gz", hash = "sha256:b9c13f1ab8b3b542f72e28f634bad4de758ab3ce4546e4301970ad6fa77c38be", size = 1088608, upload-time = "2025-03-25T05:01:28.114Z" } +sdist = { url = "https://files.pythonhosted.org/packages/85/89/0a712ca22930b8c71bced8703e5bb45669c31690ea81afe15f6cb284550c/pystache-0.6.8.tar.gz", hash = "sha256:3707518e6a4d26dd189b07c10c669b1fc17df72684617c327bd3550e7075c72c", size = 101892, upload-time = "2025-03-18T11:54:47.595Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/05/e7/df2285f3d08fee213f2d041540fa4fc9ca6c2d44cf36d3a035bf2a8d2bcc/pyparsing-3.2.3-py3-none-any.whl", hash = "sha256:a749938e02d6fd0b59b356ca504a24982314bb090c383e3cf201c95ef7e2bfcf", size = 111120, upload-time = "2025-03-25T05:01:24.908Z" }, + { url = "https://files.pythonhosted.org/packages/fa/78/ffd13a516219129cef6a754a11ba2a1c0d69f1e281af4f6bca9ed5327219/pystache-0.6.8-py3-none-any.whl", hash = "sha256:7211e000974a6e06bce2d4d5cad8df03bcfffefd367209117376e4527a1c3cb8", size = 82051, upload-time = "2025-03-18T11:54:45.813Z" }, ] [[package]] @@ -2746,6 +4230,92 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338, upload-time = "2024-08-06T20:32:41.93Z" }, ] +[[package]] +name = "pyzmq" +version = "27.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "implementation_name == 'pypy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/30/5f/557d2032a2f471edbcc227da724c24a1c05887b5cda1e3ae53af98b9e0a5/pyzmq-27.0.1.tar.gz", hash = "sha256:45c549204bc20e7484ffd2555f6cf02e572440ecf2f3bdd60d4404b20fddf64b", size = 281158, upload-time = "2025-08-03T05:05:40.352Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/18/a8e0da6ababbe9326116fb1c890bf1920eea880e8da621afb6bc0f39a262/pyzmq-27.0.1-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:9729190bd770314f5fbba42476abf6abe79a746eeda11d1d68fd56dd70e5c296", size = 1332721, upload-time = "2025-08-03T05:03:15.237Z" }, + { url = "https://files.pythonhosted.org/packages/75/a4/9431ba598651d60ebd50dc25755402b770322cf8432adcc07d2906e53a54/pyzmq-27.0.1-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:696900ef6bc20bef6a242973943574f96c3f97d2183c1bd3da5eea4f559631b1", size = 908249, upload-time = "2025-08-03T05:03:16.933Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/e624e1793689e4e685d2ee21c40277dd4024d9d730af20446d88f69be838/pyzmq-27.0.1-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f96a63aecec22d3f7fdea3c6c98df9e42973f5856bb6812c3d8d78c262fee808", size = 668649, upload-time = "2025-08-03T05:03:18.49Z" }, + { url = "https://files.pythonhosted.org/packages/6c/29/0652a39d4e876e0d61379047ecf7752685414ad2e253434348246f7a2a39/pyzmq-27.0.1-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c512824360ea7490390566ce00bee880e19b526b312b25cc0bc30a0fe95cb67f", size = 856601, upload-time = "2025-08-03T05:03:20.194Z" }, + { url = "https://files.pythonhosted.org/packages/36/2d/8d5355d7fc55bb6e9c581dd74f58b64fa78c994079e3a0ea09b1b5627cde/pyzmq-27.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dfb2bb5e0f7198eaacfb6796fb0330afd28f36d985a770745fba554a5903595a", size = 1657750, upload-time = "2025-08-03T05:03:22.055Z" }, + { url = "https://files.pythonhosted.org/packages/ab/f4/cd032352d5d252dc6f5ee272a34b59718ba3af1639a8a4ef4654f9535cf5/pyzmq-27.0.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4f6886c59ba93ffde09b957d3e857e7950c8fe818bd5494d9b4287bc6d5bc7f1", size = 2034312, upload-time = "2025-08-03T05:03:23.578Z" }, + { url = "https://files.pythonhosted.org/packages/e4/1a/c050d8b6597200e97a4bd29b93c769d002fa0b03083858227e0376ad59bc/pyzmq-27.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b99ea9d330e86ce1ff7f2456b33f1bf81c43862a5590faf4ef4ed3a63504bdab", size = 1893632, upload-time = "2025-08-03T05:03:25.167Z" }, + { url = "https://files.pythonhosted.org/packages/6a/29/173ce21d5097e7fcf284a090e8beb64fc683c6582b1f00fa52b1b7e867ce/pyzmq-27.0.1-cp311-cp311-win32.whl", hash = "sha256:571f762aed89025ba8cdcbe355fea56889715ec06d0264fd8b6a3f3fa38154ed", size = 566587, upload-time = "2025-08-03T05:03:26.769Z" }, + { url = "https://files.pythonhosted.org/packages/53/ab/22bd33e7086f0a2cc03a5adabff4bde414288bb62a21a7820951ef86ec20/pyzmq-27.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:ee16906c8025fa464bea1e48128c048d02359fb40bebe5333103228528506530", size = 632873, upload-time = "2025-08-03T05:03:28.685Z" }, + { url = "https://files.pythonhosted.org/packages/90/14/3e59b4a28194285ceeff725eba9aa5ba8568d1cb78aed381dec1537c705a/pyzmq-27.0.1-cp311-cp311-win_arm64.whl", hash = "sha256:ba068f28028849da725ff9185c24f832ccf9207a40f9b28ac46ab7c04994bd41", size = 558918, upload-time = "2025-08-03T05:03:30.085Z" }, + { url = "https://files.pythonhosted.org/packages/0e/9b/c0957041067c7724b310f22c398be46399297c12ed834c3bc42200a2756f/pyzmq-27.0.1-cp312-abi3-macosx_10_15_universal2.whl", hash = "sha256:af7ebce2a1e7caf30c0bb64a845f63a69e76a2fadbc1cac47178f7bb6e657bdd", size = 1305432, upload-time = "2025-08-03T05:03:32.177Z" }, + { url = "https://files.pythonhosted.org/packages/8e/55/bd3a312790858f16b7def3897a0c3eb1804e974711bf7b9dcb5f47e7f82c/pyzmq-27.0.1-cp312-abi3-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:8f617f60a8b609a13099b313e7e525e67f84ef4524b6acad396d9ff153f6e4cd", size = 895095, upload-time = "2025-08-03T05:03:33.918Z" }, + { url = "https://files.pythonhosted.org/packages/20/50/fc384631d8282809fb1029a4460d2fe90fa0370a0e866a8318ed75c8d3bb/pyzmq-27.0.1-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d59dad4173dc2a111f03e59315c7bd6e73da1a9d20a84a25cf08325b0582b1a", size = 651826, upload-time = "2025-08-03T05:03:35.818Z" }, + { url = "https://files.pythonhosted.org/packages/7e/0a/2356305c423a975000867de56888b79e44ec2192c690ff93c3109fd78081/pyzmq-27.0.1-cp312-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5b6133c8d313bde8bd0d123c169d22525300ff164c2189f849de495e1344577", size = 839751, upload-time = "2025-08-03T05:03:37.265Z" }, + { url = "https://files.pythonhosted.org/packages/d7/1b/81e95ad256ca7e7ccd47f5294c1c6da6e2b64fbace65b84fe8a41470342e/pyzmq-27.0.1-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:58cca552567423f04d06a075f4b473e78ab5bdb906febe56bf4797633f54aa4e", size = 1641359, upload-time = "2025-08-03T05:03:38.799Z" }, + { url = "https://files.pythonhosted.org/packages/50/63/9f50ec965285f4e92c265c8f18344e46b12803666d8b73b65d254d441435/pyzmq-27.0.1-cp312-abi3-musllinux_1_2_i686.whl", hash = "sha256:4b9d8e26fb600d0d69cc9933e20af08552e97cc868a183d38a5c0d661e40dfbb", size = 2020281, upload-time = "2025-08-03T05:03:40.338Z" }, + { url = "https://files.pythonhosted.org/packages/02/4a/19e3398d0dc66ad2b463e4afa1fc541d697d7bc090305f9dfb948d3dfa29/pyzmq-27.0.1-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2329f0c87f0466dce45bba32b63f47018dda5ca40a0085cc5c8558fea7d9fc55", size = 1877112, upload-time = "2025-08-03T05:03:42.012Z" }, + { url = "https://files.pythonhosted.org/packages/bf/42/c562e9151aa90ed1d70aac381ea22a929d6b3a2ce4e1d6e2e135d34fd9c6/pyzmq-27.0.1-cp312-abi3-win32.whl", hash = "sha256:57bb92abdb48467b89c2d21da1ab01a07d0745e536d62afd2e30d5acbd0092eb", size = 558177, upload-time = "2025-08-03T05:03:43.979Z" }, + { url = "https://files.pythonhosted.org/packages/40/96/5c50a7d2d2b05b19994bf7336b97db254299353dd9b49b565bb71b485f03/pyzmq-27.0.1-cp312-abi3-win_amd64.whl", hash = "sha256:ff3f8757570e45da7a5bedaa140489846510014f7a9d5ee9301c61f3f1b8a686", size = 618923, upload-time = "2025-08-03T05:03:45.438Z" }, + { url = "https://files.pythonhosted.org/packages/13/33/1ec89c8f21c89d21a2eaff7def3676e21d8248d2675705e72554fb5a6f3f/pyzmq-27.0.1-cp312-abi3-win_arm64.whl", hash = "sha256:df2c55c958d3766bdb3e9d858b911288acec09a9aab15883f384fc7180df5bed", size = 552358, upload-time = "2025-08-03T05:03:46.887Z" }, + { url = "https://files.pythonhosted.org/packages/b4/1a/49f66fe0bc2b2568dd4280f1f520ac8fafd73f8d762140e278d48aeaf7b9/pyzmq-27.0.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7fb0ee35845bef1e8c4a152d766242164e138c239e3182f558ae15cb4a891f94", size = 835949, upload-time = "2025-08-03T05:05:13.798Z" }, + { url = "https://files.pythonhosted.org/packages/49/94/443c1984b397eab59b14dd7ae8bc2ac7e8f32dbc646474453afcaa6508c4/pyzmq-27.0.1-pp311-pypy311_pp73-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:f379f11e138dfd56c3f24a04164f871a08281194dd9ddf656a278d7d080c8ad0", size = 799875, upload-time = "2025-08-03T05:05:15.632Z" }, + { url = "https://files.pythonhosted.org/packages/30/f1/fd96138a0f152786a2ba517e9c6a8b1b3516719e412a90bb5d8eea6b660c/pyzmq-27.0.1-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b978c0678cffbe8860ec9edc91200e895c29ae1ac8a7085f947f8e8864c489fb", size = 567403, upload-time = "2025-08-03T05:05:17.326Z" }, + { url = "https://files.pythonhosted.org/packages/16/57/34e53ef2b55b1428dac5aabe3a974a16c8bda3bf20549ba500e3ff6cb426/pyzmq-27.0.1-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7ebccf0d760bc92a4a7c751aeb2fef6626144aace76ee8f5a63abeb100cae87f", size = 747032, upload-time = "2025-08-03T05:05:19.074Z" }, + { url = "https://files.pythonhosted.org/packages/81/b7/769598c5ae336fdb657946950465569cf18803140fe89ce466d7f0a57c11/pyzmq-27.0.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:77fed80e30fa65708546c4119840a46691290efc231f6bfb2ac2a39b52e15811", size = 544566, upload-time = "2025-08-03T05:05:20.798Z" }, +] + +[[package]] +name = "ragaai-catalyst" +version = "2.2.6b7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "google-genai" }, + { name = "gputil" }, + { name = "groq" }, + { name = "ipynbname" }, + { name = "langchain" }, + { name = "langchain-core" }, + { name = "litellm" }, + { name = "llama-index" }, + { name = "markdown" }, + { name = "openai" }, + { name = "openinference-instrumentation-anthropic" }, + { name = "openinference-instrumentation-bedrock" }, + { name = "openinference-instrumentation-crewai" }, + { name = "openinference-instrumentation-google-adk" }, + { name = "openinference-instrumentation-groq" }, + { name = "openinference-instrumentation-haystack" }, + { name = "openinference-instrumentation-langchain" }, + { name = "openinference-instrumentation-litellm" }, + { name = "openinference-instrumentation-llama-index" }, + { name = "openinference-instrumentation-mistralai" }, + { name = "openinference-instrumentation-openai" }, + { name = "openinference-instrumentation-openai-agents" }, + { name = "openinference-instrumentation-smolagents" }, + { name = "openinference-instrumentation-vertexai" }, + { name = "opentelemetry-exporter-otlp" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "pandas" }, + { name = "psutil" }, + { name = "py-cpuinfo" }, + { name = "pyopenssl" }, + { name = "pypdf" }, + { name = "requests" }, + { name = "rich" }, + { name = "tenacity" }, + { name = "tiktoken" }, + { name = "tomli" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/16/93/92bad2ae2a539e25976a4791d786692af89fa8274c6fafd2e2c4742ab8e1/ragaai_catalyst-2.2.6b7.tar.gz", hash = "sha256:6d28132ed83d16ff9ebdf0b7edf7ac4d941f3f8e669abc0b8b1535050580b513", size = 40941954, upload-time = "2025-07-16T07:56:31.566Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/3f/61cad60f6d617fed4c8cee67285e0173622df417d68a16883f4ebffa39df/ragaai_catalyst-2.2.6b7-py3-none-any.whl", hash = "sha256:50f6a49f5c24845e0eb5676ac2d8b6ea37d2de02d2cef0f608e6ed8606b9a3a9", size = 443209, upload-time = "2025-07-16T07:56:26.31Z" }, +] + [[package]] name = "ragas" version = "0.2.15" @@ -2797,45 +4367,43 @@ wheels = [ [[package]] name = "regex" -version = "2024.11.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8e/5f/bd69653fbfb76cf8604468d3b4ec4c403197144c7bfe0e6a5fc9e02a07cb/regex-2024.11.6.tar.gz", hash = "sha256:7ab159b063c52a0333c884e4679f8d7a85112ee3078fe3d9004b2dd875585519", size = 399494, upload-time = "2024-11-06T20:12:31.635Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/58/58/7e4d9493a66c88a7da6d205768119f51af0f684fe7be7bac8328e217a52c/regex-2024.11.6-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5478c6962ad548b54a591778e93cd7c456a7a29f8eca9c49e4f9a806dcc5d638", size = 482669, upload-time = "2024-11-06T20:09:31.064Z" }, - { url = "https://files.pythonhosted.org/packages/34/4c/8f8e631fcdc2ff978609eaeef1d6994bf2f028b59d9ac67640ed051f1218/regex-2024.11.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2c89a8cc122b25ce6945f0423dc1352cb9593c68abd19223eebbd4e56612c5b7", size = 287684, upload-time = "2024-11-06T20:09:32.915Z" }, - { url = "https://files.pythonhosted.org/packages/c5/1b/f0e4d13e6adf866ce9b069e191f303a30ab1277e037037a365c3aad5cc9c/regex-2024.11.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:94d87b689cdd831934fa3ce16cc15cd65748e6d689f5d2b8f4f4df2065c9fa20", size = 284589, upload-time = "2024-11-06T20:09:35.504Z" }, - { url = "https://files.pythonhosted.org/packages/25/4d/ab21047f446693887f25510887e6820b93f791992994f6498b0318904d4a/regex-2024.11.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1062b39a0a2b75a9c694f7a08e7183a80c63c0d62b301418ffd9c35f55aaa114", size = 792121, upload-time = "2024-11-06T20:09:37.701Z" }, - { url = "https://files.pythonhosted.org/packages/45/ee/c867e15cd894985cb32b731d89576c41a4642a57850c162490ea34b78c3b/regex-2024.11.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:167ed4852351d8a750da48712c3930b031f6efdaa0f22fa1933716bfcd6bf4a3", size = 831275, upload-time = "2024-11-06T20:09:40.371Z" }, - { url = "https://files.pythonhosted.org/packages/b3/12/b0f480726cf1c60f6536fa5e1c95275a77624f3ac8fdccf79e6727499e28/regex-2024.11.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2d548dafee61f06ebdb584080621f3e0c23fff312f0de1afc776e2a2ba99a74f", size = 818257, upload-time = "2024-11-06T20:09:43.059Z" }, - { url = "https://files.pythonhosted.org/packages/bf/ce/0d0e61429f603bac433910d99ef1a02ce45a8967ffbe3cbee48599e62d88/regex-2024.11.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2a19f302cd1ce5dd01a9099aaa19cae6173306d1302a43b627f62e21cf18ac0", size = 792727, upload-time = "2024-11-06T20:09:48.19Z" }, - { url = "https://files.pythonhosted.org/packages/e4/c1/243c83c53d4a419c1556f43777ccb552bccdf79d08fda3980e4e77dd9137/regex-2024.11.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bec9931dfb61ddd8ef2ebc05646293812cb6b16b60cf7c9511a832b6f1854b55", size = 780667, upload-time = "2024-11-06T20:09:49.828Z" }, - { url = "https://files.pythonhosted.org/packages/c5/f4/75eb0dd4ce4b37f04928987f1d22547ddaf6c4bae697623c1b05da67a8aa/regex-2024.11.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9714398225f299aa85267fd222f7142fcb5c769e73d7733344efc46f2ef5cf89", size = 776963, upload-time = "2024-11-06T20:09:51.819Z" }, - { url = "https://files.pythonhosted.org/packages/16/5d/95c568574e630e141a69ff8a254c2f188b4398e813c40d49228c9bbd9875/regex-2024.11.6-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:202eb32e89f60fc147a41e55cb086db2a3f8cb82f9a9a88440dcfc5d37faae8d", size = 784700, upload-time = "2024-11-06T20:09:53.982Z" }, - { url = "https://files.pythonhosted.org/packages/8e/b5/f8495c7917f15cc6fee1e7f395e324ec3e00ab3c665a7dc9d27562fd5290/regex-2024.11.6-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4181b814e56078e9b00427ca358ec44333765f5ca1b45597ec7446d3a1ef6e34", size = 848592, upload-time = "2024-11-06T20:09:56.222Z" }, - { url = "https://files.pythonhosted.org/packages/1c/80/6dd7118e8cb212c3c60b191b932dc57db93fb2e36fb9e0e92f72a5909af9/regex-2024.11.6-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:068376da5a7e4da51968ce4c122a7cd31afaaec4fccc7856c92f63876e57b51d", size = 852929, upload-time = "2024-11-06T20:09:58.642Z" }, - { url = "https://files.pythonhosted.org/packages/11/9b/5a05d2040297d2d254baf95eeeb6df83554e5e1df03bc1a6687fc4ba1f66/regex-2024.11.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ac10f2c4184420d881a3475fb2c6f4d95d53a8d50209a2500723d831036f7c45", size = 781213, upload-time = "2024-11-06T20:10:00.867Z" }, - { url = "https://files.pythonhosted.org/packages/26/b7/b14e2440156ab39e0177506c08c18accaf2b8932e39fb092074de733d868/regex-2024.11.6-cp311-cp311-win32.whl", hash = "sha256:c36f9b6f5f8649bb251a5f3f66564438977b7ef8386a52460ae77e6070d309d9", size = 261734, upload-time = "2024-11-06T20:10:03.361Z" }, - { url = "https://files.pythonhosted.org/packages/80/32/763a6cc01d21fb3819227a1cc3f60fd251c13c37c27a73b8ff4315433a8e/regex-2024.11.6-cp311-cp311-win_amd64.whl", hash = "sha256:02e28184be537f0e75c1f9b2f8847dc51e08e6e171c6bde130b2687e0c33cf60", size = 274052, upload-time = "2024-11-06T20:10:05.179Z" }, - { url = "https://files.pythonhosted.org/packages/ba/30/9a87ce8336b172cc232a0db89a3af97929d06c11ceaa19d97d84fa90a8f8/regex-2024.11.6-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:52fb28f528778f184f870b7cf8f225f5eef0a8f6e3778529bdd40c7b3920796a", size = 483781, upload-time = "2024-11-06T20:10:07.07Z" }, - { url = "https://files.pythonhosted.org/packages/01/e8/00008ad4ff4be8b1844786ba6636035f7ef926db5686e4c0f98093612add/regex-2024.11.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdd6028445d2460f33136c55eeb1f601ab06d74cb3347132e1c24250187500d9", size = 288455, upload-time = "2024-11-06T20:10:09.117Z" }, - { url = "https://files.pythonhosted.org/packages/60/85/cebcc0aff603ea0a201667b203f13ba75d9fc8668fab917ac5b2de3967bc/regex-2024.11.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:805e6b60c54bf766b251e94526ebad60b7de0c70f70a4e6210ee2891acb70bf2", size = 284759, upload-time = "2024-11-06T20:10:11.155Z" }, - { url = "https://files.pythonhosted.org/packages/94/2b/701a4b0585cb05472a4da28ee28fdfe155f3638f5e1ec92306d924e5faf0/regex-2024.11.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b85c2530be953a890eaffde05485238f07029600e8f098cdf1848d414a8b45e4", size = 794976, upload-time = "2024-11-06T20:10:13.24Z" }, - { url = "https://files.pythonhosted.org/packages/4b/bf/fa87e563bf5fee75db8915f7352e1887b1249126a1be4813837f5dbec965/regex-2024.11.6-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb26437975da7dc36b7efad18aa9dd4ea569d2357ae6b783bf1118dabd9ea577", size = 833077, upload-time = "2024-11-06T20:10:15.37Z" }, - { url = "https://files.pythonhosted.org/packages/a1/56/7295e6bad94b047f4d0834e4779491b81216583c00c288252ef625c01d23/regex-2024.11.6-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:abfa5080c374a76a251ba60683242bc17eeb2c9818d0d30117b4486be10c59d3", size = 823160, upload-time = "2024-11-06T20:10:19.027Z" }, - { url = "https://files.pythonhosted.org/packages/fb/13/e3b075031a738c9598c51cfbc4c7879e26729c53aa9cca59211c44235314/regex-2024.11.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b7fa6606c2881c1db9479b0eaa11ed5dfa11c8d60a474ff0e095099f39d98e", size = 796896, upload-time = "2024-11-06T20:10:21.85Z" }, - { url = "https://files.pythonhosted.org/packages/24/56/0b3f1b66d592be6efec23a795b37732682520b47c53da5a32c33ed7d84e3/regex-2024.11.6-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0c32f75920cf99fe6b6c539c399a4a128452eaf1af27f39bce8909c9a3fd8cbe", size = 783997, upload-time = "2024-11-06T20:10:24.329Z" }, - { url = "https://files.pythonhosted.org/packages/f9/a1/eb378dada8b91c0e4c5f08ffb56f25fcae47bf52ad18f9b2f33b83e6d498/regex-2024.11.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:982e6d21414e78e1f51cf595d7f321dcd14de1f2881c5dc6a6e23bbbbd68435e", size = 781725, upload-time = "2024-11-06T20:10:28.067Z" }, - { url = "https://files.pythonhosted.org/packages/83/f2/033e7dec0cfd6dda93390089864732a3409246ffe8b042e9554afa9bff4e/regex-2024.11.6-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a7c2155f790e2fb448faed6dd241386719802296ec588a8b9051c1f5c481bc29", size = 789481, upload-time = "2024-11-06T20:10:31.612Z" }, - { url = "https://files.pythonhosted.org/packages/83/23/15d4552ea28990a74e7696780c438aadd73a20318c47e527b47a4a5a596d/regex-2024.11.6-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:149f5008d286636e48cd0b1dd65018548944e495b0265b45e1bffecce1ef7f39", size = 852896, upload-time = "2024-11-06T20:10:34.054Z" }, - { url = "https://files.pythonhosted.org/packages/e3/39/ed4416bc90deedbfdada2568b2cb0bc1fdb98efe11f5378d9892b2a88f8f/regex-2024.11.6-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e5364a4502efca094731680e80009632ad6624084aff9a23ce8c8c6820de3e51", size = 860138, upload-time = "2024-11-06T20:10:36.142Z" }, - { url = "https://files.pythonhosted.org/packages/93/2d/dd56bb76bd8e95bbce684326302f287455b56242a4f9c61f1bc76e28360e/regex-2024.11.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0a86e7eeca091c09e021db8eb72d54751e527fa47b8d5787caf96d9831bd02ad", size = 787692, upload-time = "2024-11-06T20:10:38.394Z" }, - { url = "https://files.pythonhosted.org/packages/0b/55/31877a249ab7a5156758246b9c59539abbeba22461b7d8adc9e8475ff73e/regex-2024.11.6-cp312-cp312-win32.whl", hash = "sha256:32f9a4c643baad4efa81d549c2aadefaeba12249b2adc5af541759237eee1c54", size = 262135, upload-time = "2024-11-06T20:10:40.367Z" }, - { url = "https://files.pythonhosted.org/packages/38/ec/ad2d7de49a600cdb8dd78434a1aeffe28b9d6fc42eb36afab4a27ad23384/regex-2024.11.6-cp312-cp312-win_amd64.whl", hash = "sha256:a93c194e2df18f7d264092dc8539b8ffb86b45b899ab976aa15d48214138e81b", size = 273567, upload-time = "2024-11-06T20:10:43.467Z" }, +version = "2025.7.34" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/de/e13fa6dc61d78b30ba47481f99933a3b49a57779d625c392d8036770a60d/regex-2025.7.34.tar.gz", hash = "sha256:9ead9765217afd04a86822dfcd4ed2747dfe426e887da413b15ff0ac2457e21a", size = 400714, upload-time = "2025-07-31T00:21:16.262Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/85/f497b91577169472f7c1dc262a5ecc65e39e146fc3a52c571e5daaae4b7d/regex-2025.7.34-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:da304313761b8500b8e175eb2040c4394a875837d5635f6256d6fa0377ad32c8", size = 484594, upload-time = "2025-07-31T00:19:13.927Z" }, + { url = "https://files.pythonhosted.org/packages/1c/c5/ad2a5c11ce9e6257fcbfd6cd965d07502f6054aaa19d50a3d7fd991ec5d1/regex-2025.7.34-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:35e43ebf5b18cd751ea81455b19acfdec402e82fe0dc6143edfae4c5c4b3909a", size = 289294, upload-time = "2025-07-31T00:19:15.395Z" }, + { url = "https://files.pythonhosted.org/packages/8e/01/83ffd9641fcf5e018f9b51aa922c3e538ac9439424fda3df540b643ecf4f/regex-2025.7.34-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:96bbae4c616726f4661fe7bcad5952e10d25d3c51ddc388189d8864fbc1b3c68", size = 285933, upload-time = "2025-07-31T00:19:16.704Z" }, + { url = "https://files.pythonhosted.org/packages/77/20/5edab2e5766f0259bc1da7381b07ce6eb4401b17b2254d02f492cd8a81a8/regex-2025.7.34-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9feab78a1ffa4f2b1e27b1bcdaad36f48c2fed4870264ce32f52a393db093c78", size = 792335, upload-time = "2025-07-31T00:19:18.561Z" }, + { url = "https://files.pythonhosted.org/packages/30/bd/744d3ed8777dce8487b2606b94925e207e7c5931d5870f47f5b643a4580a/regex-2025.7.34-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f14b36e6d4d07f1a5060f28ef3b3561c5d95eb0651741474ce4c0a4c56ba8719", size = 858605, upload-time = "2025-07-31T00:19:20.204Z" }, + { url = "https://files.pythonhosted.org/packages/99/3d/93754176289718d7578c31d151047e7b8acc7a8c20e7706716f23c49e45e/regex-2025.7.34-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:85c3a958ef8b3d5079c763477e1f09e89d13ad22198a37e9d7b26b4b17438b33", size = 905780, upload-time = "2025-07-31T00:19:21.876Z" }, + { url = "https://files.pythonhosted.org/packages/ee/2e/c689f274a92deffa03999a430505ff2aeace408fd681a90eafa92fdd6930/regex-2025.7.34-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:37555e4ae0b93358fa7c2d240a4291d4a4227cc7c607d8f85596cdb08ec0a083", size = 798868, upload-time = "2025-07-31T00:19:23.222Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9e/39673688805d139b33b4a24851a71b9978d61915c4d72b5ffda324d0668a/regex-2025.7.34-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee38926f31f1aa61b0232a3a11b83461f7807661c062df9eb88769d86e6195c3", size = 781784, upload-time = "2025-07-31T00:19:24.59Z" }, + { url = "https://files.pythonhosted.org/packages/18/bd/4c1cab12cfabe14beaa076523056b8ab0c882a8feaf0a6f48b0a75dab9ed/regex-2025.7.34-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a664291c31cae9c4a30589bd8bc2ebb56ef880c9c6264cb7643633831e606a4d", size = 852837, upload-time = "2025-07-31T00:19:25.911Z" }, + { url = "https://files.pythonhosted.org/packages/cb/21/663d983cbb3bba537fc213a579abbd0f263fb28271c514123f3c547ab917/regex-2025.7.34-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:f3e5c1e0925e77ec46ddc736b756a6da50d4df4ee3f69536ffb2373460e2dafd", size = 844240, upload-time = "2025-07-31T00:19:27.688Z" }, + { url = "https://files.pythonhosted.org/packages/8e/2d/9beeeb913bc5d32faa913cf8c47e968da936af61ec20af5d269d0f84a100/regex-2025.7.34-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d428fc7731dcbb4e2ffe43aeb8f90775ad155e7db4347a639768bc6cd2df881a", size = 787139, upload-time = "2025-07-31T00:19:29.475Z" }, + { url = "https://files.pythonhosted.org/packages/eb/f5/9b9384415fdc533551be2ba805dd8c4621873e5df69c958f403bfd3b2b6e/regex-2025.7.34-cp311-cp311-win32.whl", hash = "sha256:e154a7ee7fa18333ad90b20e16ef84daaeac61877c8ef942ec8dfa50dc38b7a1", size = 264019, upload-time = "2025-07-31T00:19:31.129Z" }, + { url = "https://files.pythonhosted.org/packages/18/9d/e069ed94debcf4cc9626d652a48040b079ce34c7e4fb174f16874958d485/regex-2025.7.34-cp311-cp311-win_amd64.whl", hash = "sha256:24257953d5c1d6d3c129ab03414c07fc1a47833c9165d49b954190b2b7f21a1a", size = 276047, upload-time = "2025-07-31T00:19:32.497Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cf/3bafbe9d1fd1db77355e7fbbbf0d0cfb34501a8b8e334deca14f94c7b315/regex-2025.7.34-cp311-cp311-win_arm64.whl", hash = "sha256:3157aa512b9e606586900888cd469a444f9b898ecb7f8931996cb715f77477f0", size = 268362, upload-time = "2025-07-31T00:19:34.094Z" }, + { url = "https://files.pythonhosted.org/packages/ff/f0/31d62596c75a33f979317658e8d261574785c6cd8672c06741ce2e2e2070/regex-2025.7.34-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7f7211a746aced993bef487de69307a38c5ddd79257d7be83f7b202cb59ddb50", size = 485492, upload-time = "2025-07-31T00:19:35.57Z" }, + { url = "https://files.pythonhosted.org/packages/d8/16/b818d223f1c9758c3434be89aa1a01aae798e0e0df36c1f143d1963dd1ee/regex-2025.7.34-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fb31080f2bd0681484b275461b202b5ad182f52c9ec606052020fe13eb13a72f", size = 290000, upload-time = "2025-07-31T00:19:37.175Z" }, + { url = "https://files.pythonhosted.org/packages/cd/70/69506d53397b4bd6954061bae75677ad34deb7f6ca3ba199660d6f728ff5/regex-2025.7.34-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0200a5150c4cf61e407038f4b4d5cdad13e86345dac29ff9dab3d75d905cf130", size = 286072, upload-time = "2025-07-31T00:19:38.612Z" }, + { url = "https://files.pythonhosted.org/packages/b0/73/536a216d5f66084fb577bb0543b5cb7de3272eb70a157f0c3a542f1c2551/regex-2025.7.34-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:739a74970e736df0773788377969c9fea3876c2fc13d0563f98e5503e5185f46", size = 797341, upload-time = "2025-07-31T00:19:40.119Z" }, + { url = "https://files.pythonhosted.org/packages/26/af/733f8168449e56e8f404bb807ea7189f59507cbea1b67a7bbcd92f8bf844/regex-2025.7.34-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4fef81b2f7ea6a2029161ed6dea9ae13834c28eb5a95b8771828194a026621e4", size = 862556, upload-time = "2025-07-31T00:19:41.556Z" }, + { url = "https://files.pythonhosted.org/packages/19/dd/59c464d58c06c4f7d87de4ab1f590e430821345a40c5d345d449a636d15f/regex-2025.7.34-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea74cf81fe61a7e9d77989050d0089a927ab758c29dac4e8e1b6c06fccf3ebf0", size = 910762, upload-time = "2025-07-31T00:19:43Z" }, + { url = "https://files.pythonhosted.org/packages/37/a8/b05ccf33ceca0815a1e253693b2c86544932ebcc0049c16b0fbdf18b688b/regex-2025.7.34-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e4636a7f3b65a5f340ed9ddf53585c42e3ff37101d383ed321bfe5660481744b", size = 801892, upload-time = "2025-07-31T00:19:44.645Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9a/b993cb2e634cc22810afd1652dba0cae156c40d4864285ff486c73cd1996/regex-2025.7.34-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6cef962d7834437fe8d3da6f9bfc6f93f20f218266dcefec0560ed7765f5fe01", size = 786551, upload-time = "2025-07-31T00:19:46.127Z" }, + { url = "https://files.pythonhosted.org/packages/2d/79/7849d67910a0de4e26834b5bb816e028e35473f3d7ae563552ea04f58ca2/regex-2025.7.34-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:cbe1698e5b80298dbce8df4d8d1182279fbdaf1044e864cbc9d53c20e4a2be77", size = 856457, upload-time = "2025-07-31T00:19:47.562Z" }, + { url = "https://files.pythonhosted.org/packages/91/c6/de516bc082524b27e45cb4f54e28bd800c01efb26d15646a65b87b13a91e/regex-2025.7.34-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:32b9f9bcf0f605eb094b08e8da72e44badabb63dde6b83bd530580b488d1c6da", size = 848902, upload-time = "2025-07-31T00:19:49.312Z" }, + { url = "https://files.pythonhosted.org/packages/7d/22/519ff8ba15f732db099b126f039586bd372da6cd4efb810d5d66a5daeda1/regex-2025.7.34-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:524c868ba527eab4e8744a9287809579f54ae8c62fbf07d62aacd89f6026b282", size = 788038, upload-time = "2025-07-31T00:19:50.794Z" }, + { url = "https://files.pythonhosted.org/packages/3f/7d/aabb467d8f57d8149895d133c88eb809a1a6a0fe262c1d508eb9dfabb6f9/regex-2025.7.34-cp312-cp312-win32.whl", hash = "sha256:d600e58ee6d036081c89696d2bdd55d507498a7180df2e19945c6642fac59588", size = 264417, upload-time = "2025-07-31T00:19:52.292Z" }, + { url = "https://files.pythonhosted.org/packages/3b/39/bd922b55a4fc5ad5c13753274e5b536f5b06ec8eb9747675668491c7ab7a/regex-2025.7.34-cp312-cp312-win_amd64.whl", hash = "sha256:9a9ab52a466a9b4b91564437b36417b76033e8778e5af8f36be835d8cb370d62", size = 275387, upload-time = "2025-07-31T00:19:53.593Z" }, + { url = "https://files.pythonhosted.org/packages/f7/3c/c61d2fdcecb754a40475a3d1ef9a000911d3e3fc75c096acf44b0dfb786a/regex-2025.7.34-cp312-cp312-win_arm64.whl", hash = "sha256:c83aec91af9c6fbf7c743274fd952272403ad9a9db05fe9bfc9df8d12b45f176", size = 268482, upload-time = "2025-07-31T00:19:55.183Z" }, ] [[package]] name = "requests" -version = "2.32.4" +version = "2.32.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, @@ -2843,9 +4411,21 @@ dependencies = [ { name = "idna" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e1/0a/929373653770d8a0d7ea76c37de6e41f11eb07559b103b1c02cafb3f7cf8/requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422", size = 135258, upload-time = "2025-06-09T16:43:07.34Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, +] + +[[package]] +name = "requests-file" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/97/bf44e6c6bd8ddbb99943baf7ba8b1a8485bcd2fe0e55e5708d7fee4ff1ae/requests_file-2.1.0.tar.gz", hash = "sha256:0f549a3f3b0699415ac04d167e9cb39bccfb730cb832b4d20be3d9867356e658", size = 6891, upload-time = "2024-05-21T16:28:00.24Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/e4/56027c4a6b4ae70ca9de302488c5ca95ad4a39e190093d6c1a8ace08341b/requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c", size = 64847, upload-time = "2025-06-09T16:43:05.728Z" }, + { url = "https://files.pythonhosted.org/packages/d7/25/dd878a121fcfdf38f52850f11c512e13ec87c2ea72385933818e5b6c15ce/requests_file-2.1.0-py2.py3-none-any.whl", hash = "sha256:cf270de5a4c5874e84599fc5778303d496c10ae5e870bfa378818f35d21bda5c", size = 4244, upload-time = "2024-05-21T16:27:57.733Z" }, ] [[package]] @@ -2861,28 +4441,28 @@ wheels = [ ] [[package]] -name = "requests-toolbelt" -version = "1.0.0" +name = "requests-gssapi" +version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "gssapi" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", size = 206888, upload-time = "2023-05-01T04:11:33.229Z" } +sdist = { url = "https://files.pythonhosted.org/packages/48/7b/f83ca4b28fbfe122461d8faf27f71ee9141e47473d5520a4f26fa0af04a6/requests_gssapi-1.4.0.tar.gz", hash = "sha256:ba27eb329f4840d965bc8fa5d360c627c74349efa6156ca501ad89afc6a134f4", size = 19090, upload-time = "2025-10-16T04:09:01.402Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, + { url = "https://files.pythonhosted.org/packages/1e/3e/7f76b3a1816ea2557c9d67d52f14f2fb230865a1e5d9e3b28e2fe74fa49d/requests_gssapi-1.4.0-py3-none-any.whl", hash = "sha256:146a1d1e74d7ca0b9d21af0f8d2d1ca259e9228ff57fbbf23f92e25217da4520", size = 12697, upload-time = "2025-10-16T04:08:59.942Z" }, ] [[package]] -name = "responses" -version = "0.18.0" +name = "requests-toolbelt" +version = "1.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "requests" }, - { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/03/a5/186653e51cb20fe3ac793403334d4d077fbb7bb18a9c5c2fce8304d5a2e2/responses-0.18.0.tar.gz", hash = "sha256:380cad4c1c1dc942e5e8a8eaae0b4d4edf708f4f010db8b7bcfafad1fcd254ff", size = 45885, upload-time = "2022-02-02T19:59:52.834Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", size = 206888, upload-time = "2023-05-01T04:11:33.229Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/79/f3/2b3a6dc5986303b3dd1bbbcf482022acb2583c428cd23f0b6d37b1a1a519/responses-0.18.0-py3-none-any.whl", hash = "sha256:15c63ad16de13ee8e7182d99c9334f64fd81f1ee79f90748d527c28f7ca9dd51", size = 38735, upload-time = "2022-02-02T19:59:52.833Z" }, + { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, ] [[package]] @@ -2900,66 +4480,81 @@ wheels = [ [[package]] name = "rpds-py" -version = "0.26.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a5/aa/4456d84bbb54adc6a916fb10c9b374f78ac840337644e4a5eda229c81275/rpds_py-0.26.0.tar.gz", hash = "sha256:20dae58a859b0906f0685642e591056f1e787f3a8b39c8e8749a45dc7d26bdb0", size = 27385, upload-time = "2025-07-01T15:57:13.958Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/09/4c/4ee8f7e512030ff79fda1df3243c88d70fc874634e2dbe5df13ba4210078/rpds_py-0.26.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:9e8cb77286025bdb21be2941d64ac6ca016130bfdcd228739e8ab137eb4406ed", size = 372610, upload-time = "2025-07-01T15:53:58.844Z" }, - { url = "https://files.pythonhosted.org/packages/fa/9d/3dc16be00f14fc1f03c71b1d67c8df98263ab2710a2fbd65a6193214a527/rpds_py-0.26.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e09330b21d98adc8ccb2dbb9fc6cb434e8908d4c119aeaa772cb1caab5440a0", size = 358032, upload-time = "2025-07-01T15:53:59.985Z" }, - { url = "https://files.pythonhosted.org/packages/e7/5a/7f1bf8f045da2866324a08ae80af63e64e7bfaf83bd31f865a7b91a58601/rpds_py-0.26.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c9c1b92b774b2e68d11193dc39620d62fd8ab33f0a3c77ecdabe19c179cdbc1", size = 381525, upload-time = "2025-07-01T15:54:01.162Z" }, - { url = "https://files.pythonhosted.org/packages/45/8a/04479398c755a066ace10e3d158866beb600867cacae194c50ffa783abd0/rpds_py-0.26.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:824e6d3503ab990d7090768e4dfd9e840837bae057f212ff9f4f05ec6d1975e7", size = 397089, upload-time = "2025-07-01T15:54:02.319Z" }, - { url = "https://files.pythonhosted.org/packages/72/88/9203f47268db488a1b6d469d69c12201ede776bb728b9d9f29dbfd7df406/rpds_py-0.26.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8ad7fd2258228bf288f2331f0a6148ad0186b2e3643055ed0db30990e59817a6", size = 514255, upload-time = "2025-07-01T15:54:03.38Z" }, - { url = "https://files.pythonhosted.org/packages/f5/b4/01ce5d1e853ddf81fbbd4311ab1eff0b3cf162d559288d10fd127e2588b5/rpds_py-0.26.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0dc23bbb3e06ec1ea72d515fb572c1fea59695aefbffb106501138762e1e915e", size = 402283, upload-time = "2025-07-01T15:54:04.923Z" }, - { url = "https://files.pythonhosted.org/packages/34/a2/004c99936997bfc644d590a9defd9e9c93f8286568f9c16cdaf3e14429a7/rpds_py-0.26.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d80bf832ac7b1920ee29a426cdca335f96a2b5caa839811803e999b41ba9030d", size = 383881, upload-time = "2025-07-01T15:54:06.482Z" }, - { url = "https://files.pythonhosted.org/packages/05/1b/ef5fba4a8f81ce04c427bfd96223f92f05e6cd72291ce9d7523db3b03a6c/rpds_py-0.26.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0919f38f5542c0a87e7b4afcafab6fd2c15386632d249e9a087498571250abe3", size = 415822, upload-time = "2025-07-01T15:54:07.605Z" }, - { url = "https://files.pythonhosted.org/packages/16/80/5c54195aec456b292f7bd8aa61741c8232964063fd8a75fdde9c1e982328/rpds_py-0.26.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d422b945683e409000c888e384546dbab9009bb92f7c0b456e217988cf316107", size = 558347, upload-time = "2025-07-01T15:54:08.591Z" }, - { url = "https://files.pythonhosted.org/packages/f2/1c/1845c1b1fd6d827187c43afe1841d91678d7241cbdb5420a4c6de180a538/rpds_py-0.26.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:77a7711fa562ba2da1aa757e11024ad6d93bad6ad7ede5afb9af144623e5f76a", size = 587956, upload-time = "2025-07-01T15:54:09.963Z" }, - { url = "https://files.pythonhosted.org/packages/2e/ff/9e979329dd131aa73a438c077252ddabd7df6d1a7ad7b9aacf6261f10faa/rpds_py-0.26.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:238e8c8610cb7c29460e37184f6799547f7e09e6a9bdbdab4e8edb90986a2318", size = 554363, upload-time = "2025-07-01T15:54:11.073Z" }, - { url = "https://files.pythonhosted.org/packages/00/8b/d78cfe034b71ffbe72873a136e71acc7a831a03e37771cfe59f33f6de8a2/rpds_py-0.26.0-cp311-cp311-win32.whl", hash = "sha256:893b022bfbdf26d7bedb083efeea624e8550ca6eb98bf7fea30211ce95b9201a", size = 220123, upload-time = "2025-07-01T15:54:12.382Z" }, - { url = "https://files.pythonhosted.org/packages/94/c1/3c8c94c7dd3905dbfde768381ce98778500a80db9924731d87ddcdb117e9/rpds_py-0.26.0-cp311-cp311-win_amd64.whl", hash = "sha256:87a5531de9f71aceb8af041d72fc4cab4943648d91875ed56d2e629bef6d4c03", size = 231732, upload-time = "2025-07-01T15:54:13.434Z" }, - { url = "https://files.pythonhosted.org/packages/67/93/e936fbed1b734eabf36ccb5d93c6a2e9246fbb13c1da011624b7286fae3e/rpds_py-0.26.0-cp311-cp311-win_arm64.whl", hash = "sha256:de2713f48c1ad57f89ac25b3cb7daed2156d8e822cf0eca9b96a6f990718cc41", size = 221917, upload-time = "2025-07-01T15:54:14.559Z" }, - { url = "https://files.pythonhosted.org/packages/ea/86/90eb87c6f87085868bd077c7a9938006eb1ce19ed4d06944a90d3560fce2/rpds_py-0.26.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:894514d47e012e794f1350f076c427d2347ebf82f9b958d554d12819849a369d", size = 363933, upload-time = "2025-07-01T15:54:15.734Z" }, - { url = "https://files.pythonhosted.org/packages/63/78/4469f24d34636242c924626082b9586f064ada0b5dbb1e9d096ee7a8e0c6/rpds_py-0.26.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc921b96fa95a097add244da36a1d9e4f3039160d1d30f1b35837bf108c21136", size = 350447, upload-time = "2025-07-01T15:54:16.922Z" }, - { url = "https://files.pythonhosted.org/packages/ad/91/c448ed45efdfdade82348d5e7995e15612754826ea640afc20915119734f/rpds_py-0.26.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e1157659470aa42a75448b6e943c895be8c70531c43cb78b9ba990778955582", size = 384711, upload-time = "2025-07-01T15:54:18.101Z" }, - { url = "https://files.pythonhosted.org/packages/ec/43/e5c86fef4be7f49828bdd4ecc8931f0287b1152c0bb0163049b3218740e7/rpds_py-0.26.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:521ccf56f45bb3a791182dc6b88ae5f8fa079dd705ee42138c76deb1238e554e", size = 400865, upload-time = "2025-07-01T15:54:19.295Z" }, - { url = "https://files.pythonhosted.org/packages/55/34/e00f726a4d44f22d5c5fe2e5ddd3ac3d7fd3f74a175607781fbdd06fe375/rpds_py-0.26.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9def736773fd56b305c0eef698be5192c77bfa30d55a0e5885f80126c4831a15", size = 517763, upload-time = "2025-07-01T15:54:20.858Z" }, - { url = "https://files.pythonhosted.org/packages/52/1c/52dc20c31b147af724b16104500fba13e60123ea0334beba7b40e33354b4/rpds_py-0.26.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdad4ea3b4513b475e027be79e5a0ceac8ee1c113a1a11e5edc3c30c29f964d8", size = 406651, upload-time = "2025-07-01T15:54:22.508Z" }, - { url = "https://files.pythonhosted.org/packages/2e/77/87d7bfabfc4e821caa35481a2ff6ae0b73e6a391bb6b343db2c91c2b9844/rpds_py-0.26.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82b165b07f416bdccf5c84546a484cc8f15137ca38325403864bfdf2b5b72f6a", size = 386079, upload-time = "2025-07-01T15:54:23.987Z" }, - { url = "https://files.pythonhosted.org/packages/e3/d4/7f2200c2d3ee145b65b3cddc4310d51f7da6a26634f3ac87125fd789152a/rpds_py-0.26.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d04cab0a54b9dba4d278fe955a1390da3cf71f57feb78ddc7cb67cbe0bd30323", size = 421379, upload-time = "2025-07-01T15:54:25.073Z" }, - { url = "https://files.pythonhosted.org/packages/ae/13/9fdd428b9c820869924ab62236b8688b122baa22d23efdd1c566938a39ba/rpds_py-0.26.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:79061ba1a11b6a12743a2b0f72a46aa2758613d454aa6ba4f5a265cc48850158", size = 562033, upload-time = "2025-07-01T15:54:26.225Z" }, - { url = "https://files.pythonhosted.org/packages/f3/e1/b69686c3bcbe775abac3a4c1c30a164a2076d28df7926041f6c0eb5e8d28/rpds_py-0.26.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:f405c93675d8d4c5ac87364bb38d06c988e11028a64b52a47158a355079661f3", size = 591639, upload-time = "2025-07-01T15:54:27.424Z" }, - { url = "https://files.pythonhosted.org/packages/5c/c9/1e3d8c8863c84a90197ac577bbc3d796a92502124c27092413426f670990/rpds_py-0.26.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dafd4c44b74aa4bed4b250f1aed165b8ef5de743bcca3b88fc9619b6087093d2", size = 557105, upload-time = "2025-07-01T15:54:29.93Z" }, - { url = "https://files.pythonhosted.org/packages/9f/c5/90c569649057622959f6dcc40f7b516539608a414dfd54b8d77e3b201ac0/rpds_py-0.26.0-cp312-cp312-win32.whl", hash = "sha256:3da5852aad63fa0c6f836f3359647870e21ea96cf433eb393ffa45263a170d44", size = 223272, upload-time = "2025-07-01T15:54:31.128Z" }, - { url = "https://files.pythonhosted.org/packages/7d/16/19f5d9f2a556cfed454eebe4d354c38d51c20f3db69e7b4ce6cff904905d/rpds_py-0.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:cf47cfdabc2194a669dcf7a8dbba62e37a04c5041d2125fae0233b720da6f05c", size = 234995, upload-time = "2025-07-01T15:54:32.195Z" }, - { url = "https://files.pythonhosted.org/packages/83/f0/7935e40b529c0e752dfaa7880224771b51175fce08b41ab4a92eb2fbdc7f/rpds_py-0.26.0-cp312-cp312-win_arm64.whl", hash = "sha256:20ab1ae4fa534f73647aad289003f1104092890849e0266271351922ed5574f8", size = 223198, upload-time = "2025-07-01T15:54:33.271Z" }, - { url = "https://files.pythonhosted.org/packages/51/f2/b5c85b758a00c513bb0389f8fc8e61eb5423050c91c958cdd21843faa3e6/rpds_py-0.26.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f61a9326f80ca59214d1cceb0a09bb2ece5b2563d4e0cd37bfd5515c28510674", size = 373505, upload-time = "2025-07-01T15:56:34.716Z" }, - { url = "https://files.pythonhosted.org/packages/23/e0/25db45e391251118e915e541995bb5f5ac5691a3b98fb233020ba53afc9b/rpds_py-0.26.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:183f857a53bcf4b1b42ef0f57ca553ab56bdd170e49d8091e96c51c3d69ca696", size = 359468, upload-time = "2025-07-01T15:56:36.219Z" }, - { url = "https://files.pythonhosted.org/packages/0b/73/dd5ee6075bb6491be3a646b301dfd814f9486d924137a5098e61f0487e16/rpds_py-0.26.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:941c1cfdf4799d623cf3aa1d326a6b4fdb7a5799ee2687f3516738216d2262fb", size = 382680, upload-time = "2025-07-01T15:56:37.644Z" }, - { url = "https://files.pythonhosted.org/packages/2f/10/84b522ff58763a5c443f5bcedc1820240e454ce4e620e88520f04589e2ea/rpds_py-0.26.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72a8d9564a717ee291f554eeb4bfeafe2309d5ec0aa6c475170bdab0f9ee8e88", size = 397035, upload-time = "2025-07-01T15:56:39.241Z" }, - { url = "https://files.pythonhosted.org/packages/06/ea/8667604229a10a520fcbf78b30ccc278977dcc0627beb7ea2c96b3becef0/rpds_py-0.26.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:511d15193cbe013619dd05414c35a7dedf2088fcee93c6bbb7c77859765bd4e8", size = 514922, upload-time = "2025-07-01T15:56:40.645Z" }, - { url = "https://files.pythonhosted.org/packages/24/e6/9ed5b625c0661c4882fc8cdf302bf8e96c73c40de99c31e0b95ed37d508c/rpds_py-0.26.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aea1f9741b603a8d8fedb0ed5502c2bc0accbc51f43e2ad1337fe7259c2b77a5", size = 402822, upload-time = "2025-07-01T15:56:42.137Z" }, - { url = "https://files.pythonhosted.org/packages/8a/58/212c7b6fd51946047fb45d3733da27e2fa8f7384a13457c874186af691b1/rpds_py-0.26.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4019a9d473c708cf2f16415688ef0b4639e07abaa569d72f74745bbeffafa2c7", size = 384336, upload-time = "2025-07-01T15:56:44.239Z" }, - { url = "https://files.pythonhosted.org/packages/aa/f5/a40ba78748ae8ebf4934d4b88e77b98497378bc2c24ba55ebe87a4e87057/rpds_py-0.26.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:093d63b4b0f52d98ebae33b8c50900d3d67e0666094b1be7a12fffd7f65de74b", size = 416871, upload-time = "2025-07-01T15:56:46.284Z" }, - { url = "https://files.pythonhosted.org/packages/d5/a6/33b1fc0c9f7dcfcfc4a4353daa6308b3ece22496ceece348b3e7a7559a09/rpds_py-0.26.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2abe21d8ba64cded53a2a677e149ceb76dcf44284202d737178afe7ba540c1eb", size = 559439, upload-time = "2025-07-01T15:56:48.549Z" }, - { url = "https://files.pythonhosted.org/packages/71/2d/ceb3f9c12f8cfa56d34995097f6cd99da1325642c60d1b6680dd9df03ed8/rpds_py-0.26.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:4feb7511c29f8442cbbc28149a92093d32e815a28aa2c50d333826ad2a20fdf0", size = 588380, upload-time = "2025-07-01T15:56:50.086Z" }, - { url = "https://files.pythonhosted.org/packages/c8/ed/9de62c2150ca8e2e5858acf3f4f4d0d180a38feef9fdab4078bea63d8dba/rpds_py-0.26.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:e99685fc95d386da368013e7fb4269dd39c30d99f812a8372d62f244f662709c", size = 555334, upload-time = "2025-07-01T15:56:51.703Z" }, +version = "0.27.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/d9/991a0dee12d9fc53ed027e26a26a64b151d77252ac477e22666b9688bc16/rpds_py-0.27.0.tar.gz", hash = "sha256:8b23cf252f180cda89220b378d917180f29d313cd6a07b2431c0d3b776aae86f", size = 27420, upload-time = "2025-08-07T08:26:39.624Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/c1/49d515434c1752e40f5e35b985260cf27af052593378580a2f139a5be6b8/rpds_py-0.27.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:dbc2ab5d10544eb485baa76c63c501303b716a5c405ff2469a1d8ceffaabf622", size = 371577, upload-time = "2025-08-07T08:23:25.379Z" }, + { url = "https://files.pythonhosted.org/packages/e1/6d/bf2715b2fee5087fa13b752b5fd573f1a93e4134c74d275f709e38e54fe7/rpds_py-0.27.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7ec85994f96a58cf7ed288caa344b7fe31fd1d503bdf13d7331ead5f70ab60d5", size = 354959, upload-time = "2025-08-07T08:23:26.767Z" }, + { url = "https://files.pythonhosted.org/packages/a3/5c/e7762808c746dd19733a81373c10da43926f6a6adcf4920a21119697a60a/rpds_py-0.27.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:190d7285cd3bb6d31d37a0534d7359c1ee191eb194c511c301f32a4afa5a1dd4", size = 381485, upload-time = "2025-08-07T08:23:27.869Z" }, + { url = "https://files.pythonhosted.org/packages/40/51/0d308eb0b558309ca0598bcba4243f52c4cd20e15fe991b5bd75824f2e61/rpds_py-0.27.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c10d92fb6d7fd827e44055fcd932ad93dac6a11e832d51534d77b97d1d85400f", size = 396816, upload-time = "2025-08-07T08:23:29.424Z" }, + { url = "https://files.pythonhosted.org/packages/5c/aa/2d585ec911d78f66458b2c91252134ca0c7c70f687a72c87283173dc0c96/rpds_py-0.27.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dd2c1d27ebfe6a015cfa2005b7fe8c52d5019f7bbdd801bc6f7499aab9ae739e", size = 514950, upload-time = "2025-08-07T08:23:30.576Z" }, + { url = "https://files.pythonhosted.org/packages/0b/ef/aced551cc1148179557aed84343073adadf252c91265263ee6203458a186/rpds_py-0.27.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4790c9d5dd565ddb3e9f656092f57268951398cef52e364c405ed3112dc7c7c1", size = 402132, upload-time = "2025-08-07T08:23:32.428Z" }, + { url = "https://files.pythonhosted.org/packages/4b/ac/cf644803d8d417653fe2b3604186861d62ea6afaef1b2284045741baef17/rpds_py-0.27.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4300e15e7d03660f04be84a125d1bdd0e6b2f674bc0723bc0fd0122f1a4585dc", size = 383660, upload-time = "2025-08-07T08:23:33.829Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ec/caf47c55ce02b76cbaeeb2d3b36a73da9ca2e14324e3d75cf72b59dcdac5/rpds_py-0.27.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:59195dc244fc183209cf8a93406889cadde47dfd2f0a6b137783aa9c56d67c85", size = 401730, upload-time = "2025-08-07T08:23:34.97Z" }, + { url = "https://files.pythonhosted.org/packages/0b/71/c1f355afdcd5b99ffc253422aa4bdcb04ccf1491dcd1bda3688a0c07fd61/rpds_py-0.27.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fae4a01ef8c4cb2bbe92ef2063149596907dc4a881a8d26743b3f6b304713171", size = 416122, upload-time = "2025-08-07T08:23:36.062Z" }, + { url = "https://files.pythonhosted.org/packages/38/0f/f4b5b1eda724ed0e04d2b26d8911cdc131451a7ee4c4c020a1387e5c6ded/rpds_py-0.27.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e3dc8d4ede2dbae6c0fc2b6c958bf51ce9fd7e9b40c0f5b8835c3fde44f5807d", size = 558771, upload-time = "2025-08-07T08:23:37.478Z" }, + { url = "https://files.pythonhosted.org/packages/93/c0/5f8b834db2289ab48d5cffbecbb75e35410103a77ac0b8da36bf9544ec1c/rpds_py-0.27.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c3782fb753aa825b4ccabc04292e07897e2fd941448eabf666856c5530277626", size = 587876, upload-time = "2025-08-07T08:23:38.662Z" }, + { url = "https://files.pythonhosted.org/packages/d2/dd/1a1df02ab8eb970115cff2ae31a6f73916609b900dc86961dc382b8c2e5e/rpds_py-0.27.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:887ab1f12b0d227e9260558a4a2320024b20102207ada65c43e1ffc4546df72e", size = 554359, upload-time = "2025-08-07T08:23:39.897Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e4/95a014ab0d51ab6e3bebbdb476a42d992d2bbf9c489d24cff9fda998e925/rpds_py-0.27.0-cp311-cp311-win32.whl", hash = "sha256:5d6790ff400254137b81b8053b34417e2c46921e302d655181d55ea46df58cf7", size = 218084, upload-time = "2025-08-07T08:23:41.086Z" }, + { url = "https://files.pythonhosted.org/packages/49/78/f8d5b71ec65a0376b0de31efcbb5528ce17a9b7fdd19c3763303ccfdedec/rpds_py-0.27.0-cp311-cp311-win_amd64.whl", hash = "sha256:e24d8031a2c62f34853756d9208eeafa6b940a1efcbfe36e8f57d99d52bb7261", size = 230085, upload-time = "2025-08-07T08:23:42.143Z" }, + { url = "https://files.pythonhosted.org/packages/e7/d3/84429745184091e06b4cc70f8597408e314c2d2f7f5e13249af9ffab9e3d/rpds_py-0.27.0-cp311-cp311-win_arm64.whl", hash = "sha256:08680820d23df1df0a0260f714d12966bc6c42d02e8055a91d61e03f0c47dda0", size = 222112, upload-time = "2025-08-07T08:23:43.233Z" }, + { url = "https://files.pythonhosted.org/packages/cd/17/e67309ca1ac993fa1888a0d9b2f5ccc1f67196ace32e76c9f8e1dbbbd50c/rpds_py-0.27.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:19c990fdf5acecbf0623e906ae2e09ce1c58947197f9bced6bbd7482662231c4", size = 362611, upload-time = "2025-08-07T08:23:44.773Z" }, + { url = "https://files.pythonhosted.org/packages/93/2e/28c2fb84aa7aa5d75933d1862d0f7de6198ea22dfd9a0cca06e8a4e7509e/rpds_py-0.27.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6c27a7054b5224710fcfb1a626ec3ff4f28bcb89b899148c72873b18210e446b", size = 347680, upload-time = "2025-08-07T08:23:46.014Z" }, + { url = "https://files.pythonhosted.org/packages/44/3e/9834b4c8f4f5fe936b479e623832468aa4bd6beb8d014fecaee9eac6cdb1/rpds_py-0.27.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09965b314091829b378b60607022048953e25f0b396c2b70e7c4c81bcecf932e", size = 384600, upload-time = "2025-08-07T08:23:48Z" }, + { url = "https://files.pythonhosted.org/packages/19/78/744123c7b38865a965cd9e6f691fde7ef989a00a256fa8bf15b75240d12f/rpds_py-0.27.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:14f028eb47f59e9169bfdf9f7ceafd29dd64902141840633683d0bad5b04ff34", size = 400697, upload-time = "2025-08-07T08:23:49.407Z" }, + { url = "https://files.pythonhosted.org/packages/32/97/3c3d32fe7daee0a1f1a678b6d4dfb8c4dcf88197fa2441f9da7cb54a8466/rpds_py-0.27.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6168af0be75bba990a39f9431cdfae5f0ad501f4af32ae62e8856307200517b8", size = 517781, upload-time = "2025-08-07T08:23:50.557Z" }, + { url = "https://files.pythonhosted.org/packages/b2/be/28f0e3e733680aa13ecec1212fc0f585928a206292f14f89c0b8a684cad1/rpds_py-0.27.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ab47fe727c13c09d0e6f508e3a49e545008e23bf762a245b020391b621f5b726", size = 406449, upload-time = "2025-08-07T08:23:51.732Z" }, + { url = "https://files.pythonhosted.org/packages/95/ae/5d15c83e337c082d0367053baeb40bfba683f42459f6ebff63a2fd7e5518/rpds_py-0.27.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fa01b3d5e3b7d97efab65bd3d88f164e289ec323a8c033c5c38e53ee25c007e", size = 386150, upload-time = "2025-08-07T08:23:52.822Z" }, + { url = "https://files.pythonhosted.org/packages/bf/65/944e95f95d5931112829e040912b25a77b2e7ed913ea5fe5746aa5c1ce75/rpds_py-0.27.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:6c135708e987f46053e0a1246a206f53717f9fadfba27174a9769ad4befba5c3", size = 406100, upload-time = "2025-08-07T08:23:54.339Z" }, + { url = "https://files.pythonhosted.org/packages/21/a4/1664b83fae02894533cd11dc0b9f91d673797c2185b7be0f7496107ed6c5/rpds_py-0.27.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc327f4497b7087d06204235199daf208fd01c82d80465dc5efa4ec9df1c5b4e", size = 421345, upload-time = "2025-08-07T08:23:55.832Z" }, + { url = "https://files.pythonhosted.org/packages/7c/26/b7303941c2b0823bfb34c71378249f8beedce57301f400acb04bb345d025/rpds_py-0.27.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e57906e38583a2cba67046a09c2637e23297618dc1f3caddbc493f2be97c93f", size = 561891, upload-time = "2025-08-07T08:23:56.951Z" }, + { url = "https://files.pythonhosted.org/packages/9b/c8/48623d64d4a5a028fa99576c768a6159db49ab907230edddc0b8468b998b/rpds_py-0.27.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f4f69d7a4300fbf91efb1fb4916421bd57804c01ab938ab50ac9c4aa2212f03", size = 591756, upload-time = "2025-08-07T08:23:58.146Z" }, + { url = "https://files.pythonhosted.org/packages/b3/51/18f62617e8e61cc66334c9fb44b1ad7baae3438662098efbc55fb3fda453/rpds_py-0.27.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b4c4fbbcff474e1e5f38be1bf04511c03d492d42eec0babda5d03af3b5589374", size = 557088, upload-time = "2025-08-07T08:23:59.6Z" }, + { url = "https://files.pythonhosted.org/packages/bd/4c/e84c3a276e2496a93d245516be6b49e20499aa8ca1c94d59fada0d79addc/rpds_py-0.27.0-cp312-cp312-win32.whl", hash = "sha256:27bac29bbbf39601b2aab474daf99dbc8e7176ca3389237a23944b17f8913d97", size = 221926, upload-time = "2025-08-07T08:24:00.695Z" }, + { url = "https://files.pythonhosted.org/packages/83/89/9d0fbcef64340db0605eb0a0044f258076f3ae0a3b108983b2c614d96212/rpds_py-0.27.0-cp312-cp312-win_amd64.whl", hash = "sha256:8a06aa1197ec0281eb1d7daf6073e199eb832fe591ffa329b88bae28f25f5fe5", size = 233235, upload-time = "2025-08-07T08:24:01.846Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b0/e177aa9f39cbab060f96de4a09df77d494f0279604dc2f509263e21b05f9/rpds_py-0.27.0-cp312-cp312-win_arm64.whl", hash = "sha256:e14aab02258cb776a108107bd15f5b5e4a1bbaa61ef33b36693dfab6f89d54f9", size = 223315, upload-time = "2025-08-07T08:24:03.337Z" }, + { url = "https://files.pythonhosted.org/packages/59/64/72ab5b911fdcc48058359b0e786e5363e3fde885156116026f1a2ba9a5b5/rpds_py-0.27.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e6491658dd2569f05860bad645569145c8626ac231877b0fb2d5f9bcb7054089", size = 371658, upload-time = "2025-08-07T08:26:02.369Z" }, + { url = "https://files.pythonhosted.org/packages/6c/4b/90ff04b4da055db53d8fea57640d8d5d55456343a1ec9a866c0ecfe10fd1/rpds_py-0.27.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:bec77545d188f8bdd29d42bccb9191682a46fb2e655e3d1fb446d47c55ac3b8d", size = 355529, upload-time = "2025-08-07T08:26:03.83Z" }, + { url = "https://files.pythonhosted.org/packages/a4/be/527491fb1afcd86fc5ce5812eb37bc70428ee017d77fee20de18155c3937/rpds_py-0.27.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25a4aebf8ca02bbb90a9b3e7a463bbf3bee02ab1c446840ca07b1695a68ce424", size = 382822, upload-time = "2025-08-07T08:26:05.52Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a5/dcdb8725ce11e6d0913e6fcf782a13f4b8a517e8acc70946031830b98441/rpds_py-0.27.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:44524b96481a4c9b8e6c46d6afe43fa1fb485c261e359fbe32b63ff60e3884d8", size = 397233, upload-time = "2025-08-07T08:26:07.179Z" }, + { url = "https://files.pythonhosted.org/packages/33/f9/0947920d1927e9f144660590cc38cadb0795d78fe0d9aae0ef71c1513b7c/rpds_py-0.27.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:45d04a73c54b6a5fd2bab91a4b5bc8b426949586e61340e212a8484919183859", size = 514892, upload-time = "2025-08-07T08:26:08.622Z" }, + { url = "https://files.pythonhosted.org/packages/1d/ed/d1343398c1417c68f8daa1afce56ef6ce5cc587daaf98e29347b00a80ff2/rpds_py-0.27.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:343cf24de9ed6c728abefc5d5c851d5de06497caa7ac37e5e65dd572921ed1b5", size = 402733, upload-time = "2025-08-07T08:26:10.433Z" }, + { url = "https://files.pythonhosted.org/packages/1d/0b/646f55442cd14014fb64d143428f25667a100f82092c90087b9ea7101c74/rpds_py-0.27.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7aed8118ae20515974650d08eb724150dc2e20c2814bcc307089569995e88a14", size = 384447, upload-time = "2025-08-07T08:26:11.847Z" }, + { url = "https://files.pythonhosted.org/packages/4b/15/0596ef7529828e33a6c81ecf5013d1dd33a511a3e0be0561f83079cda227/rpds_py-0.27.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:af9d4fd79ee1cc8e7caf693ee02737daabfc0fcf2773ca0a4735b356c8ad6f7c", size = 402502, upload-time = "2025-08-07T08:26:13.537Z" }, + { url = "https://files.pythonhosted.org/packages/c3/8d/986af3c42f8454a6cafff8729d99fb178ae9b08a9816325ac7a8fa57c0c0/rpds_py-0.27.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f0396e894bd1e66c74ecbc08b4f6a03dc331140942c4b1d345dd131b68574a60", size = 416651, upload-time = "2025-08-07T08:26:14.923Z" }, + { url = "https://files.pythonhosted.org/packages/e9/9a/b4ec3629b7b447e896eec574469159b5b60b7781d3711c914748bf32de05/rpds_py-0.27.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:59714ab0a5af25d723d8e9816638faf7f4254234decb7d212715c1aa71eee7be", size = 559460, upload-time = "2025-08-07T08:26:16.295Z" }, + { url = "https://files.pythonhosted.org/packages/61/63/d1e127b40c3e4733b3a6f26ae7a063cdf2bc1caa5272c89075425c7d397a/rpds_py-0.27.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:88051c3b7d5325409f433c5a40328fcb0685fc04e5db49ff936e910901d10114", size = 588072, upload-time = "2025-08-07T08:26:17.776Z" }, + { url = "https://files.pythonhosted.org/packages/04/7e/8ffc71a8f6833d9c9fb999f5b0ee736b8b159fd66968e05c7afc2dbcd57e/rpds_py-0.27.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:181bc29e59e5e5e6e9d63b143ff4d5191224d355e246b5a48c88ce6b35c4e466", size = 555083, upload-time = "2025-08-07T08:26:19.301Z" }, +] + +[[package]] +name = "rsa" +version = "4.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/8a/22b7beea3ee0d44b1916c0c1cb0ee3af23b700b6da9f04991899d0c555d4/rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75", size = 29034, upload-time = "2025-04-16T09:51:18.218Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", size = 34696, upload-time = "2025-04-16T09:51:17.142Z" }, ] [[package]] name = "s3transfer" -version = "0.13.0" +version = "0.13.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ed/5d/9dcc100abc6711e8247af5aa561fc07c4a046f72f659c3adea9a449e191a/s3transfer-0.13.0.tar.gz", hash = "sha256:f5e6db74eb7776a37208001113ea7aa97695368242b364d73e91c981ac522177", size = 150232, upload-time = "2025-05-22T19:24:50.245Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/05/d52bf1e65044b4e5e27d4e63e8d1579dbdec54fce685908ae09bc3720030/s3transfer-0.13.1.tar.gz", hash = "sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf", size = 150589, upload-time = "2025-07-18T19:22:42.31Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/18/17/22bf8155aa0ea2305eefa3a6402e040df7ebe512d1310165eda1e233c3f8/s3transfer-0.13.0-py3-none-any.whl", hash = "sha256:0148ef34d6dd964d0d8cf4311b2b21c474693e57c2e069ec708ce043d2b527be", size = 85152, upload-time = "2025-05-22T19:24:48.703Z" }, + { url = "https://files.pythonhosted.org/packages/6d/4f/d073e09df851cfa251ef7840007d04db3293a0482ce607d2b993926089be/s3transfer-0.13.1-py3-none-any.whl", hash = "sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724", size = 85308, upload-time = "2025-07-18T19:22:40.947Z" }, ] [[package]] name = "scikit-learn" -version = "1.7.0" +version = "1.7.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "joblib" }, @@ -2967,47 +4562,47 @@ dependencies = [ { name = "scipy" }, { name = "threadpoolctl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/df/3b/29fa87e76b1d7b3b77cc1fcbe82e6e6b8cd704410705b008822de530277c/scikit_learn-1.7.0.tar.gz", hash = "sha256:c01e869b15aec88e2cdb73d27f15bdbe03bce8e2fb43afbe77c45d399e73a5a3", size = 7178217, upload-time = "2025-06-05T22:02:46.703Z" } +sdist = { url = "https://files.pythonhosted.org/packages/41/84/5f4af978fff619706b8961accac84780a6d298d82a8873446f72edb4ead0/scikit_learn-1.7.1.tar.gz", hash = "sha256:24b3f1e976a4665aa74ee0fcaac2b8fccc6ae77c8e07ab25da3ba6d3292b9802", size = 7190445, upload-time = "2025-07-18T08:01:54.5Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/42/c6b41711c2bee01c4800ad8da2862c0b6d2956a399d23ce4d77f2ca7f0c7/scikit_learn-1.7.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8ef09b1615e1ad04dc0d0054ad50634514818a8eb3ee3dee99af3bffc0ef5007", size = 11719657, upload-time = "2025-06-05T22:01:56.345Z" }, - { url = "https://files.pythonhosted.org/packages/a3/24/44acca76449e391b6b2522e67a63c0454b7c1f060531bdc6d0118fb40851/scikit_learn-1.7.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:7d7240c7b19edf6ed93403f43b0fcb0fe95b53bc0b17821f8fb88edab97085ef", size = 10712636, upload-time = "2025-06-05T22:01:59.093Z" }, - { url = "https://files.pythonhosted.org/packages/9f/1b/fcad1ccb29bdc9b96bcaa2ed8345d56afb77b16c0c47bafe392cc5d1d213/scikit_learn-1.7.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:80bd3bd4e95381efc47073a720d4cbab485fc483966f1709f1fd559afac57ab8", size = 12242817, upload-time = "2025-06-05T22:02:01.43Z" }, - { url = "https://files.pythonhosted.org/packages/c6/38/48b75c3d8d268a3f19837cb8a89155ead6e97c6892bb64837183ea41db2b/scikit_learn-1.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9dbe48d69aa38ecfc5a6cda6c5df5abef0c0ebdb2468e92437e2053f84abb8bc", size = 12873961, upload-time = "2025-06-05T22:02:03.951Z" }, - { url = "https://files.pythonhosted.org/packages/f4/5a/ba91b8c57aa37dbd80d5ff958576a9a8c14317b04b671ae7f0d09b00993a/scikit_learn-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:8fa979313b2ffdfa049ed07252dc94038def3ecd49ea2a814db5401c07f1ecfa", size = 10717277, upload-time = "2025-06-05T22:02:06.77Z" }, - { url = "https://files.pythonhosted.org/packages/70/3a/bffab14e974a665a3ee2d79766e7389572ffcaad941a246931c824afcdb2/scikit_learn-1.7.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c2c7243d34aaede0efca7a5a96d67fddaebb4ad7e14a70991b9abee9dc5c0379", size = 11646758, upload-time = "2025-06-05T22:02:09.51Z" }, - { url = "https://files.pythonhosted.org/packages/58/d8/f3249232fa79a70cb40595282813e61453c1e76da3e1a44b77a63dd8d0cb/scikit_learn-1.7.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:9f39f6a811bf3f15177b66c82cbe0d7b1ebad9f190737dcdef77cfca1ea3c19c", size = 10673971, upload-time = "2025-06-05T22:02:12.217Z" }, - { url = "https://files.pythonhosted.org/packages/67/93/eb14c50533bea2f77758abe7d60a10057e5f2e2cdcf0a75a14c6bc19c734/scikit_learn-1.7.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63017a5f9a74963d24aac7590287149a8d0f1a0799bbe7173c0d8ba1523293c0", size = 11818428, upload-time = "2025-06-05T22:02:14.947Z" }, - { url = "https://files.pythonhosted.org/packages/08/17/804cc13b22a8663564bb0b55fb89e661a577e4e88a61a39740d58b909efe/scikit_learn-1.7.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b2f8a0b1e73e9a08b7cc498bb2aeab36cdc1f571f8ab2b35c6e5d1c7115d97d", size = 12505887, upload-time = "2025-06-05T22:02:17.824Z" }, - { url = "https://files.pythonhosted.org/packages/68/c7/4e956281a077f4835458c3f9656c666300282d5199039f26d9de1dabd9be/scikit_learn-1.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:34cc8d9d010d29fb2b7cbcd5ccc24ffdd80515f65fe9f1e4894ace36b267ce19", size = 10668129, upload-time = "2025-06-05T22:02:20.536Z" }, + { url = "https://files.pythonhosted.org/packages/b4/bd/a23177930abd81b96daffa30ef9c54ddbf544d3226b8788ce4c3ef1067b4/scikit_learn-1.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:90c8494ea23e24c0fb371afc474618c1019dc152ce4a10e4607e62196113851b", size = 9334838, upload-time = "2025-07-18T08:01:11.239Z" }, + { url = "https://files.pythonhosted.org/packages/8d/a1/d3a7628630a711e2ac0d1a482910da174b629f44e7dd8cfcd6924a4ef81a/scikit_learn-1.7.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:bb870c0daf3bf3be145ec51df8ac84720d9972170786601039f024bf6d61a518", size = 8651241, upload-time = "2025-07-18T08:01:13.234Z" }, + { url = "https://files.pythonhosted.org/packages/26/92/85ec172418f39474c1cd0221d611345d4f433fc4ee2fc68e01f524ccc4e4/scikit_learn-1.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:40daccd1b5623f39e8943ab39735cadf0bdce80e67cdca2adcb5426e987320a8", size = 9718677, upload-time = "2025-07-18T08:01:15.649Z" }, + { url = "https://files.pythonhosted.org/packages/df/ce/abdb1dcbb1d2b66168ec43b23ee0cee356b4cc4100ddee3943934ebf1480/scikit_learn-1.7.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30d1f413cfc0aa5a99132a554f1d80517563c34a9d3e7c118fde2d273c6fe0f7", size = 9511189, upload-time = "2025-07-18T08:01:18.013Z" }, + { url = "https://files.pythonhosted.org/packages/b2/3b/47b5eaee01ef2b5a80ba3f7f6ecf79587cb458690857d4777bfd77371c6f/scikit_learn-1.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:c711d652829a1805a95d7fe96654604a8f16eab5a9e9ad87b3e60173415cb650", size = 8914794, upload-time = "2025-07-18T08:01:20.357Z" }, + { url = "https://files.pythonhosted.org/packages/cb/16/57f176585b35ed865f51b04117947fe20f130f78940c6477b6d66279c9c2/scikit_learn-1.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3cee419b49b5bbae8796ecd690f97aa412ef1674410c23fc3257c6b8b85b8087", size = 9260431, upload-time = "2025-07-18T08:01:22.77Z" }, + { url = "https://files.pythonhosted.org/packages/67/4e/899317092f5efcab0e9bc929e3391341cec8fb0e816c4789686770024580/scikit_learn-1.7.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:2fd8b8d35817b0d9ebf0b576f7d5ffbbabdb55536b0655a8aaae629d7ffd2e1f", size = 8637191, upload-time = "2025-07-18T08:01:24.731Z" }, + { url = "https://files.pythonhosted.org/packages/f3/1b/998312db6d361ded1dd56b457ada371a8d8d77ca2195a7d18fd8a1736f21/scikit_learn-1.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:588410fa19a96a69763202f1d6b7b91d5d7a5d73be36e189bc6396bfb355bd87", size = 9486346, upload-time = "2025-07-18T08:01:26.713Z" }, + { url = "https://files.pythonhosted.org/packages/ad/09/a2aa0b4e644e5c4ede7006748f24e72863ba2ae71897fecfd832afea01b4/scikit_learn-1.7.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e3142f0abe1ad1d1c31a2ae987621e41f6b578144a911ff4ac94781a583adad7", size = 9290988, upload-time = "2025-07-18T08:01:28.938Z" }, + { url = "https://files.pythonhosted.org/packages/15/fa/c61a787e35f05f17fc10523f567677ec4eeee5f95aa4798dbbbcd9625617/scikit_learn-1.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:3ddd9092c1bd469acab337d87930067c87eac6bd544f8d5027430983f1e1ae88", size = 8735568, upload-time = "2025-07-18T08:01:30.936Z" }, ] [[package]] name = "scipy" -version = "1.16.0" +version = "1.16.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/81/18/b06a83f0c5ee8cddbde5e3f3d0bb9b702abfa5136ef6d4620ff67df7eee5/scipy-1.16.0.tar.gz", hash = "sha256:b5ef54021e832869c8cfb03bc3bf20366cbcd426e02a58e8a58d7584dfbb8f62", size = 30581216, upload-time = "2025-06-22T16:27:55.782Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/f8/53fc4884df6b88afd5f5f00240bdc49fee2999c7eff3acf5953eb15bc6f8/scipy-1.16.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:deec06d831b8f6b5fb0b652433be6a09db29e996368ce5911faf673e78d20085", size = 36447362, upload-time = "2025-06-22T16:18:17.817Z" }, - { url = "https://files.pythonhosted.org/packages/c9/25/fad8aa228fa828705142a275fc593d701b1817c98361a2d6b526167d07bc/scipy-1.16.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:d30c0fe579bb901c61ab4bb7f3eeb7281f0d4c4a7b52dbf563c89da4fd2949be", size = 28547120, upload-time = "2025-06-22T16:18:24.117Z" }, - { url = "https://files.pythonhosted.org/packages/8d/be/d324ddf6b89fd1c32fecc307f04d095ce84abb52d2e88fab29d0cd8dc7a8/scipy-1.16.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:b2243561b45257f7391d0f49972fca90d46b79b8dbcb9b2cb0f9df928d370ad4", size = 20818922, upload-time = "2025-06-22T16:18:28.035Z" }, - { url = "https://files.pythonhosted.org/packages/cd/e0/cf3f39e399ac83fd0f3ba81ccc5438baba7cfe02176be0da55ff3396f126/scipy-1.16.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:e6d7dfc148135e9712d87c5f7e4f2ddc1304d1582cb3a7d698bbadedb61c7afd", size = 23409695, upload-time = "2025-06-22T16:18:32.497Z" }, - { url = "https://files.pythonhosted.org/packages/5b/61/d92714489c511d3ffd6830ac0eb7f74f243679119eed8b9048e56b9525a1/scipy-1.16.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:90452f6a9f3fe5a2cf3748e7be14f9cc7d9b124dce19667b54f5b429d680d539", size = 33444586, upload-time = "2025-06-22T16:18:37.992Z" }, - { url = "https://files.pythonhosted.org/packages/af/2c/40108915fd340c830aee332bb85a9160f99e90893e58008b659b9f3dddc0/scipy-1.16.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a2f0bf2f58031c8701a8b601df41701d2a7be17c7ffac0a4816aeba89c4cdac8", size = 35284126, upload-time = "2025-06-22T16:18:43.605Z" }, - { url = "https://files.pythonhosted.org/packages/d3/30/e9eb0ad3d0858df35d6c703cba0a7e16a18a56a9e6b211d861fc6f261c5f/scipy-1.16.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6c4abb4c11fc0b857474241b812ce69ffa6464b4bd8f4ecb786cf240367a36a7", size = 35608257, upload-time = "2025-06-22T16:18:49.09Z" }, - { url = "https://files.pythonhosted.org/packages/c8/ff/950ee3e0d612b375110d8cda211c1f787764b4c75e418a4b71f4a5b1e07f/scipy-1.16.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b370f8f6ac6ef99815b0d5c9f02e7ade77b33007d74802efc8316c8db98fd11e", size = 38040541, upload-time = "2025-06-22T16:18:55.077Z" }, - { url = "https://files.pythonhosted.org/packages/8b/c9/750d34788288d64ffbc94fdb4562f40f609d3f5ef27ab4f3a4ad00c9033e/scipy-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:a16ba90847249bedce8aa404a83fb8334b825ec4a8e742ce6012a7a5e639f95c", size = 38570814, upload-time = "2025-06-22T16:19:00.912Z" }, - { url = "https://files.pythonhosted.org/packages/01/c0/c943bc8d2bbd28123ad0f4f1eef62525fa1723e84d136b32965dcb6bad3a/scipy-1.16.0-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:7eb6bd33cef4afb9fa5f1fb25df8feeb1e52d94f21a44f1d17805b41b1da3180", size = 36459071, upload-time = "2025-06-22T16:19:06.605Z" }, - { url = "https://files.pythonhosted.org/packages/99/0d/270e2e9f1a4db6ffbf84c9a0b648499842046e4e0d9b2275d150711b3aba/scipy-1.16.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:1dbc8fdba23e4d80394ddfab7a56808e3e6489176d559c6c71935b11a2d59db1", size = 28490500, upload-time = "2025-06-22T16:19:11.775Z" }, - { url = "https://files.pythonhosted.org/packages/1c/22/01d7ddb07cff937d4326198ec8d10831367a708c3da72dfd9b7ceaf13028/scipy-1.16.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:7dcf42c380e1e3737b343dec21095c9a9ad3f9cbe06f9c05830b44b1786c9e90", size = 20762345, upload-time = "2025-06-22T16:19:15.813Z" }, - { url = "https://files.pythonhosted.org/packages/34/7f/87fd69856569ccdd2a5873fe5d7b5bbf2ad9289d7311d6a3605ebde3a94b/scipy-1.16.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:26ec28675f4a9d41587266084c626b02899db373717d9312fa96ab17ca1ae94d", size = 23418563, upload-time = "2025-06-22T16:19:20.746Z" }, - { url = "https://files.pythonhosted.org/packages/f6/f1/e4f4324fef7f54160ab749efbab6a4bf43678a9eb2e9817ed71a0a2fd8de/scipy-1.16.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:952358b7e58bd3197cfbd2f2f2ba829f258404bdf5db59514b515a8fe7a36c52", size = 33203951, upload-time = "2025-06-22T16:19:25.813Z" }, - { url = "https://files.pythonhosted.org/packages/6d/f0/b6ac354a956384fd8abee2debbb624648125b298f2c4a7b4f0d6248048a5/scipy-1.16.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:03931b4e870c6fef5b5c0970d52c9f6ddd8c8d3e934a98f09308377eba6f3824", size = 35070225, upload-time = "2025-06-22T16:19:31.416Z" }, - { url = "https://files.pythonhosted.org/packages/e5/73/5cbe4a3fd4bc3e2d67ffad02c88b83edc88f381b73ab982f48f3df1a7790/scipy-1.16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:512c4f4f85912767c351a0306824ccca6fd91307a9f4318efe8fdbd9d30562ef", size = 35389070, upload-time = "2025-06-22T16:19:37.387Z" }, - { url = "https://files.pythonhosted.org/packages/86/e8/a60da80ab9ed68b31ea5a9c6dfd3c2f199347429f229bf7f939a90d96383/scipy-1.16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e69f798847e9add03d512eaf5081a9a5c9a98757d12e52e6186ed9681247a1ac", size = 37825287, upload-time = "2025-06-22T16:19:43.375Z" }, - { url = "https://files.pythonhosted.org/packages/ea/b5/29fece1a74c6a94247f8a6fb93f5b28b533338e9c34fdcc9cfe7a939a767/scipy-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:adf9b1999323ba335adc5d1dc7add4781cb5a4b0ef1e98b79768c05c796c4e49", size = 38431929, upload-time = "2025-06-22T16:19:49.385Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/f5/4a/b927028464795439faec8eaf0b03b011005c487bb2d07409f28bf30879c4/scipy-1.16.1.tar.gz", hash = "sha256:44c76f9e8b6e8e488a586190ab38016e4ed2f8a038af7cd3defa903c0a2238b3", size = 30580861, upload-time = "2025-07-27T16:33:30.834Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/91/812adc6f74409b461e3a5fa97f4f74c769016919203138a3bf6fc24ba4c5/scipy-1.16.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:c033fa32bab91dc98ca59d0cf23bb876454e2bb02cbe592d5023138778f70030", size = 36552519, upload-time = "2025-07-27T16:26:29.658Z" }, + { url = "https://files.pythonhosted.org/packages/47/18/8e355edcf3b71418d9e9f9acd2708cc3a6c27e8f98fde0ac34b8a0b45407/scipy-1.16.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:6e5c2f74e5df33479b5cd4e97a9104c511518fbd979aa9b8f6aec18b2e9ecae7", size = 28638010, upload-time = "2025-07-27T16:26:38.196Z" }, + { url = "https://files.pythonhosted.org/packages/d9/eb/e931853058607bdfbc11b86df19ae7a08686121c203483f62f1ecae5989c/scipy-1.16.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:0a55ffe0ba0f59666e90951971a884d1ff6f4ec3275a48f472cfb64175570f77", size = 20909790, upload-time = "2025-07-27T16:26:43.93Z" }, + { url = "https://files.pythonhosted.org/packages/45/0c/be83a271d6e96750cd0be2e000f35ff18880a46f05ce8b5d3465dc0f7a2a/scipy-1.16.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:f8a5d6cd147acecc2603fbd382fed6c46f474cccfcf69ea32582e033fb54dcfe", size = 23513352, upload-time = "2025-07-27T16:26:50.017Z" }, + { url = "https://files.pythonhosted.org/packages/7c/bf/fe6eb47e74f762f933cca962db7f2c7183acfdc4483bd1c3813cfe83e538/scipy-1.16.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb18899127278058bcc09e7b9966d41a5a43740b5bb8dcba401bd983f82e885b", size = 33534643, upload-time = "2025-07-27T16:26:57.503Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ba/63f402e74875486b87ec6506a4f93f6d8a0d94d10467280f3d9d7837ce3a/scipy-1.16.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adccd93a2fa937a27aae826d33e3bfa5edf9aa672376a4852d23a7cd67a2e5b7", size = 35376776, upload-time = "2025-07-27T16:27:06.639Z" }, + { url = "https://files.pythonhosted.org/packages/c3/b4/04eb9d39ec26a1b939689102da23d505ea16cdae3dbb18ffc53d1f831044/scipy-1.16.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:18aca1646a29ee9a0625a1be5637fa798d4d81fdf426481f06d69af828f16958", size = 35698906, upload-time = "2025-07-27T16:27:14.943Z" }, + { url = "https://files.pythonhosted.org/packages/04/d6/bb5468da53321baeb001f6e4e0d9049eadd175a4a497709939128556e3ec/scipy-1.16.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d85495cef541729a70cdddbbf3e6b903421bc1af3e8e3a9a72a06751f33b7c39", size = 38129275, upload-time = "2025-07-27T16:27:23.873Z" }, + { url = "https://files.pythonhosted.org/packages/c4/94/994369978509f227cba7dfb9e623254d0d5559506fe994aef4bea3ed469c/scipy-1.16.1-cp311-cp311-win_amd64.whl", hash = "sha256:226652fca853008119c03a8ce71ffe1b3f6d2844cc1686e8f9806edafae68596", size = 38644572, upload-time = "2025-07-27T16:27:32.637Z" }, + { url = "https://files.pythonhosted.org/packages/f8/d9/ec4864f5896232133f51382b54a08de91a9d1af7a76dfa372894026dfee2/scipy-1.16.1-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:81b433bbeaf35728dad619afc002db9b189e45eebe2cd676effe1fb93fef2b9c", size = 36575194, upload-time = "2025-07-27T16:27:41.321Z" }, + { url = "https://files.pythonhosted.org/packages/5c/6d/40e81ecfb688e9d25d34a847dca361982a6addf8e31f0957b1a54fbfa994/scipy-1.16.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:886cc81fdb4c6903a3bb0464047c25a6d1016fef77bb97949817d0c0d79f9e04", size = 28594590, upload-time = "2025-07-27T16:27:49.204Z" }, + { url = "https://files.pythonhosted.org/packages/0e/37/9f65178edfcc629377ce9a64fc09baebea18c80a9e57ae09a52edf84880b/scipy-1.16.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:15240c3aac087a522b4eaedb09f0ad061753c5eebf1ea430859e5bf8640d5919", size = 20866458, upload-time = "2025-07-27T16:27:54.98Z" }, + { url = "https://files.pythonhosted.org/packages/2c/7b/749a66766871ea4cb1d1ea10f27004db63023074c22abed51f22f09770e0/scipy-1.16.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:65f81a25805f3659b48126b5053d9e823d3215e4a63730b5e1671852a1705921", size = 23539318, upload-time = "2025-07-27T16:28:01.604Z" }, + { url = "https://files.pythonhosted.org/packages/c4/db/8d4afec60eb833a666434d4541a3151eedbf2494ea6d4d468cbe877f00cd/scipy-1.16.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6c62eea7f607f122069b9bad3f99489ddca1a5173bef8a0c75555d7488b6f725", size = 33292899, upload-time = "2025-07-27T16:28:09.147Z" }, + { url = "https://files.pythonhosted.org/packages/51/1e/79023ca3bbb13a015d7d2757ecca3b81293c663694c35d6541b4dca53e98/scipy-1.16.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f965bbf3235b01c776115ab18f092a95aa74c271a52577bcb0563e85738fd618", size = 35162637, upload-time = "2025-07-27T16:28:17.535Z" }, + { url = "https://files.pythonhosted.org/packages/b6/49/0648665f9c29fdaca4c679182eb972935b3b4f5ace41d323c32352f29816/scipy-1.16.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f006e323874ffd0b0b816d8c6a8e7f9a73d55ab3b8c3f72b752b226d0e3ac83d", size = 35490507, upload-time = "2025-07-27T16:28:25.705Z" }, + { url = "https://files.pythonhosted.org/packages/62/8f/66cbb9d6bbb18d8c658f774904f42a92078707a7c71e5347e8bf2f52bb89/scipy-1.16.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e8fd15fc5085ab4cca74cb91fe0a4263b1f32e4420761ddae531ad60934c2119", size = 37923998, upload-time = "2025-07-27T16:28:34.339Z" }, + { url = "https://files.pythonhosted.org/packages/14/c3/61f273ae550fbf1667675701112e380881905e28448c080b23b5a181df7c/scipy-1.16.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7b8013c6c066609577d910d1a2a077021727af07b6fab0ee22c2f901f22352a", size = 38508060, upload-time = "2025-07-27T16:28:43.242Z" }, ] [[package]] @@ -3028,6 +4623,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl", hash = "sha256:9c824d87ba7f7ab4a1890799cec8596f15c1241cb473404ea1cb0c55e4b04746", size = 17912, upload-time = "2025-01-24T13:19:24.949Z" }, ] +[[package]] +name = "sentry-sdk" +version = "2.35.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/31/83/055dc157b719651ef13db569bb8cf2103df11174478649735c1b2bf3f6bc/sentry_sdk-2.35.0.tar.gz", hash = "sha256:5ea58d352779ce45d17bc2fa71ec7185205295b83a9dbb5707273deb64720092", size = 343014, upload-time = "2025-08-14T17:11:20.223Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/3d/742617a7c644deb0c1628dcf6bb2d2165ab7c6aab56fe5222758994007f8/sentry_sdk-2.35.0-py2.py3-none-any.whl", hash = "sha256:6e0c29b9a5d34de8575ffb04d289a987ff3053cf2c98ede445bea995e3830263", size = 363806, upload-time = "2025-08-14T17:11:18.29Z" }, +] + [[package]] name = "setuptools" version = "80.9.0" @@ -3039,15 +4647,24 @@ wheels = [ [[package]] name = "setuptools-scm" -version = "8.3.1" +version = "9.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "packaging" }, { name = "setuptools" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b9/19/7ae64b70b2429c48c3a7a4ed36f50f94687d3bfcd0ae2f152367b6410dff/setuptools_scm-8.3.1.tar.gz", hash = "sha256:3d555e92b75dacd037d32bafdf94f97af51ea29ae8c7b234cf94b7a5bd242a63", size = 78088, upload-time = "2025-04-23T11:53:19.739Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8f/8d/ffdcace33d0480d591057a30285b7c33f8dc431fed3fff7dbadf5f9f128f/setuptools_scm-9.2.0.tar.gz", hash = "sha256:6662c9b9497b6c9bf13bead9d7a9084756f68238302c5ed089fb4dbd29d102d7", size = 201229, upload-time = "2025-08-16T12:56:39.477Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/14/dd3a6053325e882fe191fb4b42289bbdfabf5f44307c302903a8a3236a0a/setuptools_scm-9.2.0-py3-none-any.whl", hash = "sha256:c551ef54e2270727ee17067881c9687ca2aedf179fa5b8f3fab9e8d73bdc421f", size = 62099, upload-time = "2025-08-16T12:56:37.912Z" }, +] + +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ab/ac/8f96ba9b4cfe3e4ea201f23f4f97165862395e9331a424ed325ae37024a8/setuptools_scm-8.3.1-py3-none-any.whl", hash = "sha256:332ca0d43791b818b841213e76b1971b7711a960761c5bea5fc5cdb5196fbce3", size = 43935, upload-time = "2025-04-23T11:53:17.922Z" }, + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] [[package]] @@ -3059,6 +4676,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "smart-open" +version = "7.3.0.post1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/2b/5e7234c68ed5bc872ad6ae77b8a421c2ed70dcb1190b44dc1abdeed5e347/smart_open-7.3.0.post1.tar.gz", hash = "sha256:ce6a3d9bc1afbf6234ad13c010b77f8cd36d24636811e3c52c3b5160f5214d1e", size = 51557, upload-time = "2025-07-03T10:06:31.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/5b/a2a3d4514c64818925f4e886d39981f1926eeb5288a4549c6b3c17ed66bb/smart_open-7.3.0.post1-py3-none-any.whl", hash = "sha256:c73661a2c24bf045c1e04e08fffc585b59af023fe783d57896f590489db66fb4", size = 61946, upload-time = "2025-07-03T10:06:29.599Z" }, +] + [[package]] name = "smmap" version = "5.0.2" @@ -3095,33 +4724,94 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/9c/0e6afc12c269578be5c0c1c9f4b49a8d32770a080260c333ac04cc1c832d/soupsieve-2.7-py3-none-any.whl", hash = "sha256:6e60cc5c1ffaf1cebcc12e8188320b72071e922c2e897f737cadce79ad5d30c4", size = 36677, upload-time = "2025-04-20T18:50:07.196Z" }, ] +[[package]] +name = "spacy" +version = "3.8.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "catalogue" }, + { name = "cymem" }, + { name = "jinja2" }, + { name = "langcodes" }, + { name = "murmurhash" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "preshed" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "setuptools" }, + { name = "spacy-legacy" }, + { name = "spacy-loggers" }, + { name = "srsly" }, + { name = "thinc" }, + { name = "tqdm" }, + { name = "typer" }, + { name = "wasabi" }, + { name = "weasel" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1e/9e/fb4e1cefe3fbd51ea6a243e5a3d2bc629baa9a28930bf4be6fe5672fa1ca/spacy-3.8.7.tar.gz", hash = "sha256:700fd174c6c552276be142c48e70bb53cae24c4dd86003c4432af9cb93e4c908", size = 1316143, upload-time = "2025-05-23T08:55:39.538Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/c5/5fbb3a4e694d4855a5bab87af9664377c48b89691f180ad3cde4faeaf35c/spacy-3.8.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bdff8b9b556468a6dd527af17f0ddf9fb0b0bee92ee7703339ddf542361cff98", size = 6746140, upload-time = "2025-05-23T08:54:23.483Z" }, + { url = "https://files.pythonhosted.org/packages/03/2a/43afac516eb82409ca47d7206f982beaf265d2ba06a72ca07cf06b290c20/spacy-3.8.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9194b7cf015ed9b4450ffb162da49c8a9305e76b468de036b0948abdfc748a37", size = 6392440, upload-time = "2025-05-23T08:54:25.12Z" }, + { url = "https://files.pythonhosted.org/packages/6f/83/2ea68c18e2b1b9a6f6b30ef63eb9d07e979626b9595acfdb5394f18923c4/spacy-3.8.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7dc38b78d48b9c2a80a3eea95f776304993f63fc307f07cdd104441442f92f1e", size = 32699126, upload-time = "2025-05-23T08:54:27.385Z" }, + { url = "https://files.pythonhosted.org/packages/0a/0a/bb90e9aa0b3c527876627567d82517aabab08006ccf63796c33b0242254d/spacy-3.8.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e43bd70772751b8fc7a14f338d087a3d297195d43d171832923ef66204b23ab", size = 33008865, upload-time = "2025-05-23T08:54:30.248Z" }, + { url = "https://files.pythonhosted.org/packages/39/dd/8e906ba378457107ab0394976ea9f7b12fdb2cad682ef1a2ccf473d61e5f/spacy-3.8.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c402bf5dcf345fd96d202378c54bc345219681e3531f911d99567d569328c45f", size = 31933169, upload-time = "2025-05-23T08:54:33.199Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b5/42df07eb837a923fbb42509864d5c7c2072d010de933dccdfb3c655b3a76/spacy-3.8.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4234189861e486d86f1269e50542d87e8a6391a1ee190652479cf1a793db115f", size = 32776322, upload-time = "2025-05-23T08:54:36.891Z" }, + { url = "https://files.pythonhosted.org/packages/92/e7/8176484801c67dcd814f141991fe0a3c9b5b4a3583ea30c2062e93d1aa6b/spacy-3.8.7-cp311-cp311-win_amd64.whl", hash = "sha256:e9d12e2eb7f36bc11dd9edae011032fe49ea100d63e83177290d3cbd80eaa650", size = 14938936, upload-time = "2025-05-23T08:54:40.322Z" }, + { url = "https://files.pythonhosted.org/packages/a5/10/89852f40f926e0902c11c34454493ba0d15530b322711e754b89a6d7dfe6/spacy-3.8.7-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:88b397e37793cea51df298e6c651a763e49877a25bead5ba349761531a456687", size = 6265335, upload-time = "2025-05-23T08:54:42.876Z" }, + { url = "https://files.pythonhosted.org/packages/16/fb/b5d54522969a632c06f4af354763467553b66d5bf0671ac39f3cceb3fd54/spacy-3.8.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f70b676955fa6959347ca86ed6edd8ff0d6eb2ba20561fdfec76924bd3e540f9", size = 5906035, upload-time = "2025-05-23T08:54:44.824Z" }, + { url = "https://files.pythonhosted.org/packages/3a/03/70f06753fd65081404ade30408535eb69f627a36ffce2107116d1aa16239/spacy-3.8.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c4b5a624797ade30c25b5b69daa35a93ee24bcc56bd79b0884b2565f76f35d6", size = 33420084, upload-time = "2025-05-23T08:54:46.889Z" }, + { url = "https://files.pythonhosted.org/packages/f9/19/b60e1ebf4985ee2b33d85705b89a5024942b65dad04dbdc3fb46f168b410/spacy-3.8.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9d83e006df66decccefa3872fa958b3756228fb216d83783595444cf42ca10c", size = 33922188, upload-time = "2025-05-23T08:54:49.781Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a3/1fb1a49dc6d982d96fffc30c3a31bb431526008eea72ac3773f6518720a6/spacy-3.8.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0dca25deba54f3eb5dcfbf63bf16e613e6c601da56f91c4a902d38533c098941", size = 31939285, upload-time = "2025-05-23T08:54:53.162Z" }, + { url = "https://files.pythonhosted.org/packages/2d/55/6cf1aff8e5c01ee683e828f3ccd9282d2aff7ca1143a9349ee3d0c1291ff/spacy-3.8.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5eef3f805a1c118d9b709a23e2d378f5f20da5a0d6258c9cfdc87c4cb234b4fc", size = 32988845, upload-time = "2025-05-23T08:54:57.776Z" }, + { url = "https://files.pythonhosted.org/packages/8c/47/c17ee61b51aa8497d8af0999224b4b62485111a55ec105a06886685b2c68/spacy-3.8.7-cp312-cp312-win_amd64.whl", hash = "sha256:25d7a68e445200c9e9dc0044f8b7278ec0ef01ccc7cb5a95d1de2bd8e3ed6be2", size = 13918682, upload-time = "2025-05-23T08:55:00.387Z" }, +] + +[[package]] +name = "spacy-legacy" +version = "3.0.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/79/91f9d7cc8db5642acad830dcc4b49ba65a7790152832c4eceb305e46d681/spacy-legacy-3.0.12.tar.gz", hash = "sha256:b37d6e0c9b6e1d7ca1cf5bc7152ab64a4c4671f59c85adaf7a3fcb870357a774", size = 23806, upload-time = "2023-01-23T09:04:15.104Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/55/12e842c70ff8828e34e543a2c7176dac4da006ca6901c9e8b43efab8bc6b/spacy_legacy-3.0.12-py2.py3-none-any.whl", hash = "sha256:476e3bd0d05f8c339ed60f40986c07387c0a71479245d6d0f4298dbd52cda55f", size = 29971, upload-time = "2023-01-23T09:04:13.45Z" }, +] + +[[package]] +name = "spacy-loggers" +version = "1.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/67/3d/926db774c9c98acf66cb4ed7faf6c377746f3e00b84b700d0868b95d0712/spacy-loggers-1.0.5.tar.gz", hash = "sha256:d60b0bdbf915a60e516cc2e653baeff946f0cfc461b452d11a4d5458c6fe5f24", size = 20811, upload-time = "2023-09-11T12:26:52.323Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/78/d1a1a026ef3af911159398c939b1509d5c36fe524c7b644f34a5146c4e16/spacy_loggers-1.0.5-py3-none-any.whl", hash = "sha256:196284c9c446cc0cdb944005384270d775fdeaf4f494d8e269466cfa497ef645", size = 22343, upload-time = "2023-09-11T12:26:50.586Z" }, +] + [[package]] name = "sqlalchemy" -version = "2.0.41" +version = "2.0.43" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/63/66/45b165c595ec89aa7dcc2c1cd222ab269bc753f1fc7a1e68f8481bd957bf/sqlalchemy-2.0.41.tar.gz", hash = "sha256:edba70118c4be3c2b1f90754d308d0b79c6fe2c0fdc52d8ddf603916f83f4db9", size = 9689424, upload-time = "2025-05-14T17:10:32.339Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/37/4e/b00e3ffae32b74b5180e15d2ab4040531ee1bef4c19755fe7926622dc958/sqlalchemy-2.0.41-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6375cd674fe82d7aa9816d1cb96ec592bac1726c11e0cafbf40eeee9a4516b5f", size = 2121232, upload-time = "2025-05-14T17:48:20.444Z" }, - { url = "https://files.pythonhosted.org/packages/ef/30/6547ebb10875302074a37e1970a5dce7985240665778cfdee2323709f749/sqlalchemy-2.0.41-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f8c9fdd15a55d9465e590a402f42082705d66b05afc3ffd2d2eb3c6ba919560", size = 2110897, upload-time = "2025-05-14T17:48:21.634Z" }, - { url = "https://files.pythonhosted.org/packages/9e/21/59df2b41b0f6c62da55cd64798232d7349a9378befa7f1bb18cf1dfd510a/sqlalchemy-2.0.41-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32f9dc8c44acdee06c8fc6440db9eae8b4af8b01e4b1aee7bdd7241c22edff4f", size = 3273313, upload-time = "2025-05-14T17:51:56.205Z" }, - { url = "https://files.pythonhosted.org/packages/62/e4/b9a7a0e5c6f79d49bcd6efb6e90d7536dc604dab64582a9dec220dab54b6/sqlalchemy-2.0.41-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90c11ceb9a1f482c752a71f203a81858625d8df5746d787a4786bca4ffdf71c6", size = 3273807, upload-time = "2025-05-14T17:55:26.928Z" }, - { url = "https://files.pythonhosted.org/packages/39/d8/79f2427251b44ddee18676c04eab038d043cff0e764d2d8bb08261d6135d/sqlalchemy-2.0.41-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:911cc493ebd60de5f285bcae0491a60b4f2a9f0f5c270edd1c4dbaef7a38fc04", size = 3209632, upload-time = "2025-05-14T17:51:59.384Z" }, - { url = "https://files.pythonhosted.org/packages/d4/16/730a82dda30765f63e0454918c982fb7193f6b398b31d63c7c3bd3652ae5/sqlalchemy-2.0.41-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03968a349db483936c249f4d9cd14ff2c296adfa1290b660ba6516f973139582", size = 3233642, upload-time = "2025-05-14T17:55:29.901Z" }, - { url = "https://files.pythonhosted.org/packages/04/61/c0d4607f7799efa8b8ea3c49b4621e861c8f5c41fd4b5b636c534fcb7d73/sqlalchemy-2.0.41-cp311-cp311-win32.whl", hash = "sha256:293cd444d82b18da48c9f71cd7005844dbbd06ca19be1ccf6779154439eec0b8", size = 2086475, upload-time = "2025-05-14T17:56:02.095Z" }, - { url = "https://files.pythonhosted.org/packages/9d/8e/8344f8ae1cb6a479d0741c02cd4f666925b2bf02e2468ddaf5ce44111f30/sqlalchemy-2.0.41-cp311-cp311-win_amd64.whl", hash = "sha256:3d3549fc3e40667ec7199033a4e40a2f669898a00a7b18a931d3efb4c7900504", size = 2110903, upload-time = "2025-05-14T17:56:03.499Z" }, - { url = "https://files.pythonhosted.org/packages/3e/2a/f1f4e068b371154740dd10fb81afb5240d5af4aa0087b88d8b308b5429c2/sqlalchemy-2.0.41-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:81f413674d85cfd0dfcd6512e10e0f33c19c21860342a4890c3a2b59479929f9", size = 2119645, upload-time = "2025-05-14T17:55:24.854Z" }, - { url = "https://files.pythonhosted.org/packages/9b/e8/c664a7e73d36fbfc4730f8cf2bf930444ea87270f2825efbe17bf808b998/sqlalchemy-2.0.41-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:598d9ebc1e796431bbd068e41e4de4dc34312b7aa3292571bb3674a0cb415dd1", size = 2107399, upload-time = "2025-05-14T17:55:28.097Z" }, - { url = "https://files.pythonhosted.org/packages/5c/78/8a9cf6c5e7135540cb682128d091d6afa1b9e48bd049b0d691bf54114f70/sqlalchemy-2.0.41-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a104c5694dfd2d864a6f91b0956eb5d5883234119cb40010115fd45a16da5e70", size = 3293269, upload-time = "2025-05-14T17:50:38.227Z" }, - { url = "https://files.pythonhosted.org/packages/3c/35/f74add3978c20de6323fb11cb5162702670cc7a9420033befb43d8d5b7a4/sqlalchemy-2.0.41-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6145afea51ff0af7f2564a05fa95eb46f542919e6523729663a5d285ecb3cf5e", size = 3303364, upload-time = "2025-05-14T17:51:49.829Z" }, - { url = "https://files.pythonhosted.org/packages/6a/d4/c990f37f52c3f7748ebe98883e2a0f7d038108c2c5a82468d1ff3eec50b7/sqlalchemy-2.0.41-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b46fa6eae1cd1c20e6e6f44e19984d438b6b2d8616d21d783d150df714f44078", size = 3229072, upload-time = "2025-05-14T17:50:39.774Z" }, - { url = "https://files.pythonhosted.org/packages/15/69/cab11fecc7eb64bc561011be2bd03d065b762d87add52a4ca0aca2e12904/sqlalchemy-2.0.41-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41836fe661cc98abfae476e14ba1906220f92c4e528771a8a3ae6a151242d2ae", size = 3268074, upload-time = "2025-05-14T17:51:51.736Z" }, - { url = "https://files.pythonhosted.org/packages/5c/ca/0c19ec16858585d37767b167fc9602593f98998a68a798450558239fb04a/sqlalchemy-2.0.41-cp312-cp312-win32.whl", hash = "sha256:a8808d5cf866c781150d36a3c8eb3adccfa41a8105d031bf27e92c251e3969d6", size = 2084514, upload-time = "2025-05-14T17:55:49.915Z" }, - { url = "https://files.pythonhosted.org/packages/7f/23/4c2833d78ff3010a4e17f984c734f52b531a8c9060a50429c9d4b0211be6/sqlalchemy-2.0.41-cp312-cp312-win_amd64.whl", hash = "sha256:5b14e97886199c1f52c14629c11d90c11fbb09e9334fa7bb5f6d068d9ced0ce0", size = 2111557, upload-time = "2025-05-14T17:55:51.349Z" }, - { url = "https://files.pythonhosted.org/packages/1c/fc/9ba22f01b5cdacc8f5ed0d22304718d2c758fce3fd49a5372b886a86f37c/sqlalchemy-2.0.41-py3-none-any.whl", hash = "sha256:57df5dc6fdb5ed1a88a1ed2195fd31927e705cad62dedd86b46972752a80f576", size = 1911224, upload-time = "2025-05-14T17:39:42.154Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/d7/bc/d59b5d97d27229b0e009bd9098cd81af71c2fa5549c580a0a67b9bed0496/sqlalchemy-2.0.43.tar.gz", hash = "sha256:788bfcef6787a7764169cfe9859fe425bf44559619e1d9f56f5bddf2ebf6f417", size = 9762949, upload-time = "2025-08-11T14:24:58.438Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/77/fa7189fe44114658002566c6fe443d3ed0ec1fa782feb72af6ef7fbe98e7/sqlalchemy-2.0.43-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:52d9b73b8fb3e9da34c2b31e6d99d60f5f99fd8c1225c9dad24aeb74a91e1d29", size = 2136472, upload-time = "2025-08-11T15:52:21.789Z" }, + { url = "https://files.pythonhosted.org/packages/99/ea/92ac27f2fbc2e6c1766bb807084ca455265707e041ba027c09c17d697867/sqlalchemy-2.0.43-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f42f23e152e4545157fa367b2435a1ace7571cab016ca26038867eb7df2c3631", size = 2126535, upload-time = "2025-08-11T15:52:23.109Z" }, + { url = "https://files.pythonhosted.org/packages/94/12/536ede80163e295dc57fff69724caf68f91bb40578b6ac6583a293534849/sqlalchemy-2.0.43-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fb1a8c5438e0c5ea51afe9c6564f951525795cf432bed0c028c1cb081276685", size = 3297521, upload-time = "2025-08-11T15:50:33.536Z" }, + { url = "https://files.pythonhosted.org/packages/03/b5/cacf432e6f1fc9d156eca0560ac61d4355d2181e751ba8c0cd9cb232c8c1/sqlalchemy-2.0.43-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db691fa174e8f7036afefe3061bc40ac2b770718be2862bfb03aabae09051aca", size = 3297343, upload-time = "2025-08-11T15:57:51.186Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ba/d4c9b526f18457667de4c024ffbc3a0920c34237b9e9dd298e44c7c00ee5/sqlalchemy-2.0.43-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fe2b3b4927d0bc03d02ad883f402d5de201dbc8894ac87d2e981e7d87430e60d", size = 3232113, upload-time = "2025-08-11T15:50:34.949Z" }, + { url = "https://files.pythonhosted.org/packages/aa/79/c0121b12b1b114e2c8a10ea297a8a6d5367bc59081b2be896815154b1163/sqlalchemy-2.0.43-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4d3d9b904ad4a6b175a2de0738248822f5ac410f52c2fd389ada0b5262d6a1e3", size = 3258240, upload-time = "2025-08-11T15:57:52.983Z" }, + { url = "https://files.pythonhosted.org/packages/79/99/a2f9be96fb382f3ba027ad42f00dbe30fdb6ba28cda5f11412eee346bec5/sqlalchemy-2.0.43-cp311-cp311-win32.whl", hash = "sha256:5cda6b51faff2639296e276591808c1726c4a77929cfaa0f514f30a5f6156921", size = 2101248, upload-time = "2025-08-11T15:55:01.855Z" }, + { url = "https://files.pythonhosted.org/packages/ee/13/744a32ebe3b4a7a9c7ea4e57babae7aa22070d47acf330d8e5a1359607f1/sqlalchemy-2.0.43-cp311-cp311-win_amd64.whl", hash = "sha256:c5d1730b25d9a07727d20ad74bc1039bbbb0a6ca24e6769861c1aa5bf2c4c4a8", size = 2126109, upload-time = "2025-08-11T15:55:04.092Z" }, + { url = "https://files.pythonhosted.org/packages/61/db/20c78f1081446095450bdc6ee6cc10045fce67a8e003a5876b6eaafc5cc4/sqlalchemy-2.0.43-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:20d81fc2736509d7a2bd33292e489b056cbae543661bb7de7ce9f1c0cd6e7f24", size = 2134891, upload-time = "2025-08-11T15:51:13.019Z" }, + { url = "https://files.pythonhosted.org/packages/45/0a/3d89034ae62b200b4396f0f95319f7d86e9945ee64d2343dcad857150fa2/sqlalchemy-2.0.43-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:25b9fc27650ff5a2c9d490c13c14906b918b0de1f8fcbb4c992712d8caf40e83", size = 2123061, upload-time = "2025-08-11T15:51:14.319Z" }, + { url = "https://files.pythonhosted.org/packages/cb/10/2711f7ff1805919221ad5bee205971254845c069ee2e7036847103ca1e4c/sqlalchemy-2.0.43-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6772e3ca8a43a65a37c88e2f3e2adfd511b0b1da37ef11ed78dea16aeae85bd9", size = 3320384, upload-time = "2025-08-11T15:52:35.088Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0e/3d155e264d2ed2778484006ef04647bc63f55b3e2d12e6a4f787747b5900/sqlalchemy-2.0.43-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a113da919c25f7f641ffbd07fbc9077abd4b3b75097c888ab818f962707eb48", size = 3329648, upload-time = "2025-08-11T15:56:34.153Z" }, + { url = "https://files.pythonhosted.org/packages/5b/81/635100fb19725c931622c673900da5efb1595c96ff5b441e07e3dd61f2be/sqlalchemy-2.0.43-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4286a1139f14b7d70141c67a8ae1582fc2b69105f1b09d9573494eb4bb4b2687", size = 3258030, upload-time = "2025-08-11T15:52:36.933Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ed/a99302716d62b4965fded12520c1cbb189f99b17a6d8cf77611d21442e47/sqlalchemy-2.0.43-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:529064085be2f4d8a6e5fab12d36ad44f1909a18848fcfbdb59cc6d4bbe48efe", size = 3294469, upload-time = "2025-08-11T15:56:35.553Z" }, + { url = "https://files.pythonhosted.org/packages/5d/a2/3a11b06715149bf3310b55a98b5c1e84a42cfb949a7b800bc75cb4e33abc/sqlalchemy-2.0.43-cp312-cp312-win32.whl", hash = "sha256:b535d35dea8bbb8195e7e2b40059e2253acb2b7579b73c1b432a35363694641d", size = 2098906, upload-time = "2025-08-11T15:55:00.645Z" }, + { url = "https://files.pythonhosted.org/packages/bc/09/405c915a974814b90aa591280623adc6ad6b322f61fd5cff80aeaef216c9/sqlalchemy-2.0.43-cp312-cp312-win_amd64.whl", hash = "sha256:1c6d85327ca688dbae7e2b06d7d84cfe4f3fffa5b5f9e21bb6ce9d0e1a0e0e0a", size = 2126260, upload-time = "2025-08-11T15:55:02.965Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d9/13bdde6521f322861fab67473cec4b1cc8999f3871953531cf61945fad92/sqlalchemy-2.0.43-py3-none-any.whl", hash = "sha256:1681c21dd2ccee222c2fe0bef671d1aef7c504087c9c4e800371cfcc8ac966fc", size = 1924759, upload-time = "2025-08-11T15:39:53.024Z" }, ] [package.optional-dependencies] @@ -3149,16 +4839,55 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f2/d1/8828ec685022e2b86ebd717743359dd5e8eab70a9c1ebff78aa733848216/sqlean_py-3.49.1-cp312-cp312-win_arm64.whl", hash = "sha256:c2537f69879adaed22b16e840d494c440ece5b49d28a5a51094e6c8ca6a2b0a5", size = 739690, upload-time = "2025-05-02T11:58:02.262Z" }, ] +[[package]] +name = "srsly" +version = "2.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "catalogue" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b7/e8/eb51b1349f50bac0222398af0942613fdc9d1453ae67cbe4bf9936a1a54b/srsly-2.5.1.tar.gz", hash = "sha256:ab1b4bf6cf3e29da23dae0493dd1517fb787075206512351421b89b4fc27c77e", size = 466464, upload-time = "2025-01-17T09:26:26.919Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/9c/a248bb49de499fe0990e3cb0fb341c2373d8863ef9a8b5799353cade5731/srsly-2.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:58f0736794ce00a71d62a39cbba1d62ea8d5be4751df956e802d147da20ecad7", size = 635917, upload-time = "2025-01-17T09:25:25.109Z" }, + { url = "https://files.pythonhosted.org/packages/41/47/1bdaad84502df973ecb8ca658117234cf7fb20e1dec60da71dce82de993f/srsly-2.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7a8269c40859806d71920396d185f4f38dc985cdb6a28d3a326a701e29a5f629", size = 634374, upload-time = "2025-01-17T09:25:26.609Z" }, + { url = "https://files.pythonhosted.org/packages/e5/2a/d73c71989fcf2a6d1fa518d75322aff4db01a8763f167f8c5e00aac11097/srsly-2.5.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:889905900401fefc1032e22b73aecbed8b4251aa363f632b2d1f86fc16f1ad8e", size = 1108390, upload-time = "2025-01-17T09:25:29.32Z" }, + { url = "https://files.pythonhosted.org/packages/35/a3/9eda9997a8bd011caed18fdaa5ce606714eb06d8dab587ed0522b3e92ab1/srsly-2.5.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf454755f22589df49c25dc799d8af7b47dce3d861dded35baf0f0b6ceab4422", size = 1110712, upload-time = "2025-01-17T09:25:31.051Z" }, + { url = "https://files.pythonhosted.org/packages/8a/ef/4b50bc05d06349f905b27f824cc23b652098efd4be19aead3af4981df647/srsly-2.5.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cc0607c8a59013a51dde5c1b4e465558728e9e0a35dcfa73c7cbefa91a0aad50", size = 1081244, upload-time = "2025-01-17T09:25:32.611Z" }, + { url = "https://files.pythonhosted.org/packages/90/af/d4a2512d9a5048d2b18efead39d4c4404bddd4972935bbc68211292a736c/srsly-2.5.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d5421ba3ab3c790e8b41939c51a1d0f44326bfc052d7a0508860fb79a47aee7f", size = 1091692, upload-time = "2025-01-17T09:25:34.15Z" }, + { url = "https://files.pythonhosted.org/packages/bb/da/657a685f63028dcb00ccdc4ac125ed347c8bff6fa0dab6a9eb3dc45f3223/srsly-2.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:b96ea5a9a0d0379a79c46d255464a372fb14c30f59a8bc113e4316d131a530ab", size = 632627, upload-time = "2025-01-17T09:25:37.36Z" }, + { url = "https://files.pythonhosted.org/packages/fb/f6/bebc20d75bd02121fc0f65ad8c92a5dd2570e870005e940faa55a263e61a/srsly-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:683b54ed63d7dfee03bc2abc4b4a5f2152f81ec217bbadbac01ef1aaf2a75790", size = 636717, upload-time = "2025-01-17T09:25:40.236Z" }, + { url = "https://files.pythonhosted.org/packages/b6/e8/9372317a4742c70b87b413335adfcdfb2bee4f88f3faba89fabb9e6abf21/srsly-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:459d987130e57e83ce9e160899afbeb871d975f811e6958158763dd9a8a20f23", size = 634697, upload-time = "2025-01-17T09:25:43.605Z" }, + { url = "https://files.pythonhosted.org/packages/d5/00/c6a7b99ab27b051a27bd26fe1a8c1885225bb8980282bf9cb99f70610368/srsly-2.5.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:184e3c98389aab68ff04aab9095bd5f1a8e5a72cc5edcba9d733bac928f5cf9f", size = 1134655, upload-time = "2025-01-17T09:25:45.238Z" }, + { url = "https://files.pythonhosted.org/packages/c2/e6/861459e8241ec3b78c111081bd5efa414ef85867e17c45b6882954468d6e/srsly-2.5.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c2a3e4856e63b7efd47591d049aaee8e5a250e098917f50d93ea68853fab78", size = 1143544, upload-time = "2025-01-17T09:25:47.485Z" }, + { url = "https://files.pythonhosted.org/packages/2d/85/8448fe874dd2042a4eceea5315cfff3af03ac77ff5073812071852c4e7e2/srsly-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:366b4708933cd8d6025c13c2cea3331f079c7bb5c25ec76fca392b6fc09818a0", size = 1098330, upload-time = "2025-01-17T09:25:52.55Z" }, + { url = "https://files.pythonhosted.org/packages/ef/7e/04d0e1417da140b2ac4053a3d4fcfc86cd59bf4829f69d370bb899f74d5d/srsly-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c8a0b03c64eb6e150d772c5149befbadd981cc734ab13184b0561c17c8cef9b1", size = 1110670, upload-time = "2025-01-17T09:25:54.02Z" }, + { url = "https://files.pythonhosted.org/packages/96/1a/a8cd627eaa81a91feb6ceab50155f4ceff3eef6107916cb87ef796958427/srsly-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:7952538f6bba91b9d8bf31a642ac9e8b9ccc0ccbb309feb88518bfb84bb0dc0d", size = 632598, upload-time = "2025-01-17T09:25:55.499Z" }, +] + [[package]] name = "sse-starlette" -version = "2.4.1" +version = "3.0.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/07/3e/eae74d8d33e3262bae0a7e023bb43d8bdd27980aa3557333f4632611151f/sse_starlette-2.4.1.tar.gz", hash = "sha256:7c8a800a1ca343e9165fc06bbda45c78e4c6166320707ae30b416c42da070926", size = 18635, upload-time = "2025-07-06T09:41:33.631Z" } +sdist = { url = "https://files.pythonhosted.org/packages/42/6f/22ed6e33f8a9e76ca0a412405f31abb844b779d52c5f96660766edcd737c/sse_starlette-3.0.2.tar.gz", hash = "sha256:ccd60b5765ebb3584d0de2d7a6e4f745672581de4f5005ab31c3a25d10b52b3a", size = 20985, upload-time = "2025-07-27T09:07:44.565Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/10/c78f463b4ef22eef8491f218f692be838282cd65480f6e423d7730dfd1fb/sse_starlette-3.0.2-py3-none-any.whl", hash = "sha256:16b7cbfddbcd4eaca11f7b586f3b8a080f1afe952c15813455b162edea619e5a", size = 11297, upload-time = "2025-07-27T09:07:43.268Z" }, +] + +[[package]] +name = "stack-data" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asttokens" }, + { name = "executing" }, + { name = "pure-eval" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/e3/55dcc2cfbc3ca9c29519eb6884dd1415ecb53b0e934862d3559ddcb7e20b/stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", size = 44707, upload-time = "2023-09-30T13:58:05.479Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e4/f1/6c7eaa8187ba789a6dd6d74430307478d2a91c23a5452ab339b6fbe15a08/sse_starlette-2.4.1-py3-none-any.whl", hash = "sha256:08b77ea898ab1a13a428b2b6f73cfe6d0e607a7b4e15b9bb23e4a37b087fd39a", size = 10824, upload-time = "2025-07-06T09:41:32.321Z" }, + { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, ] [[package]] @@ -3173,6 +4902,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8b/0c/9d30a4ebeb6db2b25a841afbb80f6ef9a854fc3b41be131d249a977b4959/starlette-0.46.2-py3-none-any.whl", hash = "sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35", size = 72037, upload-time = "2025-04-13T13:56:16.21Z" }, ] +[[package]] +name = "stdlib-list" +version = "0.11.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5d/09/8d5c564931ae23bef17420a6c72618463a59222ca4291a7dd88de8a0d490/stdlib_list-0.11.1.tar.gz", hash = "sha256:95ebd1d73da9333bba03ccc097f5bac05e3aa03e6822a0c0290f87e1047f1857", size = 60442, upload-time = "2025-02-18T15:39:38.769Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/c7/4102536de33c19d090ed2b04e90e7452e2e3dc653cf3323208034eaaca27/stdlib_list-0.11.1-py3-none-any.whl", hash = "sha256:9029ea5e3dfde8cd4294cfd4d1797be56a67fc4693c606181730148c3fd1da29", size = 83620, upload-time = "2025-02-18T15:39:37.02Z" }, +] + [[package]] name = "strawberry-graphql" version = "0.243.1" @@ -3187,6 +4925,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/b4/7c4b5ccce02f111883a56fc1331506bc4a12824a037db7f2904e0d26eb6c/strawberry_graphql-0.243.1-py3-none-any.whl", hash = "sha256:7c4ddb97cd424fa23a540816cb169ef760822c9acfb63901e6717042bcda6cfe", size = 306031, upload-time = "2024-09-26T12:24:22.338Z" }, ] +[[package]] +name = "striprtf" +version = "0.0.26" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/25/20/3d419008265346452d09e5dadfd5d045b64b40d8fc31af40588e6c76997a/striprtf-0.0.26.tar.gz", hash = "sha256:fdb2bba7ac440072d1c41eab50d8d74ae88f60a8b6575c6e2c7805dc462093aa", size = 6258, upload-time = "2023-07-20T14:30:36.29Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/cf/0fea4f4ba3fc2772ac2419278aa9f6964124d4302117d61bc055758e000c/striprtf-0.0.26-py3-none-any.whl", hash = "sha256:8c8f9d32083cdc2e8bfb149455aa1cc5a4e0a035893bedc75db8b73becb3a1bb", size = 6914, upload-time = "2023-07-20T14:30:35.338Z" }, +] + +[[package]] +name = "tabulate" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/fe/802052aecb21e3797b8f7902564ab6ea0d60ff8ca23952079064155d1ae1/tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c", size = 81090, upload-time = "2022-10-06T17:21:48.54Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252, upload-time = "2022-10-06T17:21:44.262Z" }, +] + [[package]] name = "tantivy" version = "0.22.2" @@ -3207,11 +4963,43 @@ wheels = [ [[package]] name = "tenacity" -version = "9.1.2" +version = "8.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/32/6c/57df6196ce52c464cf8556e8f697fec5d3469bb8cd319c1685c0a090e0b4/tenacity-8.3.0.tar.gz", hash = "sha256:953d4e6ad24357bceffbc9707bc74349aca9d245f68eb65419cf0c249a1949a2", size = 43608, upload-time = "2024-05-07T08:48:17.099Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/a1/6bb0cbebefb23641f068bb58a2bc56da9beb2b1c550242e3c540b37698f3/tenacity-8.3.0-py3-none-any.whl", hash = "sha256:3649f6443dbc0d9b01b9d8020a9c4ec7a1ff5f6f3c6c8a036ef371f573fe9185", size = 25934, upload-time = "2024-05-07T08:48:14.696Z" }, +] + +[[package]] +name = "thinc" +version = "8.3.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0a/d4/2b0cd0fe285e14b36db076e78c93766ff1d529d70408bd1d2a5a84f1d929/tenacity-9.1.2.tar.gz", hash = "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb", size = 48036, upload-time = "2025-04-02T08:25:09.966Z" } +dependencies = [ + { name = "blis" }, + { name = "catalogue" }, + { name = "confection" }, + { name = "cymem" }, + { name = "murmurhash" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "preshed" }, + { name = "pydantic" }, + { name = "setuptools" }, + { name = "srsly" }, + { name = "wasabi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/ff/60c9bcfe28e56c905aac8e61a838c7afe5dc3073c9beed0b63a26ace0bb7/thinc-8.3.4.tar.gz", hash = "sha256:b5925482498bbb6dca0771e375b35c915818f735891e93d93a662dab15f6ffd8", size = 193903, upload-time = "2025-01-13T12:47:51.698Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/30/643397144bfbfec6f6ef821f36f33e57d35946c44a2352d3c9f0ae847619/tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138", size = 28248, upload-time = "2025-04-02T08:25:07.678Z" }, + { url = "https://files.pythonhosted.org/packages/85/47/68187c78a04cdc31cbd3ae393068f994b60476b5ecac6dfe7d04b124aacf/thinc-8.3.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a8bb4b47358a1855803b375f4432cefdf373f46ef249b554418d2e77c7323040", size = 839320, upload-time = "2025-01-13T12:47:12.317Z" }, + { url = "https://files.pythonhosted.org/packages/49/ea/066dd415e61fcef20083bbca41c2c02e640fea71326531f2619708efee1e/thinc-8.3.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:00ed92f9a34b9794f51fcd48467c863f4eb7c5b41559aef6ef3c980c21378fec", size = 774196, upload-time = "2025-01-13T12:47:15.315Z" }, + { url = "https://files.pythonhosted.org/packages/8c/68/36c1a92a374891e0d496677c59f5f9fdc1e57bbb214c487bb8bb3e9290c2/thinc-8.3.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85691fca84a6a1506f7ddbd2c1706a5524d56f65582e76b2e260a06d9e83e86d", size = 3922504, upload-time = "2025-01-13T12:47:22.07Z" }, + { url = "https://files.pythonhosted.org/packages/ec/8a/48e463240a586e91f83c87660986e520aa91fbd839f6631ee9bc0fbb3cbd/thinc-8.3.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:eae1573fc19e514defc1bfd4f93f0b4bfc1dcefdb6d70bad1863825747f24800", size = 4932946, upload-time = "2025-01-13T12:47:24.177Z" }, + { url = "https://files.pythonhosted.org/packages/d9/98/f910b8d8113ab9b955a68e9bbf0d5bd0e828f22dd6d3c226af6ec3970817/thinc-8.3.4-cp311-cp311-win_amd64.whl", hash = "sha256:81e8638f9bdc38e366674acc4b63cf7c6267266a15477963a5db21b3d9f1aa36", size = 1490133, upload-time = "2025-01-13T12:47:26.152Z" }, + { url = "https://files.pythonhosted.org/packages/90/ff/d1b5d7e1a7f95581e9a736f50a5a9aff72327ddbbc629a68070c36acefd9/thinc-8.3.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c9da6375b106df5186bd2bfd1273bc923c01ab7d482f8942e4ee528a28965c3a", size = 825099, upload-time = "2025-01-13T12:47:27.881Z" }, + { url = "https://files.pythonhosted.org/packages/ce/0b/d207c917886dc40671361de0880ec3ea0443a718aae9dbb0a50ac0849f92/thinc-8.3.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:07091c6b5faace50857c4cf0982204969d77388d0a6f156dd2442297dceeb838", size = 761024, upload-time = "2025-01-13T12:47:29.739Z" }, + { url = "https://files.pythonhosted.org/packages/4b/a3/3ec5e9d7cbebc3257b8223a3d188216b91ab6ec1e66b6fdd99d22394bc62/thinc-8.3.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd40ad71bcd8b1b9daa0462e1255b1c1e86e901c2fd773966601f44a95878032", size = 3710390, upload-time = "2025-01-13T12:47:33.019Z" }, + { url = "https://files.pythonhosted.org/packages/40/ee/955c74e4e6ff2f694c99dcbbf7be8d478a8868503aeb3474517277c07667/thinc-8.3.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eb10823b3a3f1c6440998b11bf9a3571dd859feaed0fdb510a1c1097d9dc6a86", size = 4731524, upload-time = "2025-01-13T12:47:35.203Z" }, + { url = "https://files.pythonhosted.org/packages/a4/44/3786431e5c1eeebed3d7a4c97122896ca6d4a502b03d02c2171c417052fd/thinc-8.3.4-cp312-cp312-win_amd64.whl", hash = "sha256:b5e5e7bf5dae142fd50ed9785971292c4aab4d9ed18e4947653b6a0584d5227c", size = 1455883, upload-time = "2025-01-13T12:47:36.914Z" }, ] [[package]] @@ -3225,26 +5013,66 @@ wheels = [ [[package]] name = "tiktoken" -version = "0.9.0" +version = "0.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "regex" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ea/cf/756fedf6981e82897f2d570dd25fa597eb3f4459068ae0572d7e888cfd6f/tiktoken-0.9.0.tar.gz", hash = "sha256:d02a5ca6a938e0490e1ff957bc48c8b078c88cb83977be1625b1fd8aac792c5d", size = 35991, upload-time = "2025-02-14T06:03:01.003Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/86/ad0155a37c4f310935d5ac0b1ccf9bdb635dcb906e0a9a26b616dd55825a/tiktoken-0.11.0.tar.gz", hash = "sha256:3c518641aee1c52247c2b97e74d8d07d780092af79d5911a6ab5e79359d9b06a", size = 37648, upload-time = "2025-08-08T23:58:08.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/91/912b459799a025d2842566fe1e902f7f50d54a1ce8a0f236ab36b5bd5846/tiktoken-0.11.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:4ae374c46afadad0f501046db3da1b36cd4dfbfa52af23c998773682446097cf", size = 1059743, upload-time = "2025-08-08T23:57:37.516Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e9/6faa6870489ce64f5f75dcf91512bf35af5864583aee8fcb0dcb593121f5/tiktoken-0.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:25a512ff25dc6c85b58f5dd4f3d8c674dc05f96b02d66cdacf628d26a4e4866b", size = 999334, upload-time = "2025-08-08T23:57:38.595Z" }, + { url = "https://files.pythonhosted.org/packages/a1/3e/a05d1547cf7db9dc75d1461cfa7b556a3b48e0516ec29dfc81d984a145f6/tiktoken-0.11.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2130127471e293d385179c1f3f9cd445070c0772be73cdafb7cec9a3684c0458", size = 1129402, upload-time = "2025-08-08T23:57:39.627Z" }, + { url = "https://files.pythonhosted.org/packages/34/9a/db7a86b829e05a01fd4daa492086f708e0a8b53952e1dbc9d380d2b03677/tiktoken-0.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21e43022bf2c33f733ea9b54f6a3f6b4354b909f5a73388fb1b9347ca54a069c", size = 1184046, upload-time = "2025-08-08T23:57:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/9d/bb/52edc8e078cf062ed749248f1454e9e5cfd09979baadb830b3940e522015/tiktoken-0.11.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:adb4e308eb64380dc70fa30493e21c93475eaa11669dea313b6bbf8210bfd013", size = 1244691, upload-time = "2025-08-08T23:57:42.251Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/884b6cd7ae2570ecdcaffa02b528522b18fef1cbbfdbcaa73799807d0d3b/tiktoken-0.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:ece6b76bfeeb61a125c44bbefdfccc279b5288e6007fbedc0d32bfec602df2f2", size = 884392, upload-time = "2025-08-08T23:57:43.628Z" }, + { url = "https://files.pythonhosted.org/packages/e7/9e/eceddeffc169fc75fe0fd4f38471309f11cb1906f9b8aa39be4f5817df65/tiktoken-0.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fd9e6b23e860973cf9526544e220b223c60badf5b62e80a33509d6d40e6c8f5d", size = 1055199, upload-time = "2025-08-08T23:57:45.076Z" }, + { url = "https://files.pythonhosted.org/packages/4f/cf/5f02bfefffdc6b54e5094d2897bc80efd43050e5b09b576fd85936ee54bf/tiktoken-0.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6a76d53cee2da71ee2731c9caa747398762bda19d7f92665e882fef229cb0b5b", size = 996655, upload-time = "2025-08-08T23:57:46.304Z" }, + { url = "https://files.pythonhosted.org/packages/65/8e/c769b45ef379bc360c9978c4f6914c79fd432400a6733a8afc7ed7b0726a/tiktoken-0.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ef72aab3ea240646e642413cb363b73869fed4e604dcfd69eec63dc54d603e8", size = 1128867, upload-time = "2025-08-08T23:57:47.438Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2d/4d77f6feb9292bfdd23d5813e442b3bba883f42d0ac78ef5fdc56873f756/tiktoken-0.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f929255c705efec7a28bf515e29dc74220b2f07544a8c81b8d69e8efc4578bd", size = 1183308, upload-time = "2025-08-08T23:57:48.566Z" }, + { url = "https://files.pythonhosted.org/packages/7a/65/7ff0a65d3bb0fc5a1fb6cc71b03e0f6e71a68c5eea230d1ff1ba3fd6df49/tiktoken-0.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:61f1d15822e4404953d499fd1dcc62817a12ae9fb1e4898033ec8fe3915fdf8e", size = 1244301, upload-time = "2025-08-08T23:57:49.642Z" }, + { url = "https://files.pythonhosted.org/packages/f5/6e/5b71578799b72e5bdcef206a214c3ce860d999d579a3b56e74a6c8989ee2/tiktoken-0.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:45927a71ab6643dfd3ef57d515a5db3d199137adf551f66453be098502838b0f", size = 884282, upload-time = "2025-08-08T23:57:50.759Z" }, +] + +[[package]] +name = "tldextract" +version = "5.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "idna" }, + { name = "requests" }, + { name = "requests-file" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/97/78/182641ea38e3cfd56e9c7b3c0d48a53d432eea755003aa544af96403d4ac/tldextract-5.3.0.tar.gz", hash = "sha256:b3d2b70a1594a0ecfa6967d57251527d58e00bb5a91a74387baa0d87a0678609", size = 128502, upload-time = "2025-04-22T06:19:37.491Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/7c/ea488ef48f2f544566947ced88541bc45fae9e0e422b2edbf165ee07da99/tldextract-5.3.0-py3-none-any.whl", hash = "sha256:f70f31d10b55c83993f55e91ecb7c5d84532a8972f22ec578ecfbe5ea2292db2", size = 107384, upload-time = "2025-04-22T06:19:36.304Z" }, +] + +[[package]] +name = "tokenizers" +version = "0.21.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/2f/402986d0823f8d7ca139d969af2917fefaa9b947d1fb32f6168c509f2492/tokenizers-0.21.4.tar.gz", hash = "sha256:fa23f85fbc9a02ec5c6978da172cdcbac23498c3ca9f3645c5c68740ac007880", size = 351253, upload-time = "2025-07-28T15:48:54.325Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/ae/4613a59a2a48e761c5161237fc850eb470b4bb93696db89da51b79a871f1/tiktoken-0.9.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:f32cc56168eac4851109e9b5d327637f15fd662aa30dd79f964b7c39fbadd26e", size = 1065987, upload-time = "2025-02-14T06:02:14.174Z" }, - { url = "https://files.pythonhosted.org/packages/3f/86/55d9d1f5b5a7e1164d0f1538a85529b5fcba2b105f92db3622e5d7de6522/tiktoken-0.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:45556bc41241e5294063508caf901bf92ba52d8ef9222023f83d2483a3055348", size = 1009155, upload-time = "2025-02-14T06:02:15.384Z" }, - { url = "https://files.pythonhosted.org/packages/03/58/01fb6240df083b7c1916d1dcb024e2b761213c95d576e9f780dfb5625a76/tiktoken-0.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03935988a91d6d3216e2ec7c645afbb3d870b37bcb67ada1943ec48678e7ee33", size = 1142898, upload-time = "2025-02-14T06:02:16.666Z" }, - { url = "https://files.pythonhosted.org/packages/b1/73/41591c525680cd460a6becf56c9b17468d3711b1df242c53d2c7b2183d16/tiktoken-0.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b3d80aad8d2c6b9238fc1a5524542087c52b860b10cbf952429ffb714bc1136", size = 1197535, upload-time = "2025-02-14T06:02:18.595Z" }, - { url = "https://files.pythonhosted.org/packages/7d/7c/1069f25521c8f01a1a182f362e5c8e0337907fae91b368b7da9c3e39b810/tiktoken-0.9.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b2a21133be05dc116b1d0372af051cd2c6aa1d2188250c9b553f9fa49301b336", size = 1259548, upload-time = "2025-02-14T06:02:20.729Z" }, - { url = "https://files.pythonhosted.org/packages/6f/07/c67ad1724b8e14e2b4c8cca04b15da158733ac60136879131db05dda7c30/tiktoken-0.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:11a20e67fdf58b0e2dea7b8654a288e481bb4fc0289d3ad21291f8d0849915fb", size = 893895, upload-time = "2025-02-14T06:02:22.67Z" }, - { url = "https://files.pythonhosted.org/packages/cf/e5/21ff33ecfa2101c1bb0f9b6df750553bd873b7fb532ce2cb276ff40b197f/tiktoken-0.9.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e88f121c1c22b726649ce67c089b90ddda8b9662545a8aeb03cfef15967ddd03", size = 1065073, upload-time = "2025-02-14T06:02:24.768Z" }, - { url = "https://files.pythonhosted.org/packages/8e/03/a95e7b4863ee9ceec1c55983e4cc9558bcfd8f4f80e19c4f8a99642f697d/tiktoken-0.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a6600660f2f72369acb13a57fb3e212434ed38b045fd8cc6cdd74947b4b5d210", size = 1008075, upload-time = "2025-02-14T06:02:26.92Z" }, - { url = "https://files.pythonhosted.org/packages/40/10/1305bb02a561595088235a513ec73e50b32e74364fef4de519da69bc8010/tiktoken-0.9.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:95e811743b5dfa74f4b227927ed86cbc57cad4df859cb3b643be797914e41794", size = 1140754, upload-time = "2025-02-14T06:02:28.124Z" }, - { url = "https://files.pythonhosted.org/packages/1b/40/da42522018ca496432ffd02793c3a72a739ac04c3794a4914570c9bb2925/tiktoken-0.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99376e1370d59bcf6935c933cb9ba64adc29033b7e73f5f7569f3aad86552b22", size = 1196678, upload-time = "2025-02-14T06:02:29.845Z" }, - { url = "https://files.pythonhosted.org/packages/5c/41/1e59dddaae270ba20187ceb8aa52c75b24ffc09f547233991d5fd822838b/tiktoken-0.9.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:badb947c32739fb6ddde173e14885fb3de4d32ab9d8c591cbd013c22b4c31dd2", size = 1259283, upload-time = "2025-02-14T06:02:33.838Z" }, - { url = "https://files.pythonhosted.org/packages/5b/64/b16003419a1d7728d0d8c0d56a4c24325e7b10a21a9dd1fc0f7115c02f0a/tiktoken-0.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:5a62d7a25225bafed786a524c1b9f0910a1128f4232615bf3f8257a73aaa3b16", size = 894897, upload-time = "2025-02-14T06:02:36.265Z" }, + { url = "https://files.pythonhosted.org/packages/98/c6/fdb6f72bf6454f52eb4a2510be7fb0f614e541a2554d6210e370d85efff4/tokenizers-0.21.4-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:2ccc10a7c3bcefe0f242867dc914fc1226ee44321eb618cfe3019b5df3400133", size = 2863987, upload-time = "2025-07-28T15:48:44.877Z" }, + { url = "https://files.pythonhosted.org/packages/8d/a6/28975479e35ddc751dc1ddc97b9b69bf7fcf074db31548aab37f8116674c/tokenizers-0.21.4-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:5e2f601a8e0cd5be5cc7506b20a79112370b9b3e9cb5f13f68ab11acd6ca7d60", size = 2732457, upload-time = "2025-07-28T15:48:43.265Z" }, + { url = "https://files.pythonhosted.org/packages/aa/8f/24f39d7b5c726b7b0be95dca04f344df278a3fe3a4deb15a975d194cbb32/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:39b376f5a1aee67b4d29032ee85511bbd1b99007ec735f7f35c8a2eb104eade5", size = 3012624, upload-time = "2025-07-28T13:22:43.895Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/26358925717687a58cb74d7a508de96649544fad5778f0cd9827398dc499/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2107ad649e2cda4488d41dfd031469e9da3fcbfd6183e74e4958fa729ffbf9c6", size = 2939681, upload-time = "2025-07-28T13:22:47.499Z" }, + { url = "https://files.pythonhosted.org/packages/99/6f/cc300fea5db2ab5ddc2c8aea5757a27b89c84469899710c3aeddc1d39801/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c73012da95afafdf235ba80047699df4384fdc481527448a078ffd00e45a7d9", size = 3247445, upload-time = "2025-07-28T15:48:39.711Z" }, + { url = "https://files.pythonhosted.org/packages/be/bf/98cb4b9c3c4afd8be89cfa6423704337dc20b73eb4180397a6e0d456c334/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f23186c40395fc390d27f519679a58023f368a0aad234af145e0f39ad1212732", size = 3428014, upload-time = "2025-07-28T13:22:49.569Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/96c1cc780e6ca7f01a57c13235dd05b7bc1c0f3588512ebe9d1331b5f5ae/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cc88bb34e23a54cc42713d6d98af5f1bf79c07653d24fe984d2d695ba2c922a2", size = 3193197, upload-time = "2025-07-28T13:22:51.471Z" }, + { url = "https://files.pythonhosted.org/packages/f2/90/273b6c7ec78af547694eddeea9e05de771278bd20476525ab930cecaf7d8/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51b7eabb104f46c1c50b486520555715457ae833d5aee9ff6ae853d1130506ff", size = 3115426, upload-time = "2025-07-28T15:48:41.439Z" }, + { url = "https://files.pythonhosted.org/packages/91/43/c640d5a07e95f1cf9d2c92501f20a25f179ac53a4f71e1489a3dcfcc67ee/tokenizers-0.21.4-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:714b05b2e1af1288bd1bc56ce496c4cebb64a20d158ee802887757791191e6e2", size = 9089127, upload-time = "2025-07-28T15:48:46.472Z" }, + { url = "https://files.pythonhosted.org/packages/44/a1/dd23edd6271d4dca788e5200a807b49ec3e6987815cd9d0a07ad9c96c7c2/tokenizers-0.21.4-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:1340ff877ceedfa937544b7d79f5b7becf33a4cfb58f89b3b49927004ef66f78", size = 9055243, upload-time = "2025-07-28T15:48:48.539Z" }, + { url = "https://files.pythonhosted.org/packages/21/2b/b410d6e9021c4b7ddb57248304dc817c4d4970b73b6ee343674914701197/tokenizers-0.21.4-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:3c1f4317576e465ac9ef0d165b247825a2a4078bcd01cba6b54b867bdf9fdd8b", size = 9298237, upload-time = "2025-07-28T15:48:50.443Z" }, + { url = "https://files.pythonhosted.org/packages/b7/0a/42348c995c67e2e6e5c89ffb9cfd68507cbaeb84ff39c49ee6e0a6dd0fd2/tokenizers-0.21.4-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:c212aa4e45ec0bb5274b16b6f31dd3f1c41944025c2358faaa5782c754e84c24", size = 9461980, upload-time = "2025-07-28T15:48:52.325Z" }, + { url = "https://files.pythonhosted.org/packages/3d/d3/dacccd834404cd71b5c334882f3ba40331ad2120e69ded32cf5fda9a7436/tokenizers-0.21.4-cp39-abi3-win32.whl", hash = "sha256:6c42a930bc5f4c47f4ea775c91de47d27910881902b0f20e4990ebe045a415d0", size = 2329871, upload-time = "2025-07-28T15:48:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/41/f2/fd673d979185f5dcbac4be7d09461cbb99751554ffb6718d0013af8604cb/tokenizers-0.21.4-cp39-abi3-win_amd64.whl", hash = "sha256:475d807a5c3eb72c59ad9b5fcdb254f6e17f53dfcbb9903233b0dfa9c943b597", size = 2507568, upload-time = "2025-07-28T15:48:55.456Z" }, ] [[package]] @@ -3294,6 +5122,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/03/98/eb27cc78ad3af8e302c9d8ff4977f5026676e130d28dd7578132a457170c/toolz-1.0.0-py3-none-any.whl", hash = "sha256:292c8f1c4e7516bf9086f8850935c799a874039c8bcf959d47b600e4c44a6236", size = 56383, upload-time = "2024-10-04T16:17:01.533Z" }, ] +[[package]] +name = "tornado" +version = "6.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/09/ce/1eb500eae19f4648281bb2186927bb062d2438c2e5093d1360391afd2f90/tornado-6.5.2.tar.gz", hash = "sha256:ab53c8f9a0fa351e2c0741284e06c7a45da86afb544133201c5cc8578eb076a0", size = 510821, upload-time = "2025-08-08T18:27:00.78Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/48/6a7529df2c9cc12efd2e8f5dd219516184d703b34c06786809670df5b3bd/tornado-6.5.2-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:2436822940d37cde62771cff8774f4f00b3c8024fe482e16ca8387b8a2724db6", size = 442563, upload-time = "2025-08-08T18:26:42.945Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b5/9b575a0ed3e50b00c40b08cbce82eb618229091d09f6d14bce80fc01cb0b/tornado-6.5.2-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:583a52c7aa94ee046854ba81d9ebb6c81ec0fd30386d96f7640c96dad45a03ef", size = 440729, upload-time = "2025-08-08T18:26:44.473Z" }, + { url = "https://files.pythonhosted.org/packages/1b/4e/619174f52b120efcf23633c817fd3fed867c30bff785e2cd5a53a70e483c/tornado-6.5.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0fe179f28d597deab2842b86ed4060deec7388f1fd9c1b4a41adf8af058907e", size = 444295, upload-time = "2025-08-08T18:26:46.021Z" }, + { url = "https://files.pythonhosted.org/packages/95/fa/87b41709552bbd393c85dd18e4e3499dcd8983f66e7972926db8d96aa065/tornado-6.5.2-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b186e85d1e3536d69583d2298423744740986018e393d0321df7340e71898882", size = 443644, upload-time = "2025-08-08T18:26:47.625Z" }, + { url = "https://files.pythonhosted.org/packages/f9/41/fb15f06e33d7430ca89420283a8762a4e6b8025b800ea51796ab5e6d9559/tornado-6.5.2-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e792706668c87709709c18b353da1f7662317b563ff69f00bab83595940c7108", size = 443878, upload-time = "2025-08-08T18:26:50.599Z" }, + { url = "https://files.pythonhosted.org/packages/11/92/fe6d57da897776ad2e01e279170ea8ae726755b045fe5ac73b75357a5a3f/tornado-6.5.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:06ceb1300fd70cb20e43b1ad8aaee0266e69e7ced38fa910ad2e03285009ce7c", size = 444549, upload-time = "2025-08-08T18:26:51.864Z" }, + { url = "https://files.pythonhosted.org/packages/9b/02/c8f4f6c9204526daf3d760f4aa555a7a33ad0e60843eac025ccfd6ff4a93/tornado-6.5.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:74db443e0f5251be86cbf37929f84d8c20c27a355dd452a5cfa2aada0d001ec4", size = 443973, upload-time = "2025-08-08T18:26:53.625Z" }, + { url = "https://files.pythonhosted.org/packages/ae/2d/f5f5707b655ce2317190183868cd0f6822a1121b4baeae509ceb9590d0bd/tornado-6.5.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b5e735ab2889d7ed33b32a459cac490eda71a1ba6857b0118de476ab6c366c04", size = 443954, upload-time = "2025-08-08T18:26:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/e8/59/593bd0f40f7355806bf6573b47b8c22f8e1374c9b6fd03114bd6b7a3dcfd/tornado-6.5.2-cp39-abi3-win32.whl", hash = "sha256:c6f29e94d9b37a95013bb669616352ddb82e3bfe8326fccee50583caebc8a5f0", size = 445023, upload-time = "2025-08-08T18:26:56.677Z" }, + { url = "https://files.pythonhosted.org/packages/c7/2a/f609b420c2f564a748a2d80ebfb2ee02a73ca80223af712fca591386cafb/tornado-6.5.2-cp39-abi3-win_amd64.whl", hash = "sha256:e56a5af51cc30dd2cae649429af65ca2f6571da29504a07995175df14c18f35f", size = 445427, upload-time = "2025-08-08T18:26:57.91Z" }, + { url = "https://files.pythonhosted.org/packages/5e/4f/e1f65e8f8c76d73658b33d33b81eed4322fb5085350e4328d5c956f0c8f9/tornado-6.5.2-cp39-abi3-win_arm64.whl", hash = "sha256:d6c33dc3672e3a1f3618eb63b7ef4683a7688e7b9e6e8f0d9aa5726360a004af", size = 444456, upload-time = "2025-08-08T18:26:59.207Z" }, +] + [[package]] name = "tqdm" version = "4.67.1" @@ -3367,13 +5214,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/47/58/0262e875dd899447476a8ffde7829df3716ffa772990095c65d6de1f053c/tree_sitter_languages-1.10.2-cp312-cp312-win_amd64.whl", hash = "sha256:5e606430d736367e5787fa5a7a0c5a1ec9b85eded0b3596bbc0d83532a40810b", size = 8268983, upload-time = "2024-02-04T10:29:00.987Z" }, ] +[[package]] +name = "typer" +version = "0.16.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "rich" }, + { name = "shellingham" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/78/d90f616bf5f88f8710ad067c1f8705bf7618059836ca084e5bb2a0855d75/typer-0.16.1.tar.gz", hash = "sha256:d358c65a464a7a90f338e3bb7ff0c74ac081449e53884b12ba658cbd72990614", size = 102836, upload-time = "2025-08-18T19:18:22.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/76/06dbe78f39b2203d2a47d5facc5df5102d0561e2807396471b5f7c5a30a1/typer-0.16.1-py3-none-any.whl", hash = "sha256:90ee01cb02d9b8395ae21ee3368421faf21fa138cb2a541ed369c08cec5237c9", size = 46397, upload-time = "2025-08-18T19:18:21.663Z" }, +] + [[package]] name = "typing-extensions" -version = "4.14.1" +version = "4.15.0rc1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/98/5a/da40306b885cc8c09109dc2e1abd358d5684b1425678151cdaed4731c822/typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36", size = 107673, upload-time = "2025-07-04T13:28:34.16Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/d5/1b2b1803ecd99e11a842d054c6b6f3f7938dde30d42b4d0d99611b2ee6fd/typing_extensions-4.15.0rc1.tar.gz", hash = "sha256:49b001798e59fbb7a523f0d36e8cf2d82d8e3f9fc41b04ff958da1ed7cc3b671", size = 109126, upload-time = "2025-08-18T14:31:09.022Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/00/d631e67a838026495268c2f6884f3711a15a9a2a96cd244fdaea53b823fb/typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76", size = 43906, upload-time = "2025-07-04T13:28:32.743Z" }, + { url = "https://files.pythonhosted.org/packages/11/61/1e821439fa89ca57c8a31e285742b7aeb719c8068e06074717b398642fb7/typing_extensions-4.15.0rc1-py3-none-any.whl", hash = "sha256:8fd4191376831cd3503df0cf06a0c0e6c1dae08ea3e6af770a785eeb90934dea", size = 44640, upload-time = "2025-08-18T14:31:07.386Z" }, ] [[package]] @@ -3438,6 +5300,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/59/2d/691f741ffd72b6c84438a93749ac57bf1a3f217ac4b0ea4fd0e96119e118/ujson-5.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:38665e7d8290188b1e0d57d584eb8110951a9591363316dd41cf8686ab1d0abc", size = 42211, upload-time = "2024-05-14T02:01:17.567Z" }, ] +[[package]] +name = "unidiff" +version = "0.7.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/48/81be0ac96e423a877754153699731ef439fd7b80b4c8b5425c94ed079ebd/unidiff-0.7.5.tar.gz", hash = "sha256:2e5f0162052248946b9f0970a40e9e124236bf86c82b70821143a6fc1dea2574", size = 20931, upload-time = "2023-03-10T01:05:39.185Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/54/57c411a6e8f7bd7848c8b66e4dcaffa586bf4c02e63f2280db0327a4e6eb/unidiff-0.7.5-py2.py3-none-any.whl", hash = "sha256:c93bf2265cc1ba2a520e415ab05da587370bc2a3ae9e0414329f54f0c2fc09e8", size = 14386, upload-time = "2023-03-10T01:05:36.594Z" }, +] + [[package]] name = "univers" version = "30.12.0" @@ -3525,15 +5396,21 @@ dependencies = [ { name = "aiofiles" }, { name = "aiohttp-client-cache" }, { name = "aioresponses" }, - { name = "aiqtoolkit", extra = ["langchain", "profiling", "telemetry"] }, { name = "brotli" }, + { name = "csaf-tool" }, + { name = "cvss" }, { name = "esprima" }, + { name = "exploit-iq-commons" }, { name = "faiss-cpu" }, { name = "gitpython" }, { name = "google-search-results" }, { name = "json5" }, + { name = "jsonschema" }, + { name = "koji" }, + { name = "litellm" }, { name = "nbformat" }, { name = "nemollm" }, + { name = "nvidia-nat", extra = ["langchain", "profiling", "telemetry"] }, { name = "openinference-instrumentation-langchain" }, { name = "ordered-set" }, { name = "pydpkg" }, @@ -3541,6 +5418,7 @@ dependencies = [ { name = "tantivy" }, { name = "tree-sitter" }, { name = "tree-sitter-languages" }, + { name = "unidiff" }, { name = "univers" }, ] @@ -3567,15 +5445,21 @@ requires-dist = [ { name = "aiofiles" }, { name = "aiohttp-client-cache", specifier = "==0.11" }, { name = "aioresponses", specifier = "==0.7.6" }, - { name = "aiqtoolkit", extras = ["langchain", "profiling", "telemetry"], specifier = ">=1.2.0a2,<1.3.0" }, { name = "brotli" }, + { name = "csaf-tool", specifier = "==0.3.2" }, + { name = "cvss", specifier = "==3.6" }, { name = "esprima" }, + { name = "exploit-iq-commons", editable = "src/exploit_iq_commons" }, { name = "faiss-cpu", specifier = "==1.9.0" }, { name = "gitpython" }, { name = "google-search-results", specifier = "==2.4" }, { name = "json5" }, + { name = "jsonschema", specifier = ">=4.0.0,<5.0.0" }, + { name = "koji" }, + { name = "litellm", specifier = "<=1.75.8" }, { name = "nbformat" }, { name = "nemollm" }, + { name = "nvidia-nat", extras = ["langchain", "profiling", "telemetry"], specifier = ">=1.2.0rc8,<1.3.0" }, { name = "openinference-instrumentation-langchain", specifier = "~=0.1.31" }, { name = "ordered-set" }, { name = "pydpkg", specifier = "==1.9.4" }, @@ -3583,6 +5467,7 @@ requires-dist = [ { name = "tantivy", specifier = "==0.22.2" }, { name = "tree-sitter", specifier = "==0.21.3" }, { name = "tree-sitter-languages", specifier = "==1.10.2" }, + { name = "unidiff", specifier = ">=0.7.5" }, { name = "univers", specifier = "==30.12" }, ] @@ -3595,6 +5480,7 @@ dev = [ { name = "pylint", specifier = "==3.3.*" }, { name = "pytest", specifier = "~=8.3" }, { name = "pytest-asyncio", specifier = "==0.24.*" }, + { name = "pytest-asyncio", specifier = "~=0.24.0" }, { name = "pytest-cov", specifier = "~=6.1" }, { name = "pytest-httpserver", specifier = "==1.1.*" }, { name = "pytest-pretty", specifier = "~=1.2.0" }, @@ -3604,6 +5490,47 @@ dev = [ { name = "yapf", specifier = "==0.43.*" }, ] +[[package]] +name = "wandb" +version = "0.21.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "gitpython" }, + { name = "packaging" }, + { name = "platformdirs" }, + { name = "protobuf" }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "sentry-sdk" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/69/217598886af89350e36bc05c092a67c9c469cff1fd6446edd4c879027e36/wandb-0.21.1.tar.gz", hash = "sha256:753bbdaa3a7703344056e019425b39c17a3d31d8ca0c4d13c4efc046935b08b9", size = 40131395, upload-time = "2025-08-07T18:52:48.85Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/65/d0/589f970741f3ead9ad28d4cbb668d1e6a39848df767f004ac9c7bed8f4b5/wandb-0.21.1-py3-none-macosx_10_14_x86_64.whl", hash = "sha256:96f9eedeae428de0d88f9751fb81f1b730ae7902f35c2f5a7a904d7733f124f3", size = 21701698, upload-time = "2025-08-07T18:52:22.399Z" }, + { url = "https://files.pythonhosted.org/packages/41/6c/a6140a0f395a99902aafdfe63088b7aff509e4f14cd7dd084d47eab36f27/wandb-0.21.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:41a1ec1b98d9d7e1bcafc483bce82e184b6cbae7531328a0fe8dd0f56d96a92e", size = 21221046, upload-time = "2025-08-07T18:52:26.134Z" }, + { url = "https://files.pythonhosted.org/packages/e9/d8/dacbb30ed35141d48a387d84f2e792d4b61b5bcdbf5ffdbd3f0b57beb346/wandb-0.21.1-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:f74d4691c38318ed8611e00ca3246b4152a03ff390fdce41816bea5705452a73", size = 21885803, upload-time = "2025-08-07T18:52:28.489Z" }, + { url = "https://files.pythonhosted.org/packages/b0/48/3a7290a33b1f64e29ac8779dab4d4cdef31a9ed3c3d9ea656a4507d64332/wandb-0.21.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c8fbd60b9abf4b9bec201f311602f61394d41a3503c801750b03975a5e36d1b", size = 20825318, upload-time = "2025-08-07T18:52:31.282Z" }, + { url = "https://files.pythonhosted.org/packages/a9/54/c0a087114ff1bb6c32e64aaa58aea4342cebc0ad58b1378c0a5a831d2508/wandb-0.21.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ded9313672630c0630f5b13c598ce9aa0e932e811ebc18823fcc4d73acfb6bb", size = 22362500, upload-time = "2025-08-07T18:52:33.889Z" }, + { url = "https://files.pythonhosted.org/packages/65/68/3aae277ea9fb5d91eec066cf256755bed3a740d92b539888a7ce36cf3f6c/wandb-0.21.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:44f3194d697b409f91708c50c5f9d56e282434a0d60ac380b64f0fb6991cd630", size = 20830372, upload-time = "2025-08-07T18:52:36.76Z" }, + { url = "https://files.pythonhosted.org/packages/d2/bb/58d206e79be1f279ef06cb934ae1e208bcacd2cd73b7a7652236575010d6/wandb-0.21.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e0b68bb6dbe94f1910c665c755f438292df40c272feb1a8b42208c1df52cce26", size = 22438521, upload-time = "2025-08-07T18:52:39.672Z" }, + { url = "https://files.pythonhosted.org/packages/e7/b8/dfe01f8e4c40d5dda820fd839c39431608a3453670f79404fa28915972d2/wandb-0.21.1-py3-none-win32.whl", hash = "sha256:98306c3fb369dfafb7194270b938b000ea2bb08dbddff10c19b5a805fd5cab80", size = 21569814, upload-time = "2025-08-07T18:52:42.58Z" }, + { url = "https://files.pythonhosted.org/packages/51/ba/81c77d5d831fcddb89661c85175fcbb91d2ffecf6b0591972829da3eb42f/wandb-0.21.1-py3-none-win_amd64.whl", hash = "sha256:8be92a7e92b5cb5ce00ec0961f9dbaad7757ffdbc5b5a8f2cc7188e23f653f0a", size = 21569817, upload-time = "2025-08-07T18:52:45.559Z" }, +] + +[[package]] +name = "wasabi" +version = "1.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/f9/054e6e2f1071e963b5e746b48d1e3727470b2a490834d18ad92364929db3/wasabi-1.1.3.tar.gz", hash = "sha256:4bb3008f003809db0c3e28b4daf20906ea871a2bb43f9914197d540f4f2e0878", size = 30391, upload-time = "2024-05-31T16:56:18.99Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/7c/34330a89da55610daa5f245ddce5aab81244321101614751e7537f125133/wasabi-1.1.3-py3-none-any.whl", hash = "sha256:f76e16e8f7e79f8c4c8be49b4024ac725713ab10cd7f19350ad18a8e3f71728c", size = 27880, upload-time = "2024-05-31T16:56:16.699Z" }, +] + [[package]] name = "watchfiles" version = "1.1.0" @@ -3645,6 +5572,59 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bd/d3/254cea30f918f489db09d6a8435a7de7047f8cb68584477a515f160541d6/watchfiles-1.1.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:923fec6e5461c42bd7e3fd5ec37492c6f3468be0499bc0707b4bbbc16ac21792", size = 454009, upload-time = "2025-06-15T19:06:52.896Z" }, ] +[[package]] +name = "wcwidth" +version = "0.2.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/63/53559446a878410fc5a5974feb13d31d78d752eb18aeba59c7fef1af7598/wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5", size = 101301, upload-time = "2024-01-06T02:10:57.829Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859", size = 34166, upload-time = "2024-01-06T02:10:55.763Z" }, +] + +[[package]] +name = "weasel" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cloudpathlib" }, + { name = "confection" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "smart-open" }, + { name = "srsly" }, + { name = "typer" }, + { name = "wasabi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/1a/9c522dd61b52939c217925d3e55c95f9348b73a66a956f52608e1e59a2c0/weasel-0.4.1.tar.gz", hash = "sha256:aabc210f072e13f6744e5c3a28037f93702433405cd35673f7c6279147085aa9", size = 38417, upload-time = "2024-05-15T08:52:54.765Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/87/abd57374044e1f627f0a905ac33c1a7daab35a3a815abfea4e1bafd3fdb1/weasel-0.4.1-py3-none-any.whl", hash = "sha256:24140a090ea1ac512a2b2f479cc64192fd1d527a7f3627671268d08ed5ac418c", size = 50270, upload-time = "2024-05-15T08:52:52.977Z" }, +] + +[[package]] +name = "weave" +version = "0.51.59" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "diskcache" }, + { name = "eval-type-backport" }, + { name = "gql", extra = ["aiohttp", "requests"] }, + { name = "jsonschema" }, + { name = "nest-asyncio" }, + { name = "packaging" }, + { name = "polyfile-weave" }, + { name = "pydantic" }, + { name = "rich" }, + { name = "sentry-sdk" }, + { name = "tenacity" }, + { name = "wandb" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/53/1b0350a64837df3e29eda6149a542f3a51e706122086f82547153820e982/weave-0.51.59.tar.gz", hash = "sha256:fad34c0478f3470401274cba8fa2bfd45d14a187db0a5724bd507e356761b349", size = 480572, upload-time = "2025-07-25T22:05:07.458Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1d/bc/fa5ffb887a1ee28109b29c62416c9e0f41da8e75e6871671208b3d42b392/weave-0.51.59-py3-none-any.whl", hash = "sha256:2238578574ecdf6285efdf028c78987769720242ac75b7b84b1dbc59060468ce", size = 612468, upload-time = "2025-07-25T22:05:05.088Z" }, +] + [[package]] name = "websockets" version = "15.0.1"