diff --git a/.agents/skills/toolset-design/SKILL.md b/.agents/skills/toolset-design/SKILL.md index d05cc84aa..eb3c4e3ef 100644 --- a/.agents/skills/toolset-design/SKILL.md +++ b/.agents/skills/toolset-design/SKILL.md @@ -64,7 +64,7 @@ prompt: ```bash # Run evals with only core+config toolsets to establish baseline -mcpchecker check evals/claude-code/eval.yaml --label-selector suite= +mcpchecker check evals/core-eval-testing//eval-core.yaml --label-selector suite= ``` **What to look for in baseline results:** @@ -252,7 +252,10 @@ Then run the evals again. Compare against the baseline: - Does the LLM use the new tools correctly, or does it still fall back to `pods_exec`? - Are there tools you added that the LLM never uses? Those might not be needed. -Add eval task entries to the agent config in `evals/claude-code/eval.yaml` and `evals/openai-agent/eval.yaml` +Each toolset gets its own suite (e.g. `suite: my-domain`) and a corresponding +`eval-.yaml` per agent. Create `evals/core-eval-testing//eval-my-domain.yaml` +for each agent that has eval configs. The eval file should include the `core` and +`config` task sets alongside the new suite's tasks (see existing files for the pattern). --- @@ -267,7 +270,7 @@ See `CLAUDE.md` for full details on each step. The key files to touch: 4. **Snapshot test**: `pkg/mcp/toolsets_test.go` -- add to `TestGranularToolsetsTools` 5. **Generate snapshots**: `UPDATE_TOOLSETS_JSON=1 go test -count=1 -v ./pkg/mcp` 6. **Update docs**: `make update-readme-tools` -7. **Eval configs**: `evals/claude-code/eval.yaml`, `evals/openai-agent/eval.yaml` +7. **Eval configs**: `evals/core-eval-testing//eval-.yaml` (one per toolset per agent) 8. **Eval tasks**: `evals/tasks//*/task.yaml` --- @@ -285,4 +288,4 @@ When reviewing a toolset PR, verify: - [ ] **Results use NewToolCallResultStructured**: Not manual `json.Marshal` + `NewToolCallResult` - [ ] **Container targeting**: If exec-ing into multi-container pods, is the correct container selected? - [ ] **Snapshot tests added**: Is the toolset in `TestGranularToolsetsTools` with a snapshot file? -- [ ] **All eval configs updated**: claude-code, openai-agent +- [ ] **All eval configs updated**: `evals/core-eval-testing//eval-.yaml` (one per toolset per agent) diff --git a/.github/workflows/mcpchecker-report.yaml b/.github/workflows/mcpchecker-report.yaml index 574a68b0a..6efbe97b1 100644 --- a/.github/workflows/mcpchecker-report.yaml +++ b/.github/workflows/mcpchecker-report.yaml @@ -23,7 +23,7 @@ jobs: WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }} run: | JOB_CONCLUSION=$(gh api "repos/${REPOSITORY}/actions/runs/${WORKFLOW_RUN_ID}/jobs" \ - --jq '.jobs[] | select(.name == "Run MCP Evaluation") | .conclusion') + --jq '[.jobs[] | select(.name | startswith("Run MCP Evaluation")) | .conclusion] | first') if [[ -n "$JOB_CONCLUSION" && "$JOB_CONCLUSION" != "skipped" ]]; then echo "should-report=true" >> $GITHUB_OUTPUT @@ -36,7 +36,8 @@ jobs: if: steps.check.outputs.should-report == 'true' uses: actions/download-artifact@v8 with: - name: eval-context + pattern: eval-context-* + merge-multiple: true path: eval-context/ run-id: ${{ github.event.workflow_run.id }} github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/mcpchecker.yaml b/.github/workflows/mcpchecker.yaml index fda25cf49..60b0b726f 100644 --- a/.github/workflows/mcpchecker.yaml +++ b/.github/workflows/mcpchecker.yaml @@ -76,8 +76,7 @@ jobs: github.event.workflow_run.conclusion == 'success')) outputs: should-run: ${{ steps.check.outputs.should-run }} - toolsets: ${{ steps.suite.outputs.toolsets }} - label-selector: ${{ steps.suite.outputs.label-selector }} + matrix: ${{ steps.suite.outputs.matrix }} pr-number: ${{ steps.check.outputs.pr-number }} pr-sha: ${{ steps.check.outputs.pr-sha }} is-pr: ${{ steps.check.outputs.is-pr }} @@ -206,54 +205,75 @@ jobs: INPUT_SUITE: ${{ github.event.inputs.suite }} run: | # Suite selection: for workflow_run use the suite parsed from the review - # body (verified via API), for workflow_dispatch use the input, otherwise - # default to 'core'. + # body (verified via API), for workflow_dispatch use the input, for + # schedule run all suites in parallel, otherwise default to 'core'. if [[ "$EVENT_NAME" == "workflow_run" && -n "$VERIFIED_SUITE" ]]; then SUITE="$VERIFIED_SUITE" + elif [[ "$EVENT_NAME" == "schedule" ]]; then + SUITE="schedule-matrix" else SUITE="${INPUT_SUITE:-core}" fi - # Select label-selector and infrastructure based on suite. - # All suites use the same eval.yaml file; suite controls label-selector + infra. + # Build a JSON matrix consumed by run-evaluation via fromJSON(). + # Each entry carries suite name, eval config path, and required toolsets. + EVAL_DIR="evals/core-eval-testing/builtin-openai" + + # Helper: emit a single-entry matrix for the given suite/config/toolsets. + single_matrix() { + printf '{"include":[{"suite":"%s","eval-config":"%s","toolsets":"%s"}]}' "$1" "$2" "$3" + } + case "$SUITE" in + schedule-matrix) + # Weekly schedule: run all individual suites in parallel. + MATRIX=$(cat <<'EOFM' + {"include":[ + {"suite":"core","eval-config":"evals/core-eval-testing/builtin-openai/eval-core.yaml","toolsets":"core,config"}, + {"suite":"helm","eval-config":"evals/core-eval-testing/builtin-openai/eval-helm.yaml","toolsets":"core,config,helm"}, + {"suite":"kubevirt","eval-config":"evals/core-eval-testing/builtin-openai/eval-kubevirt.yaml","toolsets":"core,config,kubevirt,tekton"}, + {"suite":"kiali","eval-config":"evals/core-eval-testing/builtin-openai/eval-kiali.yaml","toolsets":"core,config,kiali"}, + {"suite":"tekton","eval-config":"evals/core-eval-testing/builtin-openai/eval-tekton.yaml","toolsets":"core,config,tekton"}, + {"suite":"netobserv","eval-config":"evals/core-eval-testing/builtin-openai/eval-netobserv.yaml","toolsets":"core,config,netobserv"} + ]} + EOFM + ) + ;; helm) - echo "label-selector=suite=helm" >> "$GITHUB_OUTPUT" - echo "toolsets=core,config,helm" >> "$GITHUB_OUTPUT" + MATRIX=$(single_matrix helm "${EVAL_DIR}/eval-helm.yaml" "core,config,helm") ;; kubevirt) - echo "label-selector=suite=kubevirt" >> "$GITHUB_OUTPUT" - echo "toolsets=core,config,kubevirt,tekton" >> "$GITHUB_OUTPUT" + MATRIX=$(single_matrix kubevirt "${EVAL_DIR}/eval-kubevirt.yaml" "core,config,kubevirt,tekton") ;; kiali) - echo "label-selector=suite=kiali" >> "$GITHUB_OUTPUT" - echo "toolsets=core,config,kiali" >> "$GITHUB_OUTPUT" + MATRIX=$(single_matrix kiali "${EVAL_DIR}/eval-kiali.yaml" "core,config,kiali") ;; tekton) - echo "label-selector=suite=tekton" >> "$GITHUB_OUTPUT" - echo "toolsets=core,tekton" >> "$GITHUB_OUTPUT" + MATRIX=$(single_matrix tekton "${EVAL_DIR}/eval-tekton.yaml" "core,config,tekton") ;; netobserv) - echo "label-selector=suite=netobserv" >> "$GITHUB_OUTPUT" - echo "toolsets=core,netobserv" >> "$GITHUB_OUTPUT" + MATRIX=$(single_matrix netobserv "${EVAL_DIR}/eval-netobserv.yaml" "core,config,netobserv") ;; all) - echo "label-selector=" >> "$GITHUB_OUTPUT" # No filter: run all taskSets - echo "toolsets=core,config,helm,kiali,kubevirt,tekton,netobserv" >> "$GITHUB_OUTPUT" + MATRIX=$(single_matrix all "${EVAL_DIR}/eval-all.yaml" "core,config,helm,kiali,kubevirt,tekton,netobserv") ;; *) # Default to core suite (matches 'core' or any unrecognized suite) - echo "label-selector=suite=core" >> "$GITHUB_OUTPUT" - echo "toolsets=core,config" >> "$GITHUB_OUTPUT" + MATRIX=$(single_matrix core "${EVAL_DIR}/eval-core.yaml" "core,config") ;; esac + echo "matrix=$(echo "$MATRIX" | jq -c .)" >> "$GITHUB_OUTPUT" + # Run mcpchecker evaluation with Kind cluster run-evaluation: - name: Run MCP Evaluation + name: 'Run MCP Evaluation (${{ matrix.suite }})' needs: check-trigger if: needs.check-trigger.outputs.should-run == 'true' runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.check-trigger.outputs.matrix) }} steps: - name: Checkout uses: actions/checkout@v7.0.1 @@ -277,46 +297,44 @@ jobs: - name: Setup Kind cluster run: make kind-create-cluster KIND_CLUSTER_NAME="$KIND_CLUSTER_NAME" - # Only the core suite needs a resize-capable StorageClass (for resize-pvc). - # Runs for suite=core and for the "all" run (empty label-selector). + # Core tasks always run (every eval includes them) and the resize-pvc + # task needs a resize-capable StorageClass, so always install the CSI driver. - name: Install CSI hostPath driver (resize-capable StorageClass) - if: needs.check-trigger.outputs.label-selector == 'suite=core' || needs.check-trigger.outputs.label-selector == '' run: make csi-hostpath-install - name: Install Istio/Kiali and bookinfo demo - if: contains(needs.check-trigger.outputs.toolsets, 'kiali') + if: contains(matrix.toolsets, 'kiali') run: make setup-kiali - name: Install KubeVirt - if: contains(needs.check-trigger.outputs.toolsets, 'kubevirt') + if: contains(matrix.toolsets, 'kubevirt') run: make kubevirt-install - name: Install Tekton - if: contains(needs.check-trigger.outputs.toolsets, 'tekton') + if: contains(matrix.toolsets, 'tekton') run: make tekton-install - name: Install NetObserv mock plugin - if: contains(needs.check-trigger.outputs.toolsets, 'netobserv') + if: contains(matrix.toolsets, 'netobserv') run: make setup-netobserv - name: Start MCP server run: make run-server env: - TOOLSETS: ${{ needs.check-trigger.outputs.toolsets }} + TOOLSETS: ${{ matrix.toolsets }} MCP_CONFIG_DIR: 'dev/config/mcp-configs' - name: Run mcpchecker evaluation id: mcpchecker uses: mcpchecker/mcpchecker/.github/actions/mcpchecker-action@v0.0.19 with: - eval-config: 'evals/openai-agent/eval.yaml' + eval-config: ${{ matrix.eval-config }} mcpchecker-version: 'latest' - label-selector: ${{ needs.check-trigger.outputs.label-selector }} task-filter: ${{ github.event.inputs.task-filter || '' }} output-format: 'json' verbose: ${{ github.event.inputs.verbose || 'false' }} upload-artifacts: 'true' - artifact-name: 'mcpchecker-results' + artifact-name: 'mcpchecker-results-${{ matrix.suite }}' fail-on-error: 'false' task-pass-threshold: '0.8' assertion-pass-threshold: '0.8' @@ -342,12 +360,13 @@ jobs: - name: Fetch baseline results from main if: needs.check-trigger.outputs.is-pr == 'true' env: - EVAL_CONFIG: 'evals/openai-agent/eval.yaml' + EVAL_CONFIG: ${{ matrix.eval-config }} run: | - # Derive agent name from eval-config path (same logic as commit-results job) - AGENT_NAME=$(echo "$EVAL_CONFIG" | sed 's|evals/||; s|/eval\.yaml||') + # Derive result name from eval-config path: eval-core.yaml -> builtin-openai-core + SUITE=$(basename "$EVAL_CONFIG" .yaml | sed 's/^eval-//') + RESULT_NAME="builtin-openai-${SUITE}" git fetch origin main --depth=1 - git show "origin/main:evals/results/${AGENT_NAME}-latest.json" > /tmp/baseline-results.json 2>/dev/null || true + git show "origin/main:evals/results/${RESULT_NAME}-latest.json" > /tmp/baseline-results.json 2>/dev/null || true - name: Check diff prerequisites id: diff-check @@ -413,7 +432,7 @@ jobs: if: always() && needs.check-trigger.outputs.is-pr == 'true' uses: actions/upload-artifact@v7 with: - name: eval-context + name: eval-context-${{ matrix.suite }} path: eval-context/ retention-days: 1 @@ -421,8 +440,9 @@ jobs: commit-results: name: Commit Evaluation Results needs: [check-trigger, run-evaluation] - # Only commit results on scheduled runs, not manual dispatch or PR comments - if: always() && github.event_name == 'schedule' && needs.run-evaluation.result == 'success' + # Only commit results on scheduled runs. Use != 'cancelled' so partial + # results from passing suites are still committed when some matrix jobs fail. + if: always() && github.event_name == 'schedule' && needs.run-evaluation.result != 'cancelled' runs-on: ubuntu-latest permissions: contents: write @@ -436,24 +456,31 @@ jobs: - name: Download mcpchecker results uses: actions/download-artifact@v8 with: - name: mcpchecker-results + pattern: mcpchecker-results-* path: mcpchecker-results/ - name: Copy results to evals/results - env: - EVAL_CONFIG: 'evals/openai-agent/eval.yaml' run: | - # Extract agent name from eval-config path - AGENT_NAME=$(echo "$EVAL_CONFIG" | sed 's|evals/||; s|/eval\.yaml||') - mkdir -p evals/results - # Copy the most recent mcpchecker results file with agent-specific name - RESULTS_FILE=$(ls -t mcpchecker-results/mcpchecker-*-out.json 2>/dev/null | head -1) - if [ -z "$RESULTS_FILE" ]; then - echo "Error: No mcpchecker results file found" - exit 1 - fi - cp "$RESULTS_FILE" "evals/results/${AGENT_NAME}-latest.json" + + # Each matrix job uploaded its own artifact (e.g. mcpchecker-results-core). + # Without merge-multiple, each lands in its own subdirectory. + for SUITE_DIR in mcpchecker-results/mcpchecker-results-*; do + [ -d "$SUITE_DIR" ] || continue + + # Extract suite name: mcpchecker-results-core -> core + SUITE=$(basename "$SUITE_DIR" | sed 's/^mcpchecker-results-//') + RESULT_NAME="builtin-openai-${SUITE}" + + RESULTS_FILE=$(ls -t "${SUITE_DIR}"/mcpchecker-*-out.json 2>/dev/null | head -1) + if [ -z "$RESULTS_FILE" ]; then + echo "Warning: No results file found for suite ${SUITE}, skipping" + continue + fi + + cp "$RESULTS_FILE" "evals/results/${RESULT_NAME}-latest.json" + echo "Copied ${SUITE} results to evals/results/${RESULT_NAME}-latest.json" + done - name: Create Pull Request env: diff --git a/build/evals.mk b/build/evals.mk index 6e35bde70..64a3a1197 100644 --- a/build/evals.mk +++ b/build/evals.mk @@ -11,17 +11,18 @@ CLAUDE_AGENT_ACP = $(shell pwd)/_output/tools/node_modules/.bin/claude-agent-acp CLAUDE_AGENT_ACP_VERSION ?= latest # High-level knobs for local single-suite runs, e.g.: -# make run-evals SUITE=kubevirt AGENT=claude-code MODEL=sonnet -# AGENT selects the eval config, SUITE selects the task suite label, and MODEL -# sets ANTHROPIC_MODEL for the claude-agent-acp adapter (the openai-agent ignores it). -AGENT ?= openai-agent +# make run-evals SUITE=kubevirt AGENT=acp-anthropic MODEL=sonnet +# AGENT selects the agent directory under evals/, SUITE selects the task suite +# label, and MODEL sets ANTHROPIC_MODEL for ACP agents (builtin agents ignore it). +# Available agents: builtin-openai, builtin-anthropic, builtin-google, +# acp-anthropic (Claude Code via ACP), acp-google (Gemini via ACP) +AGENT ?= builtin-openai SUITE ?= core MODEL ?= -# Prefer a per-suite eval config when one exists: those carry no llmJudge, so a -# local run needs no OpenAI key. Otherwise fall back to the agent's top-level -# config (the one CI uses; its llmJudge requires an OpenAI key). -EVAL_CONFIG ?= $(or $(wildcard evals/tasks/$(SUITE)/$(AGENT)/eval.yaml),evals/$(AGENT)/eval.yaml) +# Prefer a per-suite eval config when one exists, then try the core-eval-testing +# suite config (what CI uses). +EVAL_CONFIG ?= $(or $(wildcard evals/tasks/$(SUITE)/$(AGENT)/eval.yaml),evals/core-eval-testing/$(AGENT)/eval-$(SUITE).yaml) EVAL_LABEL_SELECTOR ?= suite=$(SUITE) EVAL_TASK_FILTER ?= EVAL_VERBOSE ?= false @@ -39,9 +40,9 @@ mcpchecker: ##@ Evals # Install the claude-agent-acp adapter locally under _output/tools, required by -# the claude-code eval agent (evals/claude-code/agent.yaml runs `claude-agent-acp`). +# the acp-anthropic eval agent (runs `claude-agent-acp`). .PHONY: claude-agent-acp -claude-agent-acp: ## Install the claude-agent-acp adapter for the claude-code eval agent +claude-agent-acp: ## Install the claude-agent-acp adapter for the acp-anthropic eval agent @[ -f $(CLAUDE_AGENT_ACP) ] || { \ set -e ;\ echo "Installing claude-agent-acp@$(CLAUDE_AGENT_ACP_VERSION) to $(CLAUDE_AGENT_ACP)..." ;\ @@ -50,7 +51,7 @@ claude-agent-acp: ## Install the claude-agent-acp adapter for the claude-code ev } .PHONY: run-evals -run-evals: mcpchecker $(if $(filter claude-code,$(AGENT)),claude-agent-acp) ## Run mcpchecker evals (knobs: SUITE, AGENT, MODEL; see evals/README.md) +run-evals: mcpchecker $(if $(filter acp-anthropic,$(AGENT)),claude-agent-acp) ## Run mcpchecker evals (knobs: SUITE, AGENT, MODEL; see evals/README.md) $(if $(MODEL),ANTHROPIC_MODEL=$(MODEL) )PATH="$(shell pwd)/_output/tools/node_modules/.bin:$(PATH)" $(MCPCHECKER) check $(EVAL_CONFIG) \ $(if $(EVAL_LABEL_SELECTOR),--label-selector $(EVAL_LABEL_SELECTOR),) \ $(if $(EVAL_TASK_FILTER),--run "$(EVAL_TASK_FILTER)",) \ diff --git a/evals/README.md b/evals/README.md index 9cb7d9e47..4b45b04b0 100644 --- a/evals/README.md +++ b/evals/README.md @@ -11,9 +11,7 @@ and others), so agents can be compared on an identical benchmark. evals/ ├── README.md # This file ├── mcp-config.yaml # Shared MCP server connection (http://localhost:8080/mcp) -├── claude-code/ # Claude Code agent config (ACP) + top-level eval.yaml -├── openai-agent/ # OpenAI-compatible agent config + top-level eval.yaml -├── core-eval-testing/ # Extra agent/provider combinations for core tasks +├── core-eval-testing/ # Agent configs and per-suite eval configs (see below) ├── results/ # Committed eval results (baselines) └── tasks/ # Shared task library, grouped by suite (see tasks/README.md) └── / # core, config, helm, kiali, kubevirt, tekton, netobserv, observability @@ -35,22 +33,22 @@ Tasks are grouped into suites via a `suite: ` label. The available suites - `mcpchecker`: `make mcpchecker` installs it under `_output/tools/bin/`. The `make run-evals` target calls that path directly; to run the bare `mcpchecker …` commands shown below by hand, add `_output/tools/bin` to your `PATH`. -- For the **Claude Code** agent only: +- For the **`acp-anthropic`** agent only: - The `claude` CLI (Claude Code) installed, on your `PATH`, and **signed in**. The agent runs through your existing Claude Code session, so your Claude subscription (or an `ANTHROPIC_API_KEY`) covers the agent and no separate key is needed. Run `claude` once interactively to log in if you have not already. - The `claude-agent-acp` adapter, which bridges mcpchecker to that `claude` CLI, - installed with `make claude-agent-acp` (installs locally under - `_output/tools/node_modules/`, so it needs Node.js/npm). `make run-evals` - prepends `_output/tools/node_modules/.bin` to `PATH` automatically. + installed with `make claude-agent-acp` (installs locally under + `_output/tools/node_modules/`, so it needs Node.js/npm). `make run-evals` + prepends `_output/tools/node_modules/.bin` to `PATH` automatically. -## Quickstart: run one suite locally with Claude Code +## Quickstart: run one suite locally with Claude Code (ACP) -This runs the `kubevirt` suite end to end with the Claude Code agent on a Sonnet -model, with **no OpenAI key required**. With the Claude Code agent both the agent -and the LLM judge run on your Claude subscription, so every suite is keyless (see -[Eval configs](#eval-configs-top-level-vs-per-suite) below). +This runs the `kubevirt` suite end to end with the `acp-anthropic` agent on a +Sonnet model, with **no OpenAI key required**. With this agent both the agent and +the LLM judge run on your Claude subscription, so every suite is keyless (see +[Eval configs](#eval-configs) below). ```bash # 1. Build the server and install the eval tooling @@ -71,8 +69,8 @@ make kubevirt-install make run-server TOOLSETS=core,config,kubevirt # 5. Run the suite. AGENT selects the agent, SUITE selects the tasks, MODEL sets -# ANTHROPIC_MODEL for the Claude Code agent. Add EVAL_TASK_FILTER= to run one task. -make run-evals SUITE=kubevirt AGENT=claude-code MODEL=sonnet +# ANTHROPIC_MODEL for ACP agents. Add EVAL_TASK_FILTER= to run one task. +make run-evals SUITE=kubevirt AGENT=acp-anthropic MODEL=sonnet # 6. Tear down make stop-server @@ -88,116 +86,105 @@ The make-target invocation in step 5 is equivalent to the underlying command: ```bash KUBECONFIG="$PWD/_output/kubeconfig" ANTHROPIC_MODEL=sonnet mcpchecker check \ - evals/tasks/kubevirt/claude-code/eval.yaml \ + evals/core-eval-testing/acp-anthropic/eval-kubevirt.yaml \ --label-selector suite=kubevirt --output json ``` ### Model selection -The model for the Claude Code agent is chosen with the `ANTHROPIC_MODEL` +The model for the `acp-anthropic` agent is chosen with the `ANTHROPIC_MODEL` environment variable (for example `ANTHROPIC_MODEL=sonnet`), which `claude-agent-acp` reads as its first-priority model source. It **forces** that model for the run, overriding whatever default your Claude Code session would otherwise use, and it runs on your existing Claude subscription (no API key needed). `make run-evals` sets it for you from the `MODEL` variable, so `make run-evals ... MODEL=sonnet` is how you pin the eval to Sonnet. The same -variable also drives the Claude Code judge (see [Eval configs](#eval-configs-top-level-vs-per-suite)), -so agent and judge share the model. +variable also drives the judge (see [Eval configs](#eval-configs)), so agent and +judge share the model. -The OpenAI-compatible agent instead takes its model from -`evals/openai-agent/agent.yaml` plus the `MODEL_BASE_URL` and `MODEL_KEY` -environment variables. +The `builtin-*` agents instead take their model from their `agent.yaml` plus the +`MODEL_BASE_URL` and `MODEL_KEY` environment variables. -## Running with the OpenAI-compatible agent +## Running with a builtin agent ```bash # Set your model credentials export MODEL_BASE_URL='https://your-api-endpoint.com/v1' export MODEL_KEY='your-api-key' -# Run the core suite (this is what CI runs) -make run-evals SUITE=core AGENT=openai-agent +# Run the core suite with builtin-openai (this is what CI runs) +make run-evals SUITE=core AGENT=builtin-openai # equivalently: -mcpchecker check evals/openai-agent/eval.yaml --label-selector suite=core +mcpchecker check evals/core-eval-testing/builtin-openai/eval-core.yaml --label-selector suite=core + +# Or use a different builtin agent: +make run-evals SUITE=core AGENT=builtin-anthropic +make run-evals SUITE=core AGENT=builtin-google ``` Different models may pick different tools (`pods_*` or `resources_*`) for the same task. The assertions accept either, for example `toolPattern: "(pods_.*|resources_.*)"`. -## Eval configs: top-level vs per-suite +## Eval configs + +`make run-evals` resolves the eval config in priority order: -There are two top-level eval configs, one per agent: +1. **Per-suite override** (`evals/tasks///eval.yaml`) — takes + precedence when present. No suite currently ships one; this is a hook for + dropping in a one-off local override. +2. **Core-eval-testing** (`evals/core-eval-testing//eval-.yaml`) — + the canonical per-suite config for each agent, and what every suite resolves to + today. **CI** runs `builtin-openai` here. Available agents: `builtin-openai`, + `builtin-anthropic`, `builtin-google`, `acp-anthropic`, `acp-google` (not all + have eval configs yet). -- **`evals/claude-code/eval.yaml`** (Claude Code): the agent runs via - `claude-agent-acp`, and the LLM judge is also `builtin.claude-code`, so **both - agent and judge run on your Claude subscription — fully keyless, for every - suite**. Judge-backed tasks are really evaluated (semantic), not skipped. This - is the recommended local path. -- **`evals/openai-agent/eval.yaml`** (OpenAI-compatible, what **CI** runs): the - agent uses `MODEL_BASE_URL`/`MODEL_KEY` and the judge is `openai:gpt-5`, so its - judge-backed tasks need an OpenAI key. +Each eval config's `llmJudge` references the same `agent.yaml` used by the agent +itself, so the judge reuses the agent's model and credentials — no separate judge +setup is needed. A task is "judge-backed" when its verify phase has an `llmJudge` step — **including the legacy `verify: contains:` short form, which is judge-evaluated (semantic), not a literal string match.** Judge-backed task counts per suite: -`helm` 0, `core` 4, `kubevirt` 4, `tekton` 5, `config` 3, `kiali` 18. With the -OpenAI agent those tasks need a key; with the Claude Code agent none do. Filter -either config to one suite with `--label-selector suite=`. - -**Per-suite** configs (`evals/tasks//{claude-code,openai-agent}/eval.yaml`, -currently only `kubevirt`) are scoped to a single suite. The claude-code one -judges with claude-code (keyless); the openai-agent one declares no judge, so its -judge steps degrade to a no-op pass (mcpchecker's `noopLLMJudge`). `make run-evals` -prefers a per-suite config when one exists for the chosen `SUITE`/`AGENT`, and -otherwise falls back to the agent's top-level config. +`helm` 0, `core` 4, `kubevirt` 4, `tekton` 5, `config` 3, `kiali` 18. Filter +any config to one suite with `--label-selector suite=`. ## Agent configuration -### Claude Code (`claude-code/agent.yaml`) +Agent configs live in `evals/core-eval-testing//agent.yaml`. There are two +kinds: -The Claude Code agent runs over the Agent Client Protocol (ACP) through the -`claude-agent-acp` adapter: +**ACP agents** run through an external CLI adapter: -```yaml -kind: Agent -metadata: - name: "claude-code-acp" -acp: - cmd: "claude-agent-acp" -``` - -### OpenAI-compatible agent (`openai-agent/agent.yaml`) +- **`acp-anthropic`** — uses `claude-agent-acp` (Claude Code via ACP, keyless) +- **`acp-google`** — uses `gemini --experimental-acp` -The OpenAI-compatible agent uses mcpchecker's built-in LLM agent: +**Builtin agents** use mcpchecker's built-in LLM agent: -```yaml -builtin: - type: "llm-agent" - model: "openai:gpt-5" -``` +- **`builtin-openai`** — `openai:gpt-5` (what CI runs) +- **`builtin-anthropic`** — `anthropic:claude-sonnet-4-6` +- **`builtin-google`** — `google:gemini-3.1-pro-preview` ## Filtering tasks by suite The `--label-selector` flag (or the `SUITE` make variable) chooses which tasks run. ```bash -# core tasks -make run-evals SUITE=core AGENT=claude-code MODEL=sonnet +# core tasks (builtin-openai is the default AGENT) +make run-evals SUITE=core # kiali tasks (needs Istio + Kiali installed; see `make setup-kiali`) -make run-evals SUITE=kiali AGENT=claude-code MODEL=sonnet +make run-evals SUITE=kiali AGENT=acp-anthropic MODEL=sonnet # netobserv tasks (needs mock plugin or real NetObserv; see tasks/netobserv/README.md) -make run-evals SUITE=netobserv AGENT=claude-code MODEL=sonnet +make run-evals SUITE=netobserv AGENT=builtin-openai ``` If you omit `--label-selector`, the eval config's own `labelSelector` settings in each `taskSets` entry determine which tasks run. -Note: with `AGENT=claude-code` every suite is keyless (agent and judge both run on -your Claude subscription). With `AGENT=openai-agent` the judge-backed tasks need an -OpenAI key — `core` 4, `config` 3, `kiali` 18, `tekton` 5, `kubevirt` 4, `helm` 0 -(see [Eval configs](#eval-configs-top-level-vs-per-suite)). +Note: with `AGENT=acp-anthropic` every suite is keyless (agent and judge both run +on your Claude subscription). With `builtin-*` agents the judge-backed tasks need +the provider's API key (see [Eval configs](#eval-configs)). ## Versions @@ -208,7 +195,7 @@ pin the binary with `make mcpchecker MCPCHECKER_VERSION=`. `make claude-agent-acp` likewise installs `@agentclientprotocol/claude-agent-acp@latest`; pin it with `make claude-agent-acp CLAUDE_AGENT_ACP_VERSION=` for local -reproducibility. CI never installs the adapter (it runs the OpenAI agent), so this +reproducibility. CI never installs the adapter (it runs `builtin-openai`), so this knob is purely local. ## Expected results diff --git a/evals/claude-code/agent.yaml b/evals/claude-code/agent.yaml deleted file mode 100644 index 8c9186911..000000000 --- a/evals/claude-code/agent.yaml +++ /dev/null @@ -1,5 +0,0 @@ -kind: Agent -metadata: - name: "claude-code-acp" -acp: - cmd: "claude-agent-acp" diff --git a/evals/claude-code/eval.yaml b/evals/claude-code/eval.yaml deleted file mode 100644 index 8fe931c9c..000000000 --- a/evals/claude-code/eval.yaml +++ /dev/null @@ -1,86 +0,0 @@ -kind: Eval -metadata: - name: "kubernetes-basic-operations" -config: - agent: - type: "file" - path: agent.yaml - mcpConfigFile: ../mcp-config.yaml - extensions: - kubernetes: - package: https://github.com/mcpchecker/kubernetes-extension@v0.0.4 - # The claude-code agent judges with claude-code too, so the whole flow (agent + - # judge) runs on one Anthropic/Claude-subscription auth with no OpenAI key. - # (The openai-agent config keeps an independent openai:gpt-5 judge for CI.) - llmJudge: - ref: - type: builtin.claude-code - taskSets: - # Kubernetes tasks - - glob: ../tasks/*/*/*.yaml - labelSelector: - suite: core - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 - - glob: ../tasks/config/*/*.yaml - labelSelector: - suite: config - assertions: - toolsUsed: - - server: kubernetes - toolPattern: "configuration_.*" - minToolCalls: 1 - maxToolCalls: 10 - - glob: ../tasks/helm/*/*.yaml - labelSelector: - suite: helm - assertions: - toolsUsed: - - server: kubernetes - toolPattern: "helm_.*" - minToolCalls: 1 - maxToolCalls: 10 - # KubeVirt tasks - - glob: ../tasks/kubevirt/*/*.yaml - labelSelector: - suite: kubevirt - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 - # Kiali tasks - - glob: ../tasks/kiali/*/*.yaml - labelSelector: - suite: kiali - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 - # Tekton tasks - - glob: ../tasks/tekton/*/*.yaml - labelSelector: - suite: tekton - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 - # NetObserv tasks - - glob: ../tasks/netobserv/*/*.yaml - labelSelector: - suite: netobserv - assertions: - toolsUsed: - - server: kubernetes - toolPattern: "netobserv_.*" - minToolCalls: 1 - maxToolCalls: 10 diff --git a/evals/core-eval-testing/README.md b/evals/core-eval-testing/README.md index 18b0da578..0573f3dd5 100644 --- a/evals/core-eval-testing/README.md +++ b/evals/core-eval-testing/README.md @@ -1,10 +1,6 @@ # Core Eval Testing -Eval configurations for running **core** task suites (core, config, helm) across different LLM providers and agent types. - -## Naming - -"Core" refers to the three foundational task suites: `core`, `config`, and `helm`. Each subdirectory contains eval configs for all three suites using a specific provider/agent combination. +Eval configurations for running task suites across different LLM providers and agent types. ## Directory structure @@ -15,12 +11,19 @@ Each subdirectory follows the pattern `-/`: Each contains: - `agent.yaml` — agent configuration (model, type) -- `eval-core.yaml` — eval config for the `core` task suite -- `eval-config.yaml` — eval config for the `config` task suite -- `eval-helm.yaml` — eval config for the `helm` task suite +- `eval-core.yaml` — eval config for the `core` + `config` task suites +- `eval-helm.yaml` — eval config for the `helm` task suite (+ core/config) +- `eval-kubevirt.yaml` — eval config for the `kubevirt` task suite (+ core/config) +- `eval-kiali.yaml` — eval config for the `kiali` task suite (+ core/config) +- `eval-tekton.yaml` — eval config for the `tekton` task suite (+ core/config) +- `eval-netobserv.yaml` — eval config for the `netobserv` task suite (+ core/config) +- `eval-all.yaml` — eval config that runs all task suites + +Not all agent directories have every eval file yet — `builtin-openai` is the most complete set, used by the CI workflow. ## Design decisions - **Shared tasks**: All eval configs reference the same task definitions via `../../tasks/*/*/*.yaml` with `labelSelector` to filter by suite. - **Shared MCP config**: All configs use `../../mcp-config.yaml` to connect to the same MCP server instance. - **Per-suite eval files**: Separate eval files per suite (instead of one combined file) allow running suites independently and setting different assertions (e.g., `maxToolCalls`). +- **Core + config always included**: Every per-suite eval file includes the `core` and `config` task sets alongside the suite-specific tasks, ensuring baseline coverage in every run. diff --git a/evals/core-eval-testing/acp-anthropic/eval-all.yaml b/evals/core-eval-testing/acp-anthropic/eval-all.yaml new file mode 120000 index 000000000..6f1317b1e --- /dev/null +++ b/evals/core-eval-testing/acp-anthropic/eval-all.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-all.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/acp-anthropic/eval-core.yaml b/evals/core-eval-testing/acp-anthropic/eval-core.yaml deleted file mode 100644 index f465668dc..000000000 --- a/evals/core-eval-testing/acp-anthropic/eval-core.yaml +++ /dev/null @@ -1,25 +0,0 @@ -kind: Eval -metadata: - name: "core-e2e" -config: - agent: - type: "file" - path: agent.yaml - mcpConfigFile: ../../mcp-config.yaml - extensions: - kubernetes: - package: https://github.com/mcpchecker/kubernetes-extension@v0.0.4 - llmJudge: - ref: - type: file - path: agent.yaml - taskSets: - - glob: ../../tasks/*/*/*.yaml - labelSelector: - suite: core - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 diff --git a/evals/core-eval-testing/acp-anthropic/eval-core.yaml b/evals/core-eval-testing/acp-anthropic/eval-core.yaml new file mode 120000 index 000000000..e37ced931 --- /dev/null +++ b/evals/core-eval-testing/acp-anthropic/eval-core.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-core.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/acp-anthropic/eval-helm.yaml b/evals/core-eval-testing/acp-anthropic/eval-helm.yaml deleted file mode 100644 index 97f837171..000000000 --- a/evals/core-eval-testing/acp-anthropic/eval-helm.yaml +++ /dev/null @@ -1,25 +0,0 @@ -kind: Eval -metadata: - name: "helm-e2e" -config: - agent: - type: "file" - path: agent.yaml - mcpConfigFile: ../../mcp-config.yaml - extensions: - kubernetes: - package: https://github.com/mcpchecker/kubernetes-extension@v0.0.4 - llmJudge: - ref: - type: file - path: agent.yaml - taskSets: - - glob: ../../tasks/*/*/*.yaml - labelSelector: - suite: helm - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 10 diff --git a/evals/core-eval-testing/acp-anthropic/eval-helm.yaml b/evals/core-eval-testing/acp-anthropic/eval-helm.yaml new file mode 120000 index 000000000..0658515a9 --- /dev/null +++ b/evals/core-eval-testing/acp-anthropic/eval-helm.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-helm.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/acp-anthropic/eval-kiali.yaml b/evals/core-eval-testing/acp-anthropic/eval-kiali.yaml new file mode 120000 index 000000000..cbfb28504 --- /dev/null +++ b/evals/core-eval-testing/acp-anthropic/eval-kiali.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-kiali.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/acp-anthropic/eval-kubevirt.yaml b/evals/core-eval-testing/acp-anthropic/eval-kubevirt.yaml new file mode 120000 index 000000000..b09ce0244 --- /dev/null +++ b/evals/core-eval-testing/acp-anthropic/eval-kubevirt.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-kubevirt.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/acp-anthropic/eval-netobserv.yaml b/evals/core-eval-testing/acp-anthropic/eval-netobserv.yaml new file mode 120000 index 000000000..615ea2702 --- /dev/null +++ b/evals/core-eval-testing/acp-anthropic/eval-netobserv.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-netobserv.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/acp-anthropic/eval-tekton.yaml b/evals/core-eval-testing/acp-anthropic/eval-tekton.yaml new file mode 120000 index 000000000..ba642ebcc --- /dev/null +++ b/evals/core-eval-testing/acp-anthropic/eval-tekton.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-tekton.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/acp-google/eval-all.yaml b/evals/core-eval-testing/acp-google/eval-all.yaml new file mode 120000 index 000000000..6f1317b1e --- /dev/null +++ b/evals/core-eval-testing/acp-google/eval-all.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-all.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/acp-google/eval-core.yaml b/evals/core-eval-testing/acp-google/eval-core.yaml deleted file mode 100644 index f465668dc..000000000 --- a/evals/core-eval-testing/acp-google/eval-core.yaml +++ /dev/null @@ -1,25 +0,0 @@ -kind: Eval -metadata: - name: "core-e2e" -config: - agent: - type: "file" - path: agent.yaml - mcpConfigFile: ../../mcp-config.yaml - extensions: - kubernetes: - package: https://github.com/mcpchecker/kubernetes-extension@v0.0.4 - llmJudge: - ref: - type: file - path: agent.yaml - taskSets: - - glob: ../../tasks/*/*/*.yaml - labelSelector: - suite: core - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 diff --git a/evals/core-eval-testing/acp-google/eval-core.yaml b/evals/core-eval-testing/acp-google/eval-core.yaml new file mode 120000 index 000000000..e37ced931 --- /dev/null +++ b/evals/core-eval-testing/acp-google/eval-core.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-core.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/acp-google/eval-helm.yaml b/evals/core-eval-testing/acp-google/eval-helm.yaml deleted file mode 100644 index 97f837171..000000000 --- a/evals/core-eval-testing/acp-google/eval-helm.yaml +++ /dev/null @@ -1,25 +0,0 @@ -kind: Eval -metadata: - name: "helm-e2e" -config: - agent: - type: "file" - path: agent.yaml - mcpConfigFile: ../../mcp-config.yaml - extensions: - kubernetes: - package: https://github.com/mcpchecker/kubernetes-extension@v0.0.4 - llmJudge: - ref: - type: file - path: agent.yaml - taskSets: - - glob: ../../tasks/*/*/*.yaml - labelSelector: - suite: helm - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 10 diff --git a/evals/core-eval-testing/acp-google/eval-helm.yaml b/evals/core-eval-testing/acp-google/eval-helm.yaml new file mode 120000 index 000000000..0658515a9 --- /dev/null +++ b/evals/core-eval-testing/acp-google/eval-helm.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-helm.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/acp-google/eval-kiali.yaml b/evals/core-eval-testing/acp-google/eval-kiali.yaml new file mode 120000 index 000000000..cbfb28504 --- /dev/null +++ b/evals/core-eval-testing/acp-google/eval-kiali.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-kiali.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/acp-google/eval-kubevirt.yaml b/evals/core-eval-testing/acp-google/eval-kubevirt.yaml new file mode 120000 index 000000000..b09ce0244 --- /dev/null +++ b/evals/core-eval-testing/acp-google/eval-kubevirt.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-kubevirt.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/acp-google/eval-netobserv.yaml b/evals/core-eval-testing/acp-google/eval-netobserv.yaml new file mode 120000 index 000000000..615ea2702 --- /dev/null +++ b/evals/core-eval-testing/acp-google/eval-netobserv.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-netobserv.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/acp-google/eval-tekton.yaml b/evals/core-eval-testing/acp-google/eval-tekton.yaml new file mode 120000 index 000000000..ba642ebcc --- /dev/null +++ b/evals/core-eval-testing/acp-google/eval-tekton.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-tekton.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/builtin-anthropic/eval-all.yaml b/evals/core-eval-testing/builtin-anthropic/eval-all.yaml new file mode 120000 index 000000000..6f1317b1e --- /dev/null +++ b/evals/core-eval-testing/builtin-anthropic/eval-all.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-all.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/builtin-anthropic/eval-core.yaml b/evals/core-eval-testing/builtin-anthropic/eval-core.yaml deleted file mode 100644 index f465668dc..000000000 --- a/evals/core-eval-testing/builtin-anthropic/eval-core.yaml +++ /dev/null @@ -1,25 +0,0 @@ -kind: Eval -metadata: - name: "core-e2e" -config: - agent: - type: "file" - path: agent.yaml - mcpConfigFile: ../../mcp-config.yaml - extensions: - kubernetes: - package: https://github.com/mcpchecker/kubernetes-extension@v0.0.4 - llmJudge: - ref: - type: file - path: agent.yaml - taskSets: - - glob: ../../tasks/*/*/*.yaml - labelSelector: - suite: core - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 diff --git a/evals/core-eval-testing/builtin-anthropic/eval-core.yaml b/evals/core-eval-testing/builtin-anthropic/eval-core.yaml new file mode 120000 index 000000000..e37ced931 --- /dev/null +++ b/evals/core-eval-testing/builtin-anthropic/eval-core.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-core.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/builtin-anthropic/eval-helm.yaml b/evals/core-eval-testing/builtin-anthropic/eval-helm.yaml deleted file mode 100644 index 97f837171..000000000 --- a/evals/core-eval-testing/builtin-anthropic/eval-helm.yaml +++ /dev/null @@ -1,25 +0,0 @@ -kind: Eval -metadata: - name: "helm-e2e" -config: - agent: - type: "file" - path: agent.yaml - mcpConfigFile: ../../mcp-config.yaml - extensions: - kubernetes: - package: https://github.com/mcpchecker/kubernetes-extension@v0.0.4 - llmJudge: - ref: - type: file - path: agent.yaml - taskSets: - - glob: ../../tasks/*/*/*.yaml - labelSelector: - suite: helm - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 10 diff --git a/evals/core-eval-testing/builtin-anthropic/eval-helm.yaml b/evals/core-eval-testing/builtin-anthropic/eval-helm.yaml new file mode 120000 index 000000000..0658515a9 --- /dev/null +++ b/evals/core-eval-testing/builtin-anthropic/eval-helm.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-helm.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/builtin-anthropic/eval-kiali.yaml b/evals/core-eval-testing/builtin-anthropic/eval-kiali.yaml new file mode 120000 index 000000000..cbfb28504 --- /dev/null +++ b/evals/core-eval-testing/builtin-anthropic/eval-kiali.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-kiali.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/builtin-anthropic/eval-kubevirt.yaml b/evals/core-eval-testing/builtin-anthropic/eval-kubevirt.yaml new file mode 120000 index 000000000..b09ce0244 --- /dev/null +++ b/evals/core-eval-testing/builtin-anthropic/eval-kubevirt.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-kubevirt.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/builtin-anthropic/eval-netobserv.yaml b/evals/core-eval-testing/builtin-anthropic/eval-netobserv.yaml new file mode 120000 index 000000000..615ea2702 --- /dev/null +++ b/evals/core-eval-testing/builtin-anthropic/eval-netobserv.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-netobserv.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/builtin-anthropic/eval-tekton.yaml b/evals/core-eval-testing/builtin-anthropic/eval-tekton.yaml new file mode 120000 index 000000000..ba642ebcc --- /dev/null +++ b/evals/core-eval-testing/builtin-anthropic/eval-tekton.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-tekton.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/builtin-google/eval-all.yaml b/evals/core-eval-testing/builtin-google/eval-all.yaml new file mode 120000 index 000000000..6f1317b1e --- /dev/null +++ b/evals/core-eval-testing/builtin-google/eval-all.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-all.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/builtin-google/eval-core.yaml b/evals/core-eval-testing/builtin-google/eval-core.yaml deleted file mode 100644 index f465668dc..000000000 --- a/evals/core-eval-testing/builtin-google/eval-core.yaml +++ /dev/null @@ -1,25 +0,0 @@ -kind: Eval -metadata: - name: "core-e2e" -config: - agent: - type: "file" - path: agent.yaml - mcpConfigFile: ../../mcp-config.yaml - extensions: - kubernetes: - package: https://github.com/mcpchecker/kubernetes-extension@v0.0.4 - llmJudge: - ref: - type: file - path: agent.yaml - taskSets: - - glob: ../../tasks/*/*/*.yaml - labelSelector: - suite: core - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 diff --git a/evals/core-eval-testing/builtin-google/eval-core.yaml b/evals/core-eval-testing/builtin-google/eval-core.yaml new file mode 120000 index 000000000..e37ced931 --- /dev/null +++ b/evals/core-eval-testing/builtin-google/eval-core.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-core.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/builtin-google/eval-helm.yaml b/evals/core-eval-testing/builtin-google/eval-helm.yaml deleted file mode 100644 index 97f837171..000000000 --- a/evals/core-eval-testing/builtin-google/eval-helm.yaml +++ /dev/null @@ -1,25 +0,0 @@ -kind: Eval -metadata: - name: "helm-e2e" -config: - agent: - type: "file" - path: agent.yaml - mcpConfigFile: ../../mcp-config.yaml - extensions: - kubernetes: - package: https://github.com/mcpchecker/kubernetes-extension@v0.0.4 - llmJudge: - ref: - type: file - path: agent.yaml - taskSets: - - glob: ../../tasks/*/*/*.yaml - labelSelector: - suite: helm - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 10 diff --git a/evals/core-eval-testing/builtin-google/eval-helm.yaml b/evals/core-eval-testing/builtin-google/eval-helm.yaml new file mode 120000 index 000000000..0658515a9 --- /dev/null +++ b/evals/core-eval-testing/builtin-google/eval-helm.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-helm.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/builtin-google/eval-kiali.yaml b/evals/core-eval-testing/builtin-google/eval-kiali.yaml new file mode 120000 index 000000000..cbfb28504 --- /dev/null +++ b/evals/core-eval-testing/builtin-google/eval-kiali.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-kiali.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/builtin-google/eval-kubevirt.yaml b/evals/core-eval-testing/builtin-google/eval-kubevirt.yaml new file mode 120000 index 000000000..b09ce0244 --- /dev/null +++ b/evals/core-eval-testing/builtin-google/eval-kubevirt.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-kubevirt.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/builtin-google/eval-netobserv.yaml b/evals/core-eval-testing/builtin-google/eval-netobserv.yaml new file mode 120000 index 000000000..615ea2702 --- /dev/null +++ b/evals/core-eval-testing/builtin-google/eval-netobserv.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-netobserv.yaml \ No newline at end of file diff --git a/evals/core-eval-testing/builtin-google/eval-tekton.yaml b/evals/core-eval-testing/builtin-google/eval-tekton.yaml new file mode 120000 index 000000000..ba642ebcc --- /dev/null +++ b/evals/core-eval-testing/builtin-google/eval-tekton.yaml @@ -0,0 +1 @@ +../builtin-openai/eval-tekton.yaml \ No newline at end of file diff --git a/evals/openai-agent/eval.yaml b/evals/core-eval-testing/builtin-openai/eval-all.yaml similarity index 80% rename from evals/openai-agent/eval.yaml rename to evals/core-eval-testing/builtin-openai/eval-all.yaml index ecc61a16b..df40b0d26 100644 --- a/evals/openai-agent/eval.yaml +++ b/evals/core-eval-testing/builtin-openai/eval-all.yaml @@ -1,21 +1,21 @@ kind: Eval metadata: - name: "openai-agent-kubernetes-test" + name: "all-e2e" config: agent: type: "file" path: agent.yaml - mcpConfigFile: ../mcp-config.yaml + mcpConfigFile: ../../mcp-config.yaml extensions: kubernetes: package: https://github.com/mcpchecker/kubernetes-extension@v0.0.4 llmJudge: ref: - type: builtin.llm-agent - model: "openai:gpt-5" + type: file + path: agent.yaml taskSets: - # Kubernetes tasks - - glob: ../tasks/*/*/*.yaml + # Core + Config tasks + - glob: ../../tasks/*/*/*.yaml labelSelector: suite: core assertions: @@ -24,7 +24,7 @@ config: toolPattern: ".*" minToolCalls: 1 maxToolCalls: 20 - - glob: ../tasks/config/*/*.yaml + - glob: ../../tasks/*/*/*.yaml labelSelector: suite: config assertions: @@ -33,7 +33,8 @@ config: toolPattern: "configuration_.*" minToolCalls: 1 maxToolCalls: 10 - - glob: ../tasks/helm/*/*.yaml + # Helm tasks + - glob: ../../tasks/*/*/*.yaml labelSelector: suite: helm assertions: @@ -43,7 +44,7 @@ config: minToolCalls: 1 maxToolCalls: 10 # KubeVirt tasks - - glob: ../tasks/kubevirt/*/*.yaml + - glob: ../../tasks/*/*/*.yaml labelSelector: suite: kubevirt assertions: @@ -53,7 +54,7 @@ config: minToolCalls: 1 maxToolCalls: 20 # Kiali tasks - - glob: ../tasks/kiali/*/*.yaml + - glob: ../../tasks/*/*/*.yaml labelSelector: suite: kiali assertions: @@ -63,7 +64,7 @@ config: minToolCalls: 1 maxToolCalls: 20 # Tekton tasks - - glob: ../tasks/tekton/*/*.yaml + - glob: ../../tasks/*/*/*.yaml labelSelector: suite: tekton assertions: @@ -73,7 +74,7 @@ config: minToolCalls: 1 maxToolCalls: 20 # NetObserv tasks - - glob: ../tasks/netobserv/*/*.yaml + - glob: ../../tasks/*/*/*.yaml labelSelector: suite: netobserv assertions: diff --git a/evals/core-eval-testing/builtin-openai/eval-config.yaml b/evals/core-eval-testing/builtin-openai/eval-config.yaml deleted file mode 100644 index 16243fc13..000000000 --- a/evals/core-eval-testing/builtin-openai/eval-config.yaml +++ /dev/null @@ -1,25 +0,0 @@ -kind: Eval -metadata: - name: "config-e2e" -config: - agent: - type: "file" - path: agent.yaml - mcpConfigFile: ../../mcp-config.yaml - extensions: - kubernetes: - package: https://github.com/mcpchecker/kubernetes-extension@v0.0.4 - llmJudge: - ref: - type: file - path: agent.yaml - taskSets: - - glob: ../../tasks/*/*/*.yaml - labelSelector: - suite: config - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 10 diff --git a/evals/core-eval-testing/builtin-openai/eval-core.yaml b/evals/core-eval-testing/builtin-openai/eval-core.yaml index f465668dc..2a4227543 100644 --- a/evals/core-eval-testing/builtin-openai/eval-core.yaml +++ b/evals/core-eval-testing/builtin-openai/eval-core.yaml @@ -23,3 +23,12 @@ config: toolPattern: ".*" minToolCalls: 1 maxToolCalls: 20 + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: config + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "configuration_.*" + minToolCalls: 1 + maxToolCalls: 10 diff --git a/evals/core-eval-testing/builtin-openai/eval-helm.yaml b/evals/core-eval-testing/builtin-openai/eval-helm.yaml index 97f837171..9d199038d 100644 --- a/evals/core-eval-testing/builtin-openai/eval-helm.yaml +++ b/evals/core-eval-testing/builtin-openai/eval-helm.yaml @@ -16,10 +16,28 @@ config: taskSets: - glob: ../../tasks/*/*/*.yaml labelSelector: - suite: helm + suite: core assertions: toolsUsed: - server: kubernetes toolPattern: ".*" minToolCalls: 1 + maxToolCalls: 20 + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: config + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "configuration_.*" + minToolCalls: 1 + maxToolCalls: 10 + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: helm + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "helm_.*" + minToolCalls: 1 maxToolCalls: 10 diff --git a/evals/core-eval-testing/builtin-anthropic/eval-config.yaml b/evals/core-eval-testing/builtin-openai/eval-kiali.yaml similarity index 51% rename from evals/core-eval-testing/builtin-anthropic/eval-config.yaml rename to evals/core-eval-testing/builtin-openai/eval-kiali.yaml index 16243fc13..f7ac0ea52 100644 --- a/evals/core-eval-testing/builtin-anthropic/eval-config.yaml +++ b/evals/core-eval-testing/builtin-openai/eval-kiali.yaml @@ -1,6 +1,6 @@ kind: Eval metadata: - name: "config-e2e" + name: "kiali-e2e" config: agent: type: "file" @@ -16,10 +16,28 @@ config: taskSets: - glob: ../../tasks/*/*/*.yaml labelSelector: - suite: config + suite: core assertions: toolsUsed: - server: kubernetes toolPattern: ".*" minToolCalls: 1 + maxToolCalls: 20 + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: config + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "configuration_.*" + minToolCalls: 1 maxToolCalls: 10 + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: kiali + assertions: + toolsUsed: + - server: kubernetes + toolPattern: ".*" + minToolCalls: 1 + maxToolCalls: 20 diff --git a/evals/core-eval-testing/acp-anthropic/eval-config.yaml b/evals/core-eval-testing/builtin-openai/eval-kubevirt.yaml similarity index 51% rename from evals/core-eval-testing/acp-anthropic/eval-config.yaml rename to evals/core-eval-testing/builtin-openai/eval-kubevirt.yaml index 16243fc13..9521d9f21 100644 --- a/evals/core-eval-testing/acp-anthropic/eval-config.yaml +++ b/evals/core-eval-testing/builtin-openai/eval-kubevirt.yaml @@ -1,6 +1,6 @@ kind: Eval metadata: - name: "config-e2e" + name: "kubevirt-e2e" config: agent: type: "file" @@ -16,10 +16,28 @@ config: taskSets: - glob: ../../tasks/*/*/*.yaml labelSelector: - suite: config + suite: core assertions: toolsUsed: - server: kubernetes toolPattern: ".*" minToolCalls: 1 + maxToolCalls: 20 + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: config + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "configuration_.*" + minToolCalls: 1 maxToolCalls: 10 + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: kubevirt + assertions: + toolsUsed: + - server: kubernetes + toolPattern: ".*" + minToolCalls: 1 + maxToolCalls: 20 diff --git a/evals/core-eval-testing/acp-google/eval-config.yaml b/evals/core-eval-testing/builtin-openai/eval-netobserv.yaml similarity index 50% rename from evals/core-eval-testing/acp-google/eval-config.yaml rename to evals/core-eval-testing/builtin-openai/eval-netobserv.yaml index 16243fc13..c89357f3f 100644 --- a/evals/core-eval-testing/acp-google/eval-config.yaml +++ b/evals/core-eval-testing/builtin-openai/eval-netobserv.yaml @@ -1,6 +1,6 @@ kind: Eval metadata: - name: "config-e2e" + name: "netobserv-e2e" config: agent: type: "file" @@ -16,10 +16,28 @@ config: taskSets: - glob: ../../tasks/*/*/*.yaml labelSelector: - suite: config + suite: core assertions: toolsUsed: - server: kubernetes toolPattern: ".*" minToolCalls: 1 + maxToolCalls: 20 + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: config + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "configuration_.*" + minToolCalls: 1 + maxToolCalls: 10 + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: netobserv + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "netobserv_.*" + minToolCalls: 1 maxToolCalls: 10 diff --git a/evals/core-eval-testing/builtin-google/eval-config.yaml b/evals/core-eval-testing/builtin-openai/eval-tekton.yaml similarity index 51% rename from evals/core-eval-testing/builtin-google/eval-config.yaml rename to evals/core-eval-testing/builtin-openai/eval-tekton.yaml index 16243fc13..7e766394b 100644 --- a/evals/core-eval-testing/builtin-google/eval-config.yaml +++ b/evals/core-eval-testing/builtin-openai/eval-tekton.yaml @@ -1,6 +1,6 @@ kind: Eval metadata: - name: "config-e2e" + name: "tekton-e2e" config: agent: type: "file" @@ -16,10 +16,28 @@ config: taskSets: - glob: ../../tasks/*/*/*.yaml labelSelector: - suite: config + suite: core assertions: toolsUsed: - server: kubernetes toolPattern: ".*" minToolCalls: 1 + maxToolCalls: 20 + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: config + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "configuration_.*" + minToolCalls: 1 maxToolCalls: 10 + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: tekton + assertions: + toolsUsed: + - server: kubernetes + toolPattern: ".*" + minToolCalls: 1 + maxToolCalls: 20 diff --git a/evals/openai-agent/agent.yaml b/evals/openai-agent/agent.yaml deleted file mode 100644 index 60a80cd67..000000000 --- a/evals/openai-agent/agent.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: Agent -metadata: - name: "openai-agent" -builtin: - type: "llm-agent" - model: "openai:gpt-5" # Change to your model -# Before running, set environment variables: -# export MODEL_BASE_URL="https://api.openai.com/v1" -# export MODEL_KEY="sk-..." diff --git a/evals/results/openai-agent-latest.json b/evals/results/builtin-openai-core-latest.json similarity index 100% rename from evals/results/openai-agent-latest.json rename to evals/results/builtin-openai-core-latest.json diff --git a/evals/tasks/kubevirt/claude-code/eval.yaml b/evals/tasks/kubevirt/claude-code/eval.yaml deleted file mode 100644 index 9c0362f43..000000000 --- a/evals/tasks/kubevirt/claude-code/eval.yaml +++ /dev/null @@ -1,23 +0,0 @@ -kind: Eval -metadata: - name: "kubevirt-basic-operations" -config: - agent: - type: "builtin.claude-code" - mcpConfigFile: ../../../mcp-config.yaml - extensions: - kubernetes: - package: https://github.com/mcpchecker/kubernetes-extension@v0.0.4 - # Judge with claude-code so judge-backed tasks (verify: contains / llmJudge) - # are really evaluated, keyless on the Claude subscription, like the agent. - llmJudge: - ref: - type: builtin.claude-code - taskSets: - - glob: ../*/task.yaml - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 diff --git a/evals/tasks/kubevirt/openai-agent/eval.yaml b/evals/tasks/kubevirt/openai-agent/eval.yaml deleted file mode 100644 index 2373b19af..000000000 --- a/evals/tasks/kubevirt/openai-agent/eval.yaml +++ /dev/null @@ -1,19 +0,0 @@ -kind: Eval -metadata: - name: "kubevirt-basic-operations" -config: - agent: - type: "builtin.llm-agent" - model: "openai:gpt-5" - mcpConfigFile: ../../../mcp-config.yaml - extensions: - kubernetes: - package: https://github.com/mcpchecker/kubernetes-extension@v0.0.4 - taskSets: - - glob: ../*/task.yaml - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 diff --git a/evals/tasks/netobserv/README.md b/evals/tasks/netobserv/README.md index 5e060246b..b5884d205 100644 --- a/evals/tasks/netobserv/README.md +++ b/evals/tasks/netobserv/README.md @@ -9,7 +9,7 @@ Before relying on these tools, the domain was validated with **baseline evals** Run a baseline locally when changing the toolset: ```bash -mcpchecker check evals/openai-agent/eval.yaml --label-selector suite=netobserv \ +mcpchecker check evals/core-eval-testing/builtin-openai/eval-netobserv.yaml --label-selector suite=netobserv \ # with toolsets = ["core"] only — tasks should fail or produce poor results ``` @@ -74,7 +74,7 @@ On a cluster with the [NetObserv operator](https://github.com/netobserv-network- 4. Run evals with the same label selector: ```bash - mcpchecker check evals/openai-agent/eval.yaml --label-selector suite=netobserv + mcpchecker check evals/core-eval-testing/builtin-openai/eval-netobserv.yaml --label-selector suite=netobserv ``` LLM judge strings in tasks assume the **mock** responses (`netobserv-eval`, `eval-flow-1`, …). On a live cluster, adjust `verify.llmJudge.contains` or rely on tool assertions only. diff --git a/go.mod b/go.mod index 914cafc51..3a499fb3c 100644 --- a/go.mod +++ b/go.mod @@ -46,16 +46,16 @@ require ( google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af gopkg.in/yaml.v3 v3.0.1 helm.sh/helm/v3 v3.21.3 - k8s.io/api v0.36.2 - k8s.io/apiextensions-apiserver v0.36.2 - k8s.io/apimachinery v0.36.2 - k8s.io/cli-runtime v0.36.2 - k8s.io/client-go v0.36.2 + k8s.io/api v0.36.3 + k8s.io/apiextensions-apiserver v0.36.3 + k8s.io/apimachinery v0.36.3 + k8s.io/cli-runtime v0.36.3 + k8s.io/client-go v0.36.3 k8s.io/klog/v2 v2.140.0 k8s.io/kube-openapi v0.0.0-20260706235625-cdb1db5517a0 - k8s.io/kubectl v0.36.2 - k8s.io/metrics v0.36.2 - k8s.io/streaming v0.36.2 + k8s.io/kubectl v0.36.3 + k8s.io/metrics v0.36.3 + k8s.io/streaming v0.36.3 k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 sigs.k8s.io/controller-runtime v0.24.1 sigs.k8s.io/controller-runtime/tools/setup-envtest v0.24.1 @@ -193,8 +193,8 @@ require ( google.golang.org/grpc v1.81.1 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - k8s.io/apiserver v0.36.2 // indirect - k8s.io/component-base v0.36.2 // indirect + k8s.io/apiserver v0.36.3 // indirect + k8s.io/component-base v0.36.3 // indirect knative.dev/pkg v0.0.0-20260318013857-98d5a706d4fd // indirect oras.land/oras-go/v2 v2.6.1 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect diff --git a/go.sum b/go.sum index 4655ae6e1..2383e5c67 100644 --- a/go.sum +++ b/go.sum @@ -555,30 +555,30 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= helm.sh/helm/v3 v3.21.3 h1:wkamdwI3liEkW6wI1l9aGqQZGxcTKyt8kx0qJLPcmCg= helm.sh/helm/v3 v3.21.3/go.mod h1:iaJ0iNsPoTZl++7h6vzQFyT0VEVtLYJiyRBDkPOOBTs= -k8s.io/api v0.36.2 h1:TF6YDLIzKfccK7cq9YpTcGX8TJmEkHVRv78DM51fRYY= -k8s.io/api v0.36.2/go.mod h1:F4LbMO4brjZYh7yFkXWhynSvtB7YauxV4c+HHkNRGNg= -k8s.io/apiextensions-apiserver v0.36.2 h1:3O5gqOj/dt2XWWbpMe+TXWpE9yU6pjM/tXxtHHJT/K4= -k8s.io/apiextensions-apiserver v0.36.2/go.mod h1:cL1tBWe8XSaP1H30iWKGo7hf6iAUUUJPEU70dskmAnA= -k8s.io/apimachinery v0.36.2 h1:0PE/W/WNy1UX61NLbXY5TMbJ6UwLL6E6lAPkYrKFxbQ= -k8s.io/apimachinery v0.36.2/go.mod h1:fvf/HOLXq9RId0rnDIbN1OEBvHXdQbLMM8nu0LcBUf4= -k8s.io/apiserver v0.36.2 h1:6vMnkmHZPeBloNkHUhmZYq7Ylv8WIB8xjyEl+eSt26E= -k8s.io/apiserver v0.36.2/go.mod h1:9PoQ2ikCytrZyZg11mGhLEF5m8Rgsb5FJmYJ4Wvnl1k= -k8s.io/cli-runtime v0.36.2 h1:CconTvEeV4DJs4ZX3HQKCFbFRGsm6OtuBM9yjmMP2VM= -k8s.io/cli-runtime v0.36.2/go.mod h1:LddcjiMf4YlnHO7c1Y7rEtDqL84FyiYVLco7V679GUU= -k8s.io/client-go v0.36.2 h1:bfgxmFKc9CgqsgX4xKLAAdmTQlWee7Ob/HlDOrJ5TBI= -k8s.io/client-go v0.36.2/go.mod h1:1vgO4OAlfPnoLcb+Rze2GF5rAr14w8qjrYMoyXJzQj0= -k8s.io/component-base v0.36.2 h1:Z0VH80O7Ng0HDZnZj3WRR3urEGa0kTwmO8CwEwjVK1w= -k8s.io/component-base v0.36.2/go.mod h1:mGfFOA7Gwpdm1VW2cwSQYbiDIlz8GD2WGwH88QSeCyA= +k8s.io/api v0.36.3 h1:NxB+05W2UGqXWFXcLO0RB5cnqnUPP5v5sVlaOH0Iz4w= +k8s.io/api v0.36.3/go.mod h1:JzLQKqRHC5+I8RVj/lS3lCg0mg6nWI9Fo/Sk3ElxHzg= +k8s.io/apiextensions-apiserver v0.36.3 h1:dPmOAPhwTtqb1bTxbFPsy18KHPhktQeO3WUPXunZIB0= +k8s.io/apiextensions-apiserver v0.36.3/go.mod h1:KTXFqgXiuw2pRoL+Wpmttqc+up9Xt/GohadPWeLLOa4= +k8s.io/apimachinery v0.36.3 h1:PkzMRBRG8joFD8EhCuQAtNPvJlxb82FwplP26HIzvAM= +k8s.io/apimachinery v0.36.3/go.mod h1:cTSjBWgPe/6CQyBKzY/hDIRWCQQQeK0mfLbml0UYFHE= +k8s.io/apiserver v0.36.3 h1:MGSg2SkdfuytiDEcRylT5mQFmmSsbx90XFUO67Y4bsQ= +k8s.io/apiserver v0.36.3/go.mod h1:fVH7zv9EUNUA7Fl7LtDKh8aB9W7u1VQPSGtWV5SjUxg= +k8s.io/cli-runtime v0.36.3 h1:g+eJ+M1sYpnNYp/q5fzaw2KejIL0Q7DH+xFl6YVoL4U= +k8s.io/cli-runtime v0.36.3/go.mod h1:hZpAqK8nSFXvvLaVCbzUPVp8e9TRLSTCfpNzMt7s3tE= +k8s.io/client-go v0.36.3 h1:M4JdVzXxYcZk4fGpfDdYnxSwhLKWCFoQsHW6t+z8Hfg= +k8s.io/client-go v0.36.3/go.mod h1:gcPwr0c87vjjG6HB6pWEqOeuYVoXSsREjzux2j6GF30= +k8s.io/component-base v0.36.3 h1:vc/UFvPCkW0irPz84LAodAL1j3f4xktPM6dDJIEheAY= +k8s.io/component-base v0.36.3/go.mod h1:hZbNFG+gCMl9EbykDGEu73feKP9/Cq6JsV4pTo9GTO8= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-openapi v0.0.0-20260706235625-cdb1db5517a0 h1:CVjOUCTXINUThEmDs25FNSna0+vnGSoTleN+wiJu6hE= k8s.io/kube-openapi v0.0.0-20260706235625-cdb1db5517a0/go.mod h1:rcZ+P5cEvHQB+m154WBOatIGBgOEPjzmLkXjkHfg3ms= -k8s.io/kubectl v0.36.2 h1:rpUGGpeL09XVOLep2yle5jrtk//JA1L6ZHfkQQtVEwk= -k8s.io/kubectl v0.36.2/go.mod h1:gVbQ3B/yb4bSR2ggQ7rd0W6icUSWs7sduH4e16Vii+0= -k8s.io/metrics v0.36.2 h1:yfUIe2Vwx2cQAIpVYcin1JXdabrRz98oTxP2HJTxHj8= -k8s.io/metrics v0.36.2/go.mod h1:Q/dNyLLzgSxPu0/e+996Du4pjutfEyyHOKgK0lkncp0= -k8s.io/streaming v0.36.2 h1:NSKthPPg9UFSKsRauVJUVGH2Dvn8fhKmY4qrMkw/p98= -k8s.io/streaming v0.36.2/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= +k8s.io/kubectl v0.36.3 h1:TesKp+XYQEjPYoFvuobcVnuvira2+/xAVlq//+kksaI= +k8s.io/kubectl v0.36.3/go.mod h1:W+NEb1CzBGmoaI1Nrpn2ETo9omNBl0AsyxnnMT40N6E= +k8s.io/metrics v0.36.3 h1:NDKceAgWS8CJCdDtM5kFACkBOa9Lxia1jUiibJfvUgQ= +k8s.io/metrics v0.36.3/go.mod h1:NTLS8ybwn+zYGwKqYublWPvmnNp8N4pV3etjtx7XWaM= +k8s.io/streaming v0.36.3 h1:9rAaqBk0C0Pc7+/fqGekj07NV+/Xrew58p647A0JT8w= +k8s.io/streaming v0.36.3/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 h1:jVkFFVfXdXP74B/zbO3hM3hpSFD0xvhQ5U686DPurkE= k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3/go.mod h1:M2s5JB1lIYP3jzZdorPLHXIPJzt9vv2muW5a6L9DtNM= knative.dev/pkg v0.0.0-20260318013857-98d5a706d4fd h1:yeh+smYaouOwhkyCPj+AYACt1MeD+EI4mXSzSbmtj10= diff --git a/vendor/modules.txt b/vendor/modules.txt index b9fc2ac0e..da6f2a287 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1023,7 +1023,7 @@ helm.sh/helm/v3/pkg/storage/driver helm.sh/helm/v3/pkg/time helm.sh/helm/v3/pkg/time/ctime helm.sh/helm/v3/pkg/uploader -# k8s.io/api v0.36.2 +# k8s.io/api v0.36.3 ## explicit; go 1.26.0 k8s.io/api/admission/v1 k8s.io/api/admission/v1beta1 @@ -1083,7 +1083,7 @@ k8s.io/api/storage/v1 k8s.io/api/storage/v1alpha1 k8s.io/api/storage/v1beta1 k8s.io/api/storagemigration/v1beta1 -# k8s.io/apiextensions-apiserver v0.36.2 +# k8s.io/apiextensions-apiserver v0.36.3 ## explicit; go 1.26.0 k8s.io/apiextensions-apiserver/pkg/apis/apiextensions k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 @@ -1094,7 +1094,7 @@ k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1 k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1 -# k8s.io/apimachinery v0.36.2 +# k8s.io/apimachinery v0.36.3 ## explicit; go 1.26.0 k8s.io/apimachinery/pkg/api/equality k8s.io/apimachinery/pkg/api/errors @@ -1161,16 +1161,16 @@ k8s.io/apimachinery/pkg/version k8s.io/apimachinery/pkg/watch k8s.io/apimachinery/third_party/forked/golang/json k8s.io/apimachinery/third_party/forked/golang/reflect -# k8s.io/apiserver v0.36.2 +# k8s.io/apiserver v0.36.3 ## explicit; go 1.26.0 k8s.io/apiserver/pkg/endpoints/deprecation -# k8s.io/cli-runtime v0.36.2 +# k8s.io/cli-runtime v0.36.3 ## explicit; go 1.26.0 k8s.io/cli-runtime/pkg/genericclioptions k8s.io/cli-runtime/pkg/genericiooptions k8s.io/cli-runtime/pkg/printers k8s.io/cli-runtime/pkg/resource -# k8s.io/client-go v0.36.2 +# k8s.io/client-go v0.36.3 ## explicit; go 1.26.0 k8s.io/client-go/applyconfigurations/admissionregistration/v1 k8s.io/client-go/applyconfigurations/admissionregistration/v1alpha1 @@ -1457,7 +1457,7 @@ k8s.io/client-go/util/keyutil k8s.io/client-go/util/retry k8s.io/client-go/util/watchlist k8s.io/client-go/util/workqueue -# k8s.io/component-base v0.36.2 +# k8s.io/component-base v0.36.3 ## explicit; go 1.26.0 k8s.io/component-base/version # k8s.io/klog/v2 v2.140.0 @@ -1489,7 +1489,7 @@ k8s.io/kube-openapi/pkg/util k8s.io/kube-openapi/pkg/util/proto k8s.io/kube-openapi/pkg/util/proto/validation k8s.io/kube-openapi/pkg/validation/spec -# k8s.io/kubectl v0.36.2 +# k8s.io/kubectl v0.36.3 ## explicit; go 1.26.0 k8s.io/kubectl/pkg/cmd/util k8s.io/kubectl/pkg/metricsutil @@ -1500,14 +1500,14 @@ k8s.io/kubectl/pkg/util/openapi k8s.io/kubectl/pkg/util/templates k8s.io/kubectl/pkg/util/term k8s.io/kubectl/pkg/validation -# k8s.io/metrics v0.36.2 +# k8s.io/metrics v0.36.3 ## explicit; go 1.26.0 k8s.io/metrics/pkg/apis/metrics k8s.io/metrics/pkg/apis/metrics/v1alpha1 k8s.io/metrics/pkg/apis/metrics/v1beta1 k8s.io/metrics/pkg/client/clientset/versioned/scheme k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1 -# k8s.io/streaming v0.36.2 +# k8s.io/streaming v0.36.3 ## explicit; go 1.26.0 k8s.io/streaming/pkg/httpstream k8s.io/streaming/pkg/httpstream/spdy