From 2d1b89e86f5c9998cd3904763e66ae29a7f5d1f0 Mon Sep 17 00:00:00 2001 From: bentito Date: Thu, 18 Jun 2026 14:57:14 -0400 Subject: [PATCH 1/8] feat(netedge): Add must-gather offline support for get_service_endpoints This commit implements offline must-gather support for the get_service_endpoints tool, using mustgather.GetProvider() to query the offline index when available. It also includes the necessary mock must-gather data and mcpchecker tasks to validate the implementation. --- evals/gemini-agent/agent.yaml | 2 +- evals/gemini-agent/eval.yaml | 8 ++-- .../get_service_endpoints/offline.yaml | 19 ++++++++++ .../tasks/netedge/inspect_route/offline.yaml | 19 ++++++++++ .../namespaces/openshift-console/routes.yaml | 23 ++++++++++++ .../openshift-ingress/endpointslices.yaml | 21 +++++++++++ evals/testdata/must-gather/timestamp | 1 + evals/testdata/must-gather/version | 1 + pkg/toolsets/netedge/endpoints.go | 37 +++++++++++++++---- 9 files changed, 118 insertions(+), 13 deletions(-) create mode 100644 evals/tasks/netedge/get_service_endpoints/offline.yaml create mode 100644 evals/tasks/netedge/inspect_route/offline.yaml create mode 100644 evals/testdata/must-gather/namespaces/openshift-console/routes.yaml create mode 100644 evals/testdata/must-gather/namespaces/openshift-ingress/endpointslices.yaml create mode 100644 evals/testdata/must-gather/timestamp create mode 100644 evals/testdata/must-gather/version diff --git a/evals/gemini-agent/agent.yaml b/evals/gemini-agent/agent.yaml index a21feef88..30941173c 100644 --- a/evals/gemini-agent/agent.yaml +++ b/evals/gemini-agent/agent.yaml @@ -3,4 +3,4 @@ metadata: name: "gemini-command-agent" acp: cmd: "gemini" - args: ["--acp"] + args: ["--acp", "--yolo"] diff --git a/evals/gemini-agent/eval.yaml b/evals/gemini-agent/eval.yaml index ff07b45b6..759a60ca1 100644 --- a/evals/gemini-agent/eval.yaml +++ b/evals/gemini-agent/eval.yaml @@ -6,10 +6,10 @@ config: type: "file" path: "agent.yaml" mcpConfigFile: "../mcp-config.yaml" - llmJudge: - ref: - type: "builtin.llm-agent" - model: "openai:gemini-2.5-pro" + # llmJudge: + # ref: + # type: "builtin.llm-agent" + # model: "googleai:gemini-2.5-pro" taskSets: - glob: ../tasks/netedge/*/*.yaml assertions: diff --git a/evals/tasks/netedge/get_service_endpoints/offline.yaml b/evals/tasks/netedge/get_service_endpoints/offline.yaml new file mode 100644 index 000000000..f3e42fd6e --- /dev/null +++ b/evals/tasks/netedge/get_service_endpoints/offline.yaml @@ -0,0 +1,19 @@ +kind: Task +metadata: + name: get-service-endpoints-offline + labels: + suite: kubernetes +steps: + verify: + contains: "Addresses" + prompt: + inline: First load the must-gather archive located at 'evals/testdata/must-gather' using the mustgather_use tool. Then use the get_service_endpoints tool to list the endpoint slices for the router-default service in the openshift-ingress namespace. Report the backend pod addresses and node names from the offline result. + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "mustgather_use" + - server: kubernetes + toolPattern: "get_service_endpoints" + args: + namespace: "openshift-ingress" + service: "router-default" \ No newline at end of file diff --git a/evals/tasks/netedge/inspect_route/offline.yaml b/evals/tasks/netedge/inspect_route/offline.yaml new file mode 100644 index 000000000..3ee73a276 --- /dev/null +++ b/evals/tasks/netedge/inspect_route/offline.yaml @@ -0,0 +1,19 @@ +kind: Task +metadata: + name: inspect-route-offline + labels: + suite: kubernetes +steps: + verify: + contains: "console" + prompt: + inline: First load the must-gather archive located at 'evals/testdata/must-gather' using the mustgather_use tool. Then inspect the default OpenShift console route in the openshift-console namespace and report key fields like host, TLS termination type, and backend service based entirely on the offline data. + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "mustgather_use" + - server: kubernetes + toolPattern: "inspect_route" + args: + namespace: "openshift-console" + route: "console" \ No newline at end of file diff --git a/evals/testdata/must-gather/namespaces/openshift-console/routes.yaml b/evals/testdata/must-gather/namespaces/openshift-console/routes.yaml new file mode 100644 index 000000000..f9b1d98d2 --- /dev/null +++ b/evals/testdata/must-gather/namespaces/openshift-console/routes.yaml @@ -0,0 +1,23 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: console + namespace: openshift-console +spec: + host: console-openshift-console.apps.example.com + port: + targetPort: https + tls: + termination: reencrypt + insecureEdgeTerminationPolicy: Redirect + to: + kind: Service + name: console + weight: 100 +status: + ingress: + - host: console-openshift-console.apps.example.com + routerName: default + conditions: + - type: Admitted + status: "True" diff --git a/evals/testdata/must-gather/namespaces/openshift-ingress/endpointslices.yaml b/evals/testdata/must-gather/namespaces/openshift-ingress/endpointslices.yaml new file mode 100644 index 000000000..9f9804fc6 --- /dev/null +++ b/evals/testdata/must-gather/namespaces/openshift-ingress/endpointslices.yaml @@ -0,0 +1,21 @@ +apiVersion: discovery.k8s.io/v1 +kind: EndpointSlice +metadata: + name: router-default-abcde + namespace: openshift-ingress + labels: + kubernetes.io/service-name: router-default +endpoints: + - addresses: + - "10.128.0.5" + nodeName: "node-1" + - addresses: + - "10.128.1.6" + nodeName: "node-2" +ports: + - name: http + port: 80 + protocol: TCP + - name: https + port: 443 + protocol: TCP diff --git a/evals/testdata/must-gather/timestamp b/evals/testdata/must-gather/timestamp new file mode 100644 index 000000000..2157328cb --- /dev/null +++ b/evals/testdata/must-gather/timestamp @@ -0,0 +1 @@ +2024-01-01T00:00:00Z \ No newline at end of file diff --git a/evals/testdata/must-gather/version b/evals/testdata/must-gather/version new file mode 100644 index 000000000..09ce0ce71 --- /dev/null +++ b/evals/testdata/must-gather/version @@ -0,0 +1 @@ +4.14.0 \ No newline at end of file diff --git a/pkg/toolsets/netedge/endpoints.go b/pkg/toolsets/netedge/endpoints.go index 878eb21d5..ee7fac80d 100644 --- a/pkg/toolsets/netedge/endpoints.go +++ b/pkg/toolsets/netedge/endpoints.go @@ -4,6 +4,8 @@ import ( "fmt" "github.com/containers/kubernetes-mcp-server/pkg/api" + mg "github.com/containers/kubernetes-mcp-server/pkg/ocp/mustgather" + "github.com/containers/kubernetes-mcp-server/pkg/toolsets/mustgather" "github.com/google/jsonschema-go/jsonschema" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -63,20 +65,39 @@ func getServiceEndpoints(params api.ToolHandlerParams) (*api.ToolCallResult, err // EndpointSlices are linked to a service via the "kubernetes.io/service-name" label labelSelector := "kubernetes.io/service-name=" + serviceName - list, err := params.DynamicClient().Resource(gvr).Namespace(namespace).List(params.Context, metav1.ListOptions{ - LabelSelector: labelSelector, - }) - if err != nil { - return api.NewToolCallResult("", fmt.Errorf("failed to list EndpointSlices for service %s/%s: %w", namespace, serviceName, err)), nil + var items []unstructured.Unstructured + + if p, mgErr := mustgather.GetProvider(); mgErr == nil && p != nil { + gvk := schema.GroupVersionKind{Group: "discovery.k8s.io", Version: "v1", Kind: "EndpointSlice"} + list, err := p.ListResources(params.Context, gvk, namespace, mg.ListOptions{}) + if err != nil { + return api.NewToolCallResult("", fmt.Errorf("failed to list EndpointSlices for service %s/%s from must-gather: %w", namespace, serviceName, err)), nil + } + + // Filter by label selector + for _, item := range list.Items { + labels := item.GetLabels() + if labels != nil && labels["kubernetes.io/service-name"] == serviceName { + items = append(items, item) + } + } + } else { + list, err := params.DynamicClient().Resource(gvr).Namespace(namespace).List(params.Context, metav1.ListOptions{ + LabelSelector: labelSelector, + }) + if err != nil { + return api.NewToolCallResult("", fmt.Errorf("failed to list EndpointSlices for service %s/%s: %w", namespace, serviceName, err)), nil + } + items = list.Items } - if len(list.Items) == 0 { + if len(items) == 0 { return api.NewToolCallResult("", fmt.Errorf("no EndpointSlices found for service %s/%s", namespace, serviceName)), nil } // Extract KeyFields from EndpointSlices var keyFields []map[string]interface{} - for _, eps := range list.Items { + for _, eps := range items { kf := map[string]interface{}{ "Name": eps.GetName(), "Namespace": eps.GetNamespace(), @@ -112,7 +133,7 @@ func getServiceEndpoints(params api.ToolHandlerParams) (*api.ToolCallResult, err resultObj := map[string]interface{}{ "KeyFields": keyFields, - "RawEndpointSlices": list.Items, + "RawEndpointSlices": items, } data, err := yaml.Marshal(resultObj) From 3e669b0b5850c0be1525081cd94c7a1f8411a1a6 Mon Sep 17 00:00:00 2001 From: bentito Date: Thu, 18 Jun 2026 14:57:38 -0400 Subject: [PATCH 2/8] fix(mustgather): Export GetProvider for cross-package use This commit exports GetProvider as a public function in pkg/toolsets/mustgather/helpers.go so that other packages (like netedge) can access the offline provider when needed. It updates all existing internal references to match the new casing. --- pkg/toolsets/mustgather/etcd.go | 4 +- pkg/toolsets/mustgather/eval_offline_test.go | 67 ++++++++++++++++++++ pkg/toolsets/mustgather/events.go | 6 +- pkg/toolsets/mustgather/helpers.go | 4 +- pkg/toolsets/mustgather/mcp_resources.go | 14 ++-- pkg/toolsets/mustgather/monitoring.go | 10 +-- pkg/toolsets/mustgather/nodes.go | 6 +- pkg/toolsets/mustgather/pod_logs.go | 6 +- pkg/toolsets/mustgather/resources.go | 2 +- pkg/toolsets/netedge/routes.go | 17 ++++- 10 files changed, 107 insertions(+), 29 deletions(-) create mode 100644 pkg/toolsets/mustgather/eval_offline_test.go diff --git a/pkg/toolsets/mustgather/etcd.go b/pkg/toolsets/mustgather/etcd.go index 7d8a20925..8c4973dc2 100644 --- a/pkg/toolsets/mustgather/etcd.go +++ b/pkg/toolsets/mustgather/etcd.go @@ -48,7 +48,7 @@ func initEtcd() []api.ServerTool { } func mustgatherETCDHealth(params api.ToolHandlerParams) (*api.ToolCallResult, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return api.NewToolCallResult("", err), nil } @@ -83,7 +83,7 @@ func mustgatherETCDHealth(params api.ToolHandlerParams) (*api.ToolCallResult, er } func mustgatherETCDObjectCount(params api.ToolHandlerParams) (*api.ToolCallResult, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return api.NewToolCallResult("", err), nil } diff --git a/pkg/toolsets/mustgather/eval_offline_test.go b/pkg/toolsets/mustgather/eval_offline_test.go new file mode 100644 index 000000000..3746a23a2 --- /dev/null +++ b/pkg/toolsets/mustgather/eval_offline_test.go @@ -0,0 +1,67 @@ +package mustgather + +import ( + "context" + "fmt" + "testing" + + "github.com/containers/kubernetes-mcp-server/pkg/api" + mg "github.com/containers/kubernetes-mcp-server/pkg/ocp/mustgather" + "github.com/containers/kubernetes-mcp-server/pkg/toolsets/netedge" +) + +type mockRequest struct { + args map[string]any +} + +func (r mockRequest) GetArguments() map[string]any { + return r.args +} + +func TestEvalOffline(t *testing.T) { + // 1. Load the must-gather provider + archivePath := "/Users/btofel/workspace/openshift-mcp-server/evals/testdata/must-gather" + p, err := mg.NewProvider(archivePath) + if err != nil { + t.Fatalf("failed to create must-gather provider: %v", err) + } + setProvider(p) + fmt.Println("Loaded must-gather archive successfully") + + // 2. Get the "get_service_endpoints" tool from netedge toolset + netedgeToolset := &netedge.Toolset{} + tools := netedgeToolset.GetTools(nil) + var endpointsTool *api.ServerTool + for _, tool := range tools { + if tool.Tool.Name == "get_service_endpoints" { + endpointsTool = &tool + break + } + } + if endpointsTool == nil { + t.Fatalf("get_service_endpoints tool not found in netedge toolset") + } + + // 3. Call get_service_endpoints + params := api.ToolHandlerParams{ + Context: context.Background(), + ToolCallRequest: mockRequest{ + args: map[string]any{ + "namespace": "openshift-ingress", + "service": "router-default", + }, + }, + } + + res, err := endpointsTool.Handler(params) + if err != nil { + t.Fatalf("handler returned error: %v", err) + } + if res.Error != nil { + t.Fatalf("tool execution error: %v", res.Error) + } + + fmt.Println("--- RESULT START ---") + fmt.Println(res.Content) + fmt.Println("--- RESULT END ---") +} diff --git a/pkg/toolsets/mustgather/events.go b/pkg/toolsets/mustgather/events.go index e69520f1b..e03c64094 100644 --- a/pkg/toolsets/mustgather/events.go +++ b/pkg/toolsets/mustgather/events.go @@ -88,7 +88,7 @@ func initEvents() []api.ServerTool { } func mustgatherEventsList(params api.ToolHandlerParams) (*api.ToolCallResult, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return api.NewToolCallResult("", err), nil } @@ -159,7 +159,7 @@ func mustgatherEventsList(params api.ToolHandlerParams) (*api.ToolCallResult, er } func mustgatherEventsByResource(params api.ToolHandlerParams) (*api.ToolCallResult, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return api.NewToolCallResult("", err), nil } @@ -215,7 +215,7 @@ func mustgatherEventsByResource(params api.ToolHandlerParams) (*api.ToolCallResu } func mustgatherEventsByTime(params api.ToolHandlerParams) (*api.ToolCallResult, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return api.NewToolCallResult("", err), nil } diff --git a/pkg/toolsets/mustgather/helpers.go b/pkg/toolsets/mustgather/helpers.go index 379592576..551e5ca07 100644 --- a/pkg/toolsets/mustgather/helpers.go +++ b/pkg/toolsets/mustgather/helpers.go @@ -20,8 +20,8 @@ func setProvider(p *mg.Provider) { provider = p } -// getProvider returns the loaded must-gather provider or an error -func getProvider() (*mg.Provider, error) { +// GetProvider returns the loaded must-gather provider or an error +func GetProvider() (*mg.Provider, error) { providerMu.RLock() defer providerMu.RUnlock() if provider == nil { diff --git a/pkg/toolsets/mustgather/mcp_resources.go b/pkg/toolsets/mustgather/mcp_resources.go index 63022c26c..424bddbee 100644 --- a/pkg/toolsets/mustgather/mcp_resources.go +++ b/pkg/toolsets/mustgather/mcp_resources.go @@ -84,7 +84,7 @@ func initMCPResourceTemplates() []api.ServerResourceTemplate { } func resourceCurrentArchive(_ context.Context) (*api.ResourceContent, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return nil, err } @@ -96,7 +96,7 @@ func resourceCurrentArchive(_ context.Context) (*api.ResourceContent, error) { } func resourceNamespaces(_ context.Context) (*api.ResourceContent, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return nil, err } @@ -108,7 +108,7 @@ func resourceNamespaces(_ context.Context) (*api.ResourceContent, error) { } func resourceETCDMembers(_ context.Context) (*api.ResourceContent, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return nil, err } @@ -120,7 +120,7 @@ func resourceETCDMembers(_ context.Context) (*api.ResourceContent, error) { } func resourceETCDEndpointStatus(_ context.Context) (*api.ResourceContent, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return nil, err } @@ -132,7 +132,7 @@ func resourceETCDEndpointStatus(_ context.Context) (*api.ResourceContent, error) } func resourcePrometheusConfig(_ context.Context) (*api.ResourceContent, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return nil, err } @@ -179,7 +179,7 @@ func resourcePrometheusConfig(_ context.Context) (*api.ResourceContent, error) { } func resourceAlertManagerStatus(_ context.Context) (*api.ResourceContent, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return nil, err } @@ -212,7 +212,7 @@ func resourceAlertManagerStatus(_ context.Context) (*api.ResourceContent, error) } func resourceGet(_ context.Context, uri string) (*api.ResourceContent, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return nil, err } diff --git a/pkg/toolsets/mustgather/monitoring.go b/pkg/toolsets/mustgather/monitoring.go index 53f1bc77e..d1433f2ea 100644 --- a/pkg/toolsets/mustgather/monitoring.go +++ b/pkg/toolsets/mustgather/monitoring.go @@ -108,7 +108,7 @@ func initMonitoring() []api.ServerTool { } func mustgatherMonitoringPrometheusStatus(params api.ToolHandlerParams) (*api.ToolCallResult, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return api.NewToolCallResult("", err), nil } @@ -154,7 +154,7 @@ func mustgatherMonitoringPrometheusStatus(params api.ToolHandlerParams) (*api.To } func mustgatherMonitoringPrometheusTargets(params api.ToolHandlerParams) (*api.ToolCallResult, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return api.NewToolCallResult("", err), nil } @@ -211,7 +211,7 @@ func mustgatherMonitoringPrometheusTargets(params api.ToolHandlerParams) (*api.T } func mustgatherMonitoringPrometheusTSDB(params api.ToolHandlerParams) (*api.ToolCallResult, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return api.NewToolCallResult("", err), nil } @@ -246,7 +246,7 @@ func mustgatherMonitoringPrometheusTSDB(params api.ToolHandlerParams) (*api.Tool } func mustgatherMonitoringPrometheusAlerts(params api.ToolHandlerParams) (*api.ToolCallResult, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return api.NewToolCallResult("", err), nil } @@ -313,7 +313,7 @@ func mustgatherMonitoringPrometheusAlerts(params api.ToolHandlerParams) (*api.To } func mustgatherMonitoringPrometheusRules(params api.ToolHandlerParams) (*api.ToolCallResult, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return api.NewToolCallResult("", err), nil } diff --git a/pkg/toolsets/mustgather/nodes.go b/pkg/toolsets/mustgather/nodes.go index c7c3f2f44..422dd3a19 100644 --- a/pkg/toolsets/mustgather/nodes.go +++ b/pkg/toolsets/mustgather/nodes.go @@ -79,7 +79,7 @@ func initNodes() []api.ServerTool { } func mustgatherNodeDiagnosticsGet(params api.ToolHandlerParams) (*api.ToolCallResult, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return api.NewToolCallResult("", err), nil } @@ -146,7 +146,7 @@ func mustgatherNodeDiagnosticsGet(params api.ToolHandlerParams) (*api.ToolCallRe } func mustgatherNodeKubeletLogs(params api.ToolHandlerParams) (*api.ToolCallResult, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return api.NewToolCallResult("", err), nil } @@ -183,7 +183,7 @@ func mustgatherNodeKubeletLogs(params api.ToolHandlerParams) (*api.ToolCallResul } func mustgatherNodeKubeletLogsGrep(params api.ToolHandlerParams) (*api.ToolCallResult, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return api.NewToolCallResult("", err), nil } diff --git a/pkg/toolsets/mustgather/pod_logs.go b/pkg/toolsets/mustgather/pod_logs.go index 20b3d576c..7f0a3350e 100644 --- a/pkg/toolsets/mustgather/pod_logs.go +++ b/pkg/toolsets/mustgather/pod_logs.go @@ -94,7 +94,7 @@ func initPodLogs() []api.ServerTool { } func mustgatherPodLogsGet(params api.ToolHandlerParams) (*api.ToolCallResult, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return api.NewToolCallResult("", err), nil } @@ -142,7 +142,7 @@ func mustgatherPodLogsGet(params api.ToolHandlerParams) (*api.ToolCallResult, er } func mustgatherPodLogsGrep(params api.ToolHandlerParams) (*api.ToolCallResult, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return api.NewToolCallResult("", err), nil } @@ -251,7 +251,7 @@ func mustgatherPodLogsGrep(params api.ToolHandlerParams) (*api.ToolCallResult, e } func mustgatherPodLogsByTime(params api.ToolHandlerParams) (*api.ToolCallResult, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return api.NewToolCallResult("", err), nil } diff --git a/pkg/toolsets/mustgather/resources.go b/pkg/toolsets/mustgather/resources.go index c6d67e0c8..754697d59 100644 --- a/pkg/toolsets/mustgather/resources.go +++ b/pkg/toolsets/mustgather/resources.go @@ -42,7 +42,7 @@ func initResources() []api.ServerTool { } func mustgatherResourcesList(params api.ToolHandlerParams) (*api.ToolCallResult, error) { - p, err := getProvider() + p, err := GetProvider() if err != nil { return api.NewToolCallResult("", err), nil } diff --git a/pkg/toolsets/netedge/routes.go b/pkg/toolsets/netedge/routes.go index 3377c90b4..2f118d2c1 100644 --- a/pkg/toolsets/netedge/routes.go +++ b/pkg/toolsets/netedge/routes.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/containers/kubernetes-mcp-server/pkg/api" + "github.com/containers/kubernetes-mcp-server/pkg/toolsets/mustgather" "github.com/google/jsonschema-go/jsonschema" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -60,9 +61,19 @@ func inspectRoute(params api.ToolHandlerParams) (*api.ToolCallResult, error) { Resource: "routes", } - route, err := params.DynamicClient().Resource(gvr).Namespace(namespace).Get(params.Context, routeName, metav1.GetOptions{}) - if err != nil { - return api.NewToolCallResult("", fmt.Errorf("failed to get route %s/%s: %w", namespace, routeName, err)), nil + var route *unstructured.Unstructured + + if p, mgErr := mustgather.GetProvider(); mgErr == nil && p != nil { + gvk := schema.GroupVersionKind{Group: "route.openshift.io", Version: "v1", Kind: "Route"} + route = p.GetResource(gvk, routeName, namespace) + if route == nil { + return api.NewToolCallResult("", fmt.Errorf("route %s/%s not found in must-gather archive", namespace, routeName)), nil + } + } else { + route, err = params.DynamicClient().Resource(gvr).Namespace(namespace).Get(params.Context, routeName, metav1.GetOptions{}) + if err != nil { + return api.NewToolCallResult("", fmt.Errorf("failed to get route %s/%s: %w", namespace, routeName, err)), nil + } } // Deep-copy the route so we can redact sensitive TLS fields without From e1037378c197e46f322ff47a8b5b590be36a04be Mon Sep 17 00:00:00 2001 From: bentito Date: Thu, 18 Jun 2026 15:47:37 -0400 Subject: [PATCH 3/8] feat(netedge): Add must-gather offline support for get_coredns_config This commit implements offline must-gather support for the get_coredns_config tool, using mustgather.GetProvider() to query the offline index when available. It also includes the necessary mock must-gather data and mcpchecker tasks to validate the implementation. --- .../netedge/get-coredns-config/offline.yaml | 16 ++++++++++++++ .../namespaces/openshift-dns/configmaps.yaml | 22 +++++++++++++++++++ pkg/toolsets/netedge/coredns.go | 18 ++++++++++++--- 3 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 evals/tasks/netedge/get-coredns-config/offline.yaml create mode 100644 evals/testdata/must-gather/namespaces/openshift-dns/configmaps.yaml diff --git a/evals/tasks/netedge/get-coredns-config/offline.yaml b/evals/tasks/netedge/get-coredns-config/offline.yaml new file mode 100644 index 000000000..7d3ed1b05 --- /dev/null +++ b/evals/tasks/netedge/get-coredns-config/offline.yaml @@ -0,0 +1,16 @@ +kind: Task +metadata: + name: get-coredns-config-offline + labels: + suite: kubernetes +steps: + verify: + contains: "prometheus :9153" + prompt: + inline: First load the must-gather archive located at 'evals/testdata/must-gather' using the mustgather_use tool. Then use the get_coredns_config tool to retrieve the CoreDNS configuration from the offline result. + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "mustgather_use" + - server: kubernetes + toolPattern: "get_coredns_config" \ No newline at end of file diff --git a/evals/testdata/must-gather/namespaces/openshift-dns/configmaps.yaml b/evals/testdata/must-gather/namespaces/openshift-dns/configmaps.yaml new file mode 100644 index 000000000..52125a999 --- /dev/null +++ b/evals/testdata/must-gather/namespaces/openshift-dns/configmaps.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: dns-default + namespace: openshift-dns +data: + Corefile: | + .:5353 { + errors + health + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + upstream + fallthrough in-addr.arpa ip6.arpa + } + prometheus :9153 + forward . /etc/resolv.conf + cache 30 + loop + reload + loadbalance + } \ No newline at end of file diff --git a/pkg/toolsets/netedge/coredns.go b/pkg/toolsets/netedge/coredns.go index 6ade4bf2a..2b53c9556 100644 --- a/pkg/toolsets/netedge/coredns.go +++ b/pkg/toolsets/netedge/coredns.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/containers/kubernetes-mcp-server/pkg/api" + "github.com/containers/kubernetes-mcp-server/pkg/toolsets/mustgather" "github.com/google/jsonschema-go/jsonschema" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -39,9 +40,20 @@ func getCoreDNSConfig(params api.ToolHandlerParams) (*api.ToolCallResult, error) Resource: "configmaps", } - cm, err := params.DynamicClient().Resource(gvr).Namespace("openshift-dns").Get(params.Context, "dns-default", metav1.GetOptions{}) - if err != nil { - return api.NewToolCallResult("", fmt.Errorf("failed to get dns-default ConfigMap: %w", err)), nil + var cm *unstructured.Unstructured + var err error + + if p, mgErr := mustgather.GetProvider(); mgErr == nil && p != nil { + gvk := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} + cm = p.GetResource(gvk, "dns-default", "openshift-dns") + if cm == nil { + return api.NewToolCallResult("", fmt.Errorf("ConfigMap openshift-dns/dns-default not found in must-gather archive")), nil + } + } else { + cm, err = params.DynamicClient().Resource(gvr).Namespace("openshift-dns").Get(params.Context, "dns-default", metav1.GetOptions{}) + if err != nil { + return api.NewToolCallResult("", fmt.Errorf("failed to get dns-default ConfigMap: %w", err)), nil + } } data, found, err := unstructured.NestedStringMap(cm.Object, "data") From 9bc5bbd3300bf0b870a3248a4680c836710d576a Mon Sep 17 00:00:00 2001 From: bentito Date: Fri, 26 Jun 2026 10:08:20 -0400 Subject: [PATCH 4/8] fix(mustgather): Resolve test import cycle by converting eval test to separate package --- pkg/toolsets/mustgather/eval_offline_test.go | 5 +++-- pkg/toolsets/mustgather/helpers.go | 4 ++-- pkg/toolsets/mustgather/use.go | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/toolsets/mustgather/eval_offline_test.go b/pkg/toolsets/mustgather/eval_offline_test.go index 3746a23a2..f3178ac9b 100644 --- a/pkg/toolsets/mustgather/eval_offline_test.go +++ b/pkg/toolsets/mustgather/eval_offline_test.go @@ -1,4 +1,4 @@ -package mustgather +package mustgather_test import ( "context" @@ -7,6 +7,7 @@ import ( "github.com/containers/kubernetes-mcp-server/pkg/api" mg "github.com/containers/kubernetes-mcp-server/pkg/ocp/mustgather" + "github.com/containers/kubernetes-mcp-server/pkg/toolsets/mustgather" "github.com/containers/kubernetes-mcp-server/pkg/toolsets/netedge" ) @@ -25,7 +26,7 @@ func TestEvalOffline(t *testing.T) { if err != nil { t.Fatalf("failed to create must-gather provider: %v", err) } - setProvider(p) + mustgather.SetProvider(p) fmt.Println("Loaded must-gather archive successfully") // 2. Get the "get_service_endpoints" tool from netedge toolset diff --git a/pkg/toolsets/mustgather/helpers.go b/pkg/toolsets/mustgather/helpers.go index 551e5ca07..a94980a9c 100644 --- a/pkg/toolsets/mustgather/helpers.go +++ b/pkg/toolsets/mustgather/helpers.go @@ -13,8 +13,8 @@ var ( provider *mg.Provider ) -// setProvider stores the loaded must-gather provider -func setProvider(p *mg.Provider) { +// SetProvider stores the loaded must-gather provider +func SetProvider(p *mg.Provider) { providerMu.Lock() defer providerMu.Unlock() provider = p diff --git a/pkg/toolsets/mustgather/use.go b/pkg/toolsets/mustgather/use.go index 5acee15d5..0d515ab55 100644 --- a/pkg/toolsets/mustgather/use.go +++ b/pkg/toolsets/mustgather/use.go @@ -48,7 +48,7 @@ func mustgatherUse(params api.ToolHandlerParams) (*api.ToolCallResult, error) { return api.NewToolCallResult("", fmt.Errorf("failed to load must-gather archive: %w", err)), nil } - setProvider(p) + SetProvider(p) metadata := p.GetMetadata() From 2930e37519814fb1116919d8e42cbd2407ccb154 Mon Sep 17 00:00:00 2001 From: bentito Date: Fri, 26 Jun 2026 10:32:53 -0400 Subject: [PATCH 5/8] fix(mustgather): Use relative path to testdata in offline eval test --- pkg/toolsets/mustgather/eval_offline_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/toolsets/mustgather/eval_offline_test.go b/pkg/toolsets/mustgather/eval_offline_test.go index f3178ac9b..ad1a59e01 100644 --- a/pkg/toolsets/mustgather/eval_offline_test.go +++ b/pkg/toolsets/mustgather/eval_offline_test.go @@ -21,7 +21,7 @@ func (r mockRequest) GetArguments() map[string]any { func TestEvalOffline(t *testing.T) { // 1. Load the must-gather provider - archivePath := "/Users/btofel/workspace/openshift-mcp-server/evals/testdata/must-gather" + archivePath := "../../../evals/testdata/must-gather" p, err := mg.NewProvider(archivePath) if err != nil { t.Fatalf("failed to create must-gather provider: %v", err) From 4739f0f049635634b6a0d229e791fcd9a13f39b2 Mon Sep 17 00:00:00 2001 From: bentito Date: Fri, 26 Jun 2026 13:47:04 -0400 Subject: [PATCH 6/8] fix(mustgather): Address CodeRabbit feedback on black-box offline testing and strengthen task assertions --- .../tasks/netedge/inspect_route/offline.yaml | 2 +- pkg/toolsets/mustgather/eval_offline_test.go | 36 +++++++++++++++---- pkg/toolsets/mustgather/helpers.go | 4 +-- pkg/toolsets/mustgather/use.go | 2 +- 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/evals/tasks/netedge/inspect_route/offline.yaml b/evals/tasks/netedge/inspect_route/offline.yaml index 3ee73a276..9fcb43519 100644 --- a/evals/tasks/netedge/inspect_route/offline.yaml +++ b/evals/tasks/netedge/inspect_route/offline.yaml @@ -5,7 +5,7 @@ metadata: suite: kubernetes steps: verify: - contains: "console" + contains: "console-openshift-console.apps.example.com" prompt: inline: First load the must-gather archive located at 'evals/testdata/must-gather' using the mustgather_use tool. Then inspect the default OpenShift console route in the openshift-console namespace and report key fields like host, TLS termination type, and backend service based entirely on the offline data. assertions: diff --git a/pkg/toolsets/mustgather/eval_offline_test.go b/pkg/toolsets/mustgather/eval_offline_test.go index ad1a59e01..5984c6052 100644 --- a/pkg/toolsets/mustgather/eval_offline_test.go +++ b/pkg/toolsets/mustgather/eval_offline_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/containers/kubernetes-mcp-server/pkg/api" - mg "github.com/containers/kubernetes-mcp-server/pkg/ocp/mustgather" "github.com/containers/kubernetes-mcp-server/pkg/toolsets/mustgather" "github.com/containers/kubernetes-mcp-server/pkg/toolsets/netedge" ) @@ -20,14 +19,37 @@ func (r mockRequest) GetArguments() map[string]any { } func TestEvalOffline(t *testing.T) { - // 1. Load the must-gather provider - archivePath := "../../../evals/testdata/must-gather" - p, err := mg.NewProvider(archivePath) + // 1. Load the must-gather provider via mustgather_use tool + mustgatherToolset := &mustgather.Toolset{} + mustgatherTools := mustgatherToolset.GetTools(nil) + var useTool *api.ServerTool + for _, tool := range mustgatherTools { + if tool.Tool.Name == "mustgather_use" { + useTool = &tool + break + } + } + if useTool == nil { + t.Fatalf("mustgather_use tool not found") + } + + useParams := api.ToolHandlerParams{ + Context: context.Background(), + ToolCallRequest: mockRequest{ + args: map[string]any{ + "path": "../../../evals/testdata/must-gather", + }, + }, + } + + useRes, err := useTool.Handler(useParams) if err != nil { - t.Fatalf("failed to create must-gather provider: %v", err) + t.Fatalf("mustgather_use returned error: %v", err) + } + if useRes.Error != nil { + t.Fatalf("mustgather_use execution error: %v", useRes.Error) } - mustgather.SetProvider(p) - fmt.Println("Loaded must-gather archive successfully") + fmt.Println("Loaded must-gather archive successfully via tool") // 2. Get the "get_service_endpoints" tool from netedge toolset netedgeToolset := &netedge.Toolset{} diff --git a/pkg/toolsets/mustgather/helpers.go b/pkg/toolsets/mustgather/helpers.go index a94980a9c..551e5ca07 100644 --- a/pkg/toolsets/mustgather/helpers.go +++ b/pkg/toolsets/mustgather/helpers.go @@ -13,8 +13,8 @@ var ( provider *mg.Provider ) -// SetProvider stores the loaded must-gather provider -func SetProvider(p *mg.Provider) { +// setProvider stores the loaded must-gather provider +func setProvider(p *mg.Provider) { providerMu.Lock() defer providerMu.Unlock() provider = p diff --git a/pkg/toolsets/mustgather/use.go b/pkg/toolsets/mustgather/use.go index 0d515ab55..5acee15d5 100644 --- a/pkg/toolsets/mustgather/use.go +++ b/pkg/toolsets/mustgather/use.go @@ -48,7 +48,7 @@ func mustgatherUse(params api.ToolHandlerParams) (*api.ToolCallResult, error) { return api.NewToolCallResult("", fmt.Errorf("failed to load must-gather archive: %w", err)), nil } - SetProvider(p) + setProvider(p) metadata := p.GetMetadata() From 3c7a023fe26f9091632550132a49a85fd731017f Mon Sep 17 00:00:00 2001 From: bentito Date: Tue, 28 Jul 2026 12:06:38 -0400 Subject: [PATCH 7/8] chore(evals): Migrate netedge evaluations to decentralized suite-specific configs Refactor NIDS/netedge evaluations to adopt the new core-eval-testing format. Standardize netedge task labels, remove deprecated global and nested eval configs, and sync build/evals.mk to support per-suite execution. --- build/evals.mk | 23 ++--- evals/claude-code/agent.yaml | 5 -- evals/claude-code/eval.yaml | 86 ------------------- .../acp-anthropic/eval-netedge.yaml | 44 ++++++++++ .../acp-google/eval-netedge.yaml | 44 ++++++++++ .../builtin-anthropic/eval-netedge.yaml | 44 ++++++++++ .../builtin-google/eval-netedge.yaml | 44 ++++++++++ .../builtin-openai/eval-netedge.yaml | 44 ++++++++++ evals/gemini-agent/agent.yaml | 6 -- evals/gemini-agent/eval.yaml | 38 -------- evals/openai-agent/agent.yaml | 9 -- evals/openai-agent/eval.yaml | 84 ------------------ evals/tasks/kubevirt/claude-code/eval.yaml | 23 ----- evals/tasks/kubevirt/openai-agent/eval.yaml | 19 ---- .../netedge/get-coredns-config/offline.yaml | 4 +- .../netedge/get-coredns-config/task.yaml | 2 + .../tasks/netedge/get-router-config/task.yaml | 2 + evals/tasks/netedge/get-router-info/task.yaml | 2 + .../netedge/get-router-sessions/task.yaml | 2 + .../get_service_endpoints/offline.yaml | 4 +- .../netedge/get_service_endpoints/task.yaml | 2 + .../tasks/netedge/inspect_route/offline.yaml | 4 +- evals/tasks/netedge/inspect_route/task.yaml | 2 + evals/tasks/netedge/probe_dns_local/task.yaml | 2 + evals/tasks/netedge/probe_http/task.yaml | 2 + .../tasks/netedge/query_prometheus/task.yaml | 2 + 26 files changed, 256 insertions(+), 287 deletions(-) delete mode 100644 evals/claude-code/agent.yaml delete mode 100644 evals/claude-code/eval.yaml create mode 100644 evals/core-eval-testing/acp-anthropic/eval-netedge.yaml create mode 100644 evals/core-eval-testing/acp-google/eval-netedge.yaml create mode 100644 evals/core-eval-testing/builtin-anthropic/eval-netedge.yaml create mode 100644 evals/core-eval-testing/builtin-google/eval-netedge.yaml create mode 100644 evals/core-eval-testing/builtin-openai/eval-netedge.yaml delete mode 100644 evals/gemini-agent/agent.yaml delete mode 100644 evals/gemini-agent/eval.yaml delete mode 100644 evals/openai-agent/agent.yaml delete mode 100644 evals/openai-agent/eval.yaml delete mode 100644 evals/tasks/kubevirt/claude-code/eval.yaml delete mode 100644 evals/tasks/kubevirt/openai-agent/eval.yaml diff --git a/build/evals.mk b/build/evals.mk index 6e35bde70..45bc43020 100644 --- a/build/evals.mk +++ b/build/evals.mk @@ -11,17 +11,18 @@ CLAUDE_AGENT_ACP = $(shell pwd)/_output/tools/node_modules/.bin/claude-agent-acp CLAUDE_AGENT_ACP_VERSION ?= latest # High-level knobs for local single-suite runs, e.g.: -# make run-evals SUITE=kubevirt AGENT=claude-code MODEL=sonnet -# AGENT selects the eval config, SUITE selects the task suite label, and MODEL -# sets ANTHROPIC_MODEL for the claude-agent-acp adapter (the openai-agent ignores it). -AGENT ?= openai-agent +# make run-evals SUITE=kubevirt AGENT=acp-google +# AGENT selects the agent directory under evals/, SUITE selects the task suite +# label, and MODEL sets ANTHROPIC_MODEL for ACP agents (builtin agents ignore it). +# Available agents: builtin-openai, builtin-anthropic, builtin-google, +# acp-anthropic (Claude Code via ACP), acp-google (Gemini via ACP) +AGENT ?= builtin-openai SUITE ?= core MODEL ?= -# Prefer a per-suite eval config when one exists: those carry no llmJudge, so a -# local run needs no OpenAI key. Otherwise fall back to the agent's top-level -# config (the one CI uses; its llmJudge requires an OpenAI key). -EVAL_CONFIG ?= $(or $(wildcard evals/tasks/$(SUITE)/$(AGENT)/eval.yaml),evals/$(AGENT)/eval.yaml) +# Prefer a per-suite eval config when one exists, then try the core-eval-testing +# suite config (what CI uses). +EVAL_CONFIG ?= $(or $(wildcard evals/tasks/$(SUITE)/$(AGENT)/eval.yaml),evals/core-eval-testing/$(AGENT)/eval-$(SUITE).yaml) EVAL_LABEL_SELECTOR ?= suite=$(SUITE) EVAL_TASK_FILTER ?= EVAL_VERBOSE ?= false @@ -39,9 +40,9 @@ mcpchecker: ##@ Evals # Install the claude-agent-acp adapter locally under _output/tools, required by -# the claude-code eval agent (evals/claude-code/agent.yaml runs `claude-agent-acp`). +# the acp-anthropic eval agent (runs `claude-agent-acp`). .PHONY: claude-agent-acp -claude-agent-acp: ## Install the claude-agent-acp adapter for the claude-code eval agent +claude-agent-acp: ## Install the claude-agent-acp adapter for the acp-anthropic eval agent @[ -f $(CLAUDE_AGENT_ACP) ] || { \ set -e ;\ echo "Installing claude-agent-acp@$(CLAUDE_AGENT_ACP_VERSION) to $(CLAUDE_AGENT_ACP)..." ;\ @@ -50,7 +51,7 @@ claude-agent-acp: ## Install the claude-agent-acp adapter for the claude-code ev } .PHONY: run-evals -run-evals: mcpchecker $(if $(filter claude-code,$(AGENT)),claude-agent-acp) ## Run mcpchecker evals (knobs: SUITE, AGENT, MODEL; see evals/README.md) +run-evals: mcpchecker $(if $(filter acp-anthropic,$(AGENT)),claude-agent-acp) ## Run mcpchecker evals (knobs: SUITE, AGENT, MODEL; see evals/README.md) $(if $(MODEL),ANTHROPIC_MODEL=$(MODEL) )PATH="$(shell pwd)/_output/tools/node_modules/.bin:$(PATH)" $(MCPCHECKER) check $(EVAL_CONFIG) \ $(if $(EVAL_LABEL_SELECTOR),--label-selector $(EVAL_LABEL_SELECTOR),) \ $(if $(EVAL_TASK_FILTER),--run "$(EVAL_TASK_FILTER)",) \ diff --git a/evals/claude-code/agent.yaml b/evals/claude-code/agent.yaml deleted file mode 100644 index 8c9186911..000000000 --- a/evals/claude-code/agent.yaml +++ /dev/null @@ -1,5 +0,0 @@ -kind: Agent -metadata: - name: "claude-code-acp" -acp: - cmd: "claude-agent-acp" diff --git a/evals/claude-code/eval.yaml b/evals/claude-code/eval.yaml deleted file mode 100644 index 8fe931c9c..000000000 --- a/evals/claude-code/eval.yaml +++ /dev/null @@ -1,86 +0,0 @@ -kind: Eval -metadata: - name: "kubernetes-basic-operations" -config: - agent: - type: "file" - path: agent.yaml - mcpConfigFile: ../mcp-config.yaml - extensions: - kubernetes: - package: https://github.com/mcpchecker/kubernetes-extension@v0.0.4 - # The claude-code agent judges with claude-code too, so the whole flow (agent + - # judge) runs on one Anthropic/Claude-subscription auth with no OpenAI key. - # (The openai-agent config keeps an independent openai:gpt-5 judge for CI.) - llmJudge: - ref: - type: builtin.claude-code - taskSets: - # Kubernetes tasks - - glob: ../tasks/*/*/*.yaml - labelSelector: - suite: core - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 - - glob: ../tasks/config/*/*.yaml - labelSelector: - suite: config - assertions: - toolsUsed: - - server: kubernetes - toolPattern: "configuration_.*" - minToolCalls: 1 - maxToolCalls: 10 - - glob: ../tasks/helm/*/*.yaml - labelSelector: - suite: helm - assertions: - toolsUsed: - - server: kubernetes - toolPattern: "helm_.*" - minToolCalls: 1 - maxToolCalls: 10 - # KubeVirt tasks - - glob: ../tasks/kubevirt/*/*.yaml - labelSelector: - suite: kubevirt - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 - # Kiali tasks - - glob: ../tasks/kiali/*/*.yaml - labelSelector: - suite: kiali - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 - # Tekton tasks - - glob: ../tasks/tekton/*/*.yaml - labelSelector: - suite: tekton - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 - # NetObserv tasks - - glob: ../tasks/netobserv/*/*.yaml - labelSelector: - suite: netobserv - assertions: - toolsUsed: - - server: kubernetes - toolPattern: "netobserv_.*" - minToolCalls: 1 - maxToolCalls: 10 diff --git a/evals/core-eval-testing/acp-anthropic/eval-netedge.yaml b/evals/core-eval-testing/acp-anthropic/eval-netedge.yaml new file mode 100644 index 000000000..12896e614 --- /dev/null +++ b/evals/core-eval-testing/acp-anthropic/eval-netedge.yaml @@ -0,0 +1,44 @@ +kind: Eval +metadata: + name: "netedge-e2e" +config: + agent: + type: "file" + path: agent.yaml + mcpConfigFile: ../../mcp-config.yaml + extensions: + kubernetes: + package: https://github.com/mcpchecker/kubernetes-extension@v0.0.3 + llmJudge: + env: + baseUrlKey: JUDGE_BASE_URL + apiKeyKey: JUDGE_API_KEY + modelNameKey: JUDGE_MODEL_NAME + taskSets: + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: core + assertions: + toolsUsed: + - server: kubernetes + toolPattern: ".*" + minToolCalls: 1 + maxToolCalls: 20 + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: config + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "configuration_.*" + minToolCalls: 1 + maxToolCalls: 10 + - glob: ../../tasks/netedge/*/*.yaml + labelSelector: + suite: netedge + assertions: + toolsUsed: + - server: kubernetes + toolPattern: ".*" + minToolCalls: 1 + maxToolCalls: 20 diff --git a/evals/core-eval-testing/acp-google/eval-netedge.yaml b/evals/core-eval-testing/acp-google/eval-netedge.yaml new file mode 100644 index 000000000..12896e614 --- /dev/null +++ b/evals/core-eval-testing/acp-google/eval-netedge.yaml @@ -0,0 +1,44 @@ +kind: Eval +metadata: + name: "netedge-e2e" +config: + agent: + type: "file" + path: agent.yaml + mcpConfigFile: ../../mcp-config.yaml + extensions: + kubernetes: + package: https://github.com/mcpchecker/kubernetes-extension@v0.0.3 + llmJudge: + env: + baseUrlKey: JUDGE_BASE_URL + apiKeyKey: JUDGE_API_KEY + modelNameKey: JUDGE_MODEL_NAME + taskSets: + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: core + assertions: + toolsUsed: + - server: kubernetes + toolPattern: ".*" + minToolCalls: 1 + maxToolCalls: 20 + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: config + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "configuration_.*" + minToolCalls: 1 + maxToolCalls: 10 + - glob: ../../tasks/netedge/*/*.yaml + labelSelector: + suite: netedge + assertions: + toolsUsed: + - server: kubernetes + toolPattern: ".*" + minToolCalls: 1 + maxToolCalls: 20 diff --git a/evals/core-eval-testing/builtin-anthropic/eval-netedge.yaml b/evals/core-eval-testing/builtin-anthropic/eval-netedge.yaml new file mode 100644 index 000000000..12896e614 --- /dev/null +++ b/evals/core-eval-testing/builtin-anthropic/eval-netedge.yaml @@ -0,0 +1,44 @@ +kind: Eval +metadata: + name: "netedge-e2e" +config: + agent: + type: "file" + path: agent.yaml + mcpConfigFile: ../../mcp-config.yaml + extensions: + kubernetes: + package: https://github.com/mcpchecker/kubernetes-extension@v0.0.3 + llmJudge: + env: + baseUrlKey: JUDGE_BASE_URL + apiKeyKey: JUDGE_API_KEY + modelNameKey: JUDGE_MODEL_NAME + taskSets: + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: core + assertions: + toolsUsed: + - server: kubernetes + toolPattern: ".*" + minToolCalls: 1 + maxToolCalls: 20 + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: config + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "configuration_.*" + minToolCalls: 1 + maxToolCalls: 10 + - glob: ../../tasks/netedge/*/*.yaml + labelSelector: + suite: netedge + assertions: + toolsUsed: + - server: kubernetes + toolPattern: ".*" + minToolCalls: 1 + maxToolCalls: 20 diff --git a/evals/core-eval-testing/builtin-google/eval-netedge.yaml b/evals/core-eval-testing/builtin-google/eval-netedge.yaml new file mode 100644 index 000000000..12896e614 --- /dev/null +++ b/evals/core-eval-testing/builtin-google/eval-netedge.yaml @@ -0,0 +1,44 @@ +kind: Eval +metadata: + name: "netedge-e2e" +config: + agent: + type: "file" + path: agent.yaml + mcpConfigFile: ../../mcp-config.yaml + extensions: + kubernetes: + package: https://github.com/mcpchecker/kubernetes-extension@v0.0.3 + llmJudge: + env: + baseUrlKey: JUDGE_BASE_URL + apiKeyKey: JUDGE_API_KEY + modelNameKey: JUDGE_MODEL_NAME + taskSets: + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: core + assertions: + toolsUsed: + - server: kubernetes + toolPattern: ".*" + minToolCalls: 1 + maxToolCalls: 20 + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: config + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "configuration_.*" + minToolCalls: 1 + maxToolCalls: 10 + - glob: ../../tasks/netedge/*/*.yaml + labelSelector: + suite: netedge + assertions: + toolsUsed: + - server: kubernetes + toolPattern: ".*" + minToolCalls: 1 + maxToolCalls: 20 diff --git a/evals/core-eval-testing/builtin-openai/eval-netedge.yaml b/evals/core-eval-testing/builtin-openai/eval-netedge.yaml new file mode 100644 index 000000000..12896e614 --- /dev/null +++ b/evals/core-eval-testing/builtin-openai/eval-netedge.yaml @@ -0,0 +1,44 @@ +kind: Eval +metadata: + name: "netedge-e2e" +config: + agent: + type: "file" + path: agent.yaml + mcpConfigFile: ../../mcp-config.yaml + extensions: + kubernetes: + package: https://github.com/mcpchecker/kubernetes-extension@v0.0.3 + llmJudge: + env: + baseUrlKey: JUDGE_BASE_URL + apiKeyKey: JUDGE_API_KEY + modelNameKey: JUDGE_MODEL_NAME + taskSets: + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: core + assertions: + toolsUsed: + - server: kubernetes + toolPattern: ".*" + minToolCalls: 1 + maxToolCalls: 20 + - glob: ../../tasks/*/*/*.yaml + labelSelector: + suite: config + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "configuration_.*" + minToolCalls: 1 + maxToolCalls: 10 + - glob: ../../tasks/netedge/*/*.yaml + labelSelector: + suite: netedge + assertions: + toolsUsed: + - server: kubernetes + toolPattern: ".*" + minToolCalls: 1 + maxToolCalls: 20 diff --git a/evals/gemini-agent/agent.yaml b/evals/gemini-agent/agent.yaml deleted file mode 100644 index 30941173c..000000000 --- a/evals/gemini-agent/agent.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Agent -metadata: - name: "gemini-command-agent" -acp: - cmd: "gemini" - args: ["--acp", "--yolo"] diff --git a/evals/gemini-agent/eval.yaml b/evals/gemini-agent/eval.yaml deleted file mode 100644 index 759a60ca1..000000000 --- a/evals/gemini-agent/eval.yaml +++ /dev/null @@ -1,38 +0,0 @@ -kind: Eval -metadata: - name: "gemini-agent-toolsets-eval" -config: - agent: - type: "file" - path: "agent.yaml" - mcpConfigFile: "../mcp-config.yaml" - # llmJudge: - # ref: - # type: "builtin.llm-agent" - # model: "googleai:gemini-2.5-pro" - taskSets: - - glob: ../tasks/netedge/*/*.yaml - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - # Observability tasks - - glob: ../tasks/observability/*/*.yaml - labelSelector: - suite: observability - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 - # Cluster diagnostics tasks - - glob: ../tasks/cluster-diagnostics/*/*.yaml - labelSelector: - suite: cluster-diagnostics - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 diff --git a/evals/openai-agent/agent.yaml b/evals/openai-agent/agent.yaml deleted file mode 100644 index 60a80cd67..000000000 --- a/evals/openai-agent/agent.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: Agent -metadata: - name: "openai-agent" -builtin: - type: "llm-agent" - model: "openai:gpt-5" # Change to your model -# Before running, set environment variables: -# export MODEL_BASE_URL="https://api.openai.com/v1" -# export MODEL_KEY="sk-..." diff --git a/evals/openai-agent/eval.yaml b/evals/openai-agent/eval.yaml deleted file mode 100644 index ecc61a16b..000000000 --- a/evals/openai-agent/eval.yaml +++ /dev/null @@ -1,84 +0,0 @@ -kind: Eval -metadata: - name: "openai-agent-kubernetes-test" -config: - agent: - type: "file" - path: agent.yaml - mcpConfigFile: ../mcp-config.yaml - extensions: - kubernetes: - package: https://github.com/mcpchecker/kubernetes-extension@v0.0.4 - llmJudge: - ref: - type: builtin.llm-agent - model: "openai:gpt-5" - taskSets: - # Kubernetes tasks - - glob: ../tasks/*/*/*.yaml - labelSelector: - suite: core - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 - - glob: ../tasks/config/*/*.yaml - labelSelector: - suite: config - assertions: - toolsUsed: - - server: kubernetes - toolPattern: "configuration_.*" - minToolCalls: 1 - maxToolCalls: 10 - - glob: ../tasks/helm/*/*.yaml - labelSelector: - suite: helm - assertions: - toolsUsed: - - server: kubernetes - toolPattern: "helm_.*" - minToolCalls: 1 - maxToolCalls: 10 - # KubeVirt tasks - - glob: ../tasks/kubevirt/*/*.yaml - labelSelector: - suite: kubevirt - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 - # Kiali tasks - - glob: ../tasks/kiali/*/*.yaml - labelSelector: - suite: kiali - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 - # Tekton tasks - - glob: ../tasks/tekton/*/*.yaml - labelSelector: - suite: tekton - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 - # NetObserv tasks - - glob: ../tasks/netobserv/*/*.yaml - labelSelector: - suite: netobserv - assertions: - toolsUsed: - - server: kubernetes - toolPattern: "netobserv_.*" - minToolCalls: 1 - maxToolCalls: 10 diff --git a/evals/tasks/kubevirt/claude-code/eval.yaml b/evals/tasks/kubevirt/claude-code/eval.yaml deleted file mode 100644 index 9c0362f43..000000000 --- a/evals/tasks/kubevirt/claude-code/eval.yaml +++ /dev/null @@ -1,23 +0,0 @@ -kind: Eval -metadata: - name: "kubevirt-basic-operations" -config: - agent: - type: "builtin.claude-code" - mcpConfigFile: ../../../mcp-config.yaml - extensions: - kubernetes: - package: https://github.com/mcpchecker/kubernetes-extension@v0.0.4 - # Judge with claude-code so judge-backed tasks (verify: contains / llmJudge) - # are really evaluated, keyless on the Claude subscription, like the agent. - llmJudge: - ref: - type: builtin.claude-code - taskSets: - - glob: ../*/task.yaml - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 diff --git a/evals/tasks/kubevirt/openai-agent/eval.yaml b/evals/tasks/kubevirt/openai-agent/eval.yaml deleted file mode 100644 index 2373b19af..000000000 --- a/evals/tasks/kubevirt/openai-agent/eval.yaml +++ /dev/null @@ -1,19 +0,0 @@ -kind: Eval -metadata: - name: "kubevirt-basic-operations" -config: - agent: - type: "builtin.llm-agent" - model: "openai:gpt-5" - mcpConfigFile: ../../../mcp-config.yaml - extensions: - kubernetes: - package: https://github.com/mcpchecker/kubernetes-extension@v0.0.4 - taskSets: - - glob: ../*/task.yaml - assertions: - toolsUsed: - - server: kubernetes - toolPattern: ".*" - minToolCalls: 1 - maxToolCalls: 20 diff --git a/evals/tasks/netedge/get-coredns-config/offline.yaml b/evals/tasks/netedge/get-coredns-config/offline.yaml index 7d3ed1b05..a95bbafde 100644 --- a/evals/tasks/netedge/get-coredns-config/offline.yaml +++ b/evals/tasks/netedge/get-coredns-config/offline.yaml @@ -2,7 +2,7 @@ kind: Task metadata: name: get-coredns-config-offline labels: - suite: kubernetes + suite: netedge steps: verify: contains: "prometheus :9153" @@ -13,4 +13,4 @@ steps: - server: kubernetes toolPattern: "mustgather_use" - server: kubernetes - toolPattern: "get_coredns_config" \ No newline at end of file + toolPattern: "get_coredns_config" diff --git a/evals/tasks/netedge/get-coredns-config/task.yaml b/evals/tasks/netedge/get-coredns-config/task.yaml index 525fe74d5..8f4ba3b15 100644 --- a/evals/tasks/netedge/get-coredns-config/task.yaml +++ b/evals/tasks/netedge/get-coredns-config/task.yaml @@ -1,6 +1,8 @@ kind: Task metadata: name: get-coredns-config + labels: + suite: netedge steps: verify: contains: "5353" diff --git a/evals/tasks/netedge/get-router-config/task.yaml b/evals/tasks/netedge/get-router-config/task.yaml index 6bb6b0d0c..124859833 100644 --- a/evals/tasks/netedge/get-router-config/task.yaml +++ b/evals/tasks/netedge/get-router-config/task.yaml @@ -1,6 +1,8 @@ kind: Task metadata: name: get-router-config + labels: + suite: netedge steps: verify: contains: "frontend public" diff --git a/evals/tasks/netedge/get-router-info/task.yaml b/evals/tasks/netedge/get-router-info/task.yaml index a47d250c8..4ec6af4fe 100644 --- a/evals/tasks/netedge/get-router-info/task.yaml +++ b/evals/tasks/netedge/get-router-info/task.yaml @@ -1,6 +1,8 @@ kind: Task metadata: name: get-router-info + labels: + suite: netedge steps: verify: contains: "Name: HAProxy" diff --git a/evals/tasks/netedge/get-router-sessions/task.yaml b/evals/tasks/netedge/get-router-sessions/task.yaml index 49d52a384..3d60658a5 100644 --- a/evals/tasks/netedge/get-router-sessions/task.yaml +++ b/evals/tasks/netedge/get-router-sessions/task.yaml @@ -1,6 +1,8 @@ kind: Task metadata: name: get-router-sessions + labels: + suite: netedge steps: verify: contains: "frontend=GLOBAL" diff --git a/evals/tasks/netedge/get_service_endpoints/offline.yaml b/evals/tasks/netedge/get_service_endpoints/offline.yaml index f3e42fd6e..4ed4567b0 100644 --- a/evals/tasks/netedge/get_service_endpoints/offline.yaml +++ b/evals/tasks/netedge/get_service_endpoints/offline.yaml @@ -2,7 +2,7 @@ kind: Task metadata: name: get-service-endpoints-offline labels: - suite: kubernetes + suite: netedge steps: verify: contains: "Addresses" @@ -16,4 +16,4 @@ steps: toolPattern: "get_service_endpoints" args: namespace: "openshift-ingress" - service: "router-default" \ No newline at end of file + service: "router-default" diff --git a/evals/tasks/netedge/get_service_endpoints/task.yaml b/evals/tasks/netedge/get_service_endpoints/task.yaml index e0eba81f7..9ec3c83a1 100644 --- a/evals/tasks/netedge/get_service_endpoints/task.yaml +++ b/evals/tasks/netedge/get_service_endpoints/task.yaml @@ -1,6 +1,8 @@ kind: Task metadata: name: get-service-endpoints + labels: + suite: netedge steps: verify: contains: "Addresses" diff --git a/evals/tasks/netedge/inspect_route/offline.yaml b/evals/tasks/netedge/inspect_route/offline.yaml index 9fcb43519..0d8368885 100644 --- a/evals/tasks/netedge/inspect_route/offline.yaml +++ b/evals/tasks/netedge/inspect_route/offline.yaml @@ -2,7 +2,7 @@ kind: Task metadata: name: inspect-route-offline labels: - suite: kubernetes + suite: netedge steps: verify: contains: "console-openshift-console.apps.example.com" @@ -16,4 +16,4 @@ steps: toolPattern: "inspect_route" args: namespace: "openshift-console" - route: "console" \ No newline at end of file + route: "console" diff --git a/evals/tasks/netedge/inspect_route/task.yaml b/evals/tasks/netedge/inspect_route/task.yaml index 6368a602d..60de4e5ce 100644 --- a/evals/tasks/netedge/inspect_route/task.yaml +++ b/evals/tasks/netedge/inspect_route/task.yaml @@ -1,6 +1,8 @@ kind: Task metadata: name: inspect-route + labels: + suite: netedge steps: verify: contains: "console" diff --git a/evals/tasks/netedge/probe_dns_local/task.yaml b/evals/tasks/netedge/probe_dns_local/task.yaml index 45c3d5ac1..d41e58cd5 100644 --- a/evals/tasks/netedge/probe_dns_local/task.yaml +++ b/evals/tasks/netedge/probe_dns_local/task.yaml @@ -1,6 +1,8 @@ kind: Task metadata: name: probe-dns-local + labels: + suite: netedge steps: verify: contains: "NOERROR" diff --git a/evals/tasks/netedge/probe_http/task.yaml b/evals/tasks/netedge/probe_http/task.yaml index aa79bce80..bf0397572 100644 --- a/evals/tasks/netedge/probe_http/task.yaml +++ b/evals/tasks/netedge/probe_http/task.yaml @@ -1,6 +1,8 @@ kind: Task metadata: name: probe-http + labels: + suite: netedge steps: verify: contains: "status_code" diff --git a/evals/tasks/netedge/query_prometheus/task.yaml b/evals/tasks/netedge/query_prometheus/task.yaml index 454b66798..8e0cef01e 100644 --- a/evals/tasks/netedge/query_prometheus/task.yaml +++ b/evals/tasks/netedge/query_prometheus/task.yaml @@ -1,6 +1,8 @@ kind: Task metadata: name: query-prometheus-ingress + labels: + suite: netedge steps: verify: contains: "ingress_error_rate" From 4a17c893c4e50e2fde43fbfa882b0d7633b7eed6 Mon Sep 17 00:00:00 2001 From: bentito Date: Tue, 28 Jul 2026 12:09:30 -0400 Subject: [PATCH 8/8] chore(ci): Migrate mcpchecker workflow to decentralized suite-specific configs Update the mcpchecker.yaml workflow to run evaluations using the newly introduced decentralized suite-specific configurations under core-eval-testing/ instead of the legacy global files. Add support for the netedge suite. --- .github/workflows/mcpchecker.yaml | 140 ++++++++++++++++++------------ 1 file changed, 86 insertions(+), 54 deletions(-) diff --git a/.github/workflows/mcpchecker.yaml b/.github/workflows/mcpchecker.yaml index fda25cf49..0b179f470 100644 --- a/.github/workflows/mcpchecker.yaml +++ b/.github/workflows/mcpchecker.yaml @@ -14,7 +14,7 @@ on: workflow_dispatch: inputs: suite: - description: 'Which task suite to run (core, helm, kubevirt, kiali, tekton, netobserv, or all)' + description: 'Which task suite to run (core, helm, kubevirt, kiali, tekton, netobserv, netedge, or all)' required: false default: 'core' type: choice @@ -25,6 +25,7 @@ on: - kiali - tekton - netobserv + - netedge - all task-filter: description: 'Regular expression to filter tasks (optional)' @@ -76,8 +77,7 @@ jobs: github.event.workflow_run.conclusion == 'success')) outputs: should-run: ${{ steps.check.outputs.should-run }} - toolsets: ${{ steps.suite.outputs.toolsets }} - label-selector: ${{ steps.suite.outputs.label-selector }} + matrix: ${{ steps.suite.outputs.matrix }} pr-number: ${{ steps.check.outputs.pr-number }} pr-sha: ${{ steps.check.outputs.pr-sha }} is-pr: ${{ steps.check.outputs.is-pr }} @@ -181,7 +181,7 @@ jobs: // Parse suite from the review body (trusted — fetched from API). const suiteMatch = triggerReview.body.match( - /\/run-mcpchecker\s+(core|helm|kubevirt|kiali|tekton|netobserv|all)/i, + /\/run-mcpchecker\s+(core|helm|kubevirt|kiali|tekton|netobserv|netedge|all)/i, ); const suite = suiteMatch ? suiteMatch[1].toLowerCase() : ''; @@ -206,54 +206,79 @@ jobs: INPUT_SUITE: ${{ github.event.inputs.suite }} run: | # Suite selection: for workflow_run use the suite parsed from the review - # body (verified via API), for workflow_dispatch use the input, otherwise - # default to 'core'. + # body (verified via API), for workflow_dispatch use the input, for + # schedule run all suites in parallel, otherwise default to 'core'. if [[ "$EVENT_NAME" == "workflow_run" && -n "$VERIFIED_SUITE" ]]; then SUITE="$VERIFIED_SUITE" + elif [[ "$EVENT_NAME" == "schedule" ]]; then + SUITE="schedule-matrix" else SUITE="${INPUT_SUITE:-core}" fi - # Select label-selector and infrastructure based on suite. - # All suites use the same eval.yaml file; suite controls label-selector + infra. + # Build a JSON matrix consumed by run-evaluation via fromJSON(). + # Each entry carries suite name, eval config path, and required toolsets. + EVAL_DIR="evals/core-eval-testing/builtin-openai" + + # Helper: emit a single-entry matrix for the given suite/config/toolsets. + single_matrix() { + printf '{"include":[{"suite":"%s","eval-config":"%s","toolsets":"%s"}]}' "$1" "$2" "$3" + } + case "$SUITE" in + schedule-matrix) + # Weekly schedule: run all individual suites in parallel. + MATRIX=$(cat <<'EOFM' + {"include":[ + {"suite":"core","eval-config":"evals/core-eval-testing/builtin-openai/eval-core.yaml","toolsets":"core,config"}, + {"suite":"helm","eval-config":"evals/core-eval-testing/builtin-openai/eval-helm.yaml","toolsets":"core,config,helm"}, + {"suite":"kubevirt","eval-config":"evals/core-eval-testing/builtin-openai/eval-kubevirt.yaml","toolsets":"core,config,kubevirt,tekton"}, + {"suite":"kiali","eval-config":"evals/core-eval-testing/builtin-openai/eval-kiali.yaml","toolsets":"core,config,kiali"}, + {"suite":"tekton","eval-config":"evals/core-eval-testing/builtin-openai/eval-tekton.yaml","toolsets":"core,config,tekton"}, + {"suite":"netobserv","eval-config":"evals/core-eval-testing/builtin-openai/eval-netobserv.yaml","toolsets":"core,config,netobserv"}, + {"suite":"netedge","eval-config":"evals/core-eval-testing/builtin-openai/eval-netedge.yaml","toolsets":"core,config,netedge"} + ]} + EOFM + ) + ;; helm) - echo "label-selector=suite=helm" >> "$GITHUB_OUTPUT" - echo "toolsets=core,config,helm" >> "$GITHUB_OUTPUT" + MATRIX=$(single_matrix helm "${EVAL_DIR}/eval-helm.yaml" "core,config,helm") ;; kubevirt) - echo "label-selector=suite=kubevirt" >> "$GITHUB_OUTPUT" - echo "toolsets=core,config,kubevirt,tekton" >> "$GITHUB_OUTPUT" + MATRIX=$(single_matrix kubevirt "${EVAL_DIR}/eval-kubevirt.yaml" "core,config,kubevirt,tekton") ;; kiali) - echo "label-selector=suite=kiali" >> "$GITHUB_OUTPUT" - echo "toolsets=core,config,kiali" >> "$GITHUB_OUTPUT" + MATRIX=$(single_matrix kiali "${EVAL_DIR}/eval-kiali.yaml" "core,config,kiali") ;; tekton) - echo "label-selector=suite=tekton" >> "$GITHUB_OUTPUT" - echo "toolsets=core,tekton" >> "$GITHUB_OUTPUT" + MATRIX=$(single_matrix tekton "${EVAL_DIR}/eval-tekton.yaml" "core,config,tekton") ;; netobserv) - echo "label-selector=suite=netobserv" >> "$GITHUB_OUTPUT" - echo "toolsets=core,netobserv" >> "$GITHUB_OUTPUT" + MATRIX=$(single_matrix netobserv "${EVAL_DIR}/eval-netobserv.yaml" "core,config,netobserv") + ;; + netedge) + MATRIX=$(single_matrix netedge "${EVAL_DIR}/eval-netedge.yaml" "core,config,netedge") ;; all) - echo "label-selector=" >> "$GITHUB_OUTPUT" # No filter: run all taskSets - echo "toolsets=core,config,helm,kiali,kubevirt,tekton,netobserv" >> "$GITHUB_OUTPUT" + MATRIX=$(single_matrix all "${EVAL_DIR}/eval-all.yaml" "core,config,helm,kiali,kubevirt,tekton,netobserv,netedge") ;; *) # Default to core suite (matches 'core' or any unrecognized suite) - echo "label-selector=suite=core" >> "$GITHUB_OUTPUT" - echo "toolsets=core,config" >> "$GITHUB_OUTPUT" + MATRIX=$(single_matrix core "${EVAL_DIR}/eval-core.yaml" "core,config") ;; esac + echo "matrix=$(echo "$MATRIX" | jq -c .)" >> "$GITHUB_OUTPUT" + # Run mcpchecker evaluation with Kind cluster run-evaluation: - name: Run MCP Evaluation + name: 'Run MCP Evaluation (${{ matrix.suite }})' needs: check-trigger if: needs.check-trigger.outputs.should-run == 'true' runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.check-trigger.outputs.matrix) }} steps: - name: Checkout uses: actions/checkout@v7.0.1 @@ -277,46 +302,44 @@ jobs: - name: Setup Kind cluster run: make kind-create-cluster KIND_CLUSTER_NAME="$KIND_CLUSTER_NAME" - # Only the core suite needs a resize-capable StorageClass (for resize-pvc). - # Runs for suite=core and for the "all" run (empty label-selector). + # Core tasks always run (every eval includes them) and the resize-pvc + # task needs a resize-capable StorageClass, so always install the CSI driver. - name: Install CSI hostPath driver (resize-capable StorageClass) - if: needs.check-trigger.outputs.label-selector == 'suite=core' || needs.check-trigger.outputs.label-selector == '' run: make csi-hostpath-install - name: Install Istio/Kiali and bookinfo demo - if: contains(needs.check-trigger.outputs.toolsets, 'kiali') + if: contains(matrix.toolsets, 'kiali') run: make setup-kiali - name: Install KubeVirt - if: contains(needs.check-trigger.outputs.toolsets, 'kubevirt') + if: contains(matrix.toolsets, 'kubevirt') run: make kubevirt-install - name: Install Tekton - if: contains(needs.check-trigger.outputs.toolsets, 'tekton') + if: contains(matrix.toolsets, 'tekton') run: make tekton-install - name: Install NetObserv mock plugin - if: contains(needs.check-trigger.outputs.toolsets, 'netobserv') + if: contains(matrix.toolsets, 'netobserv') run: make setup-netobserv - name: Start MCP server run: make run-server env: - TOOLSETS: ${{ needs.check-trigger.outputs.toolsets }} + TOOLSETS: ${{ matrix.toolsets }} MCP_CONFIG_DIR: 'dev/config/mcp-configs' - name: Run mcpchecker evaluation id: mcpchecker uses: mcpchecker/mcpchecker/.github/actions/mcpchecker-action@v0.0.19 with: - eval-config: 'evals/openai-agent/eval.yaml' + eval-config: ${{ matrix.eval-config }} mcpchecker-version: 'latest' - label-selector: ${{ needs.check-trigger.outputs.label-selector }} task-filter: ${{ github.event.inputs.task-filter || '' }} output-format: 'json' verbose: ${{ github.event.inputs.verbose || 'false' }} upload-artifacts: 'true' - artifact-name: 'mcpchecker-results' + artifact-name: 'mcpchecker-results-${{ matrix.suite }}' fail-on-error: 'false' task-pass-threshold: '0.8' assertion-pass-threshold: '0.8' @@ -342,12 +365,13 @@ jobs: - name: Fetch baseline results from main if: needs.check-trigger.outputs.is-pr == 'true' env: - EVAL_CONFIG: 'evals/openai-agent/eval.yaml' + EVAL_CONFIG: ${{ matrix.eval-config }} run: | - # Derive agent name from eval-config path (same logic as commit-results job) - AGENT_NAME=$(echo "$EVAL_CONFIG" | sed 's|evals/||; s|/eval\.yaml||') + # Derive result name from eval-config path: eval-core.yaml -> builtin-openai-core + SUITE=$(basename "$EVAL_CONFIG" .yaml | sed 's/^eval-//') + RESULT_NAME="builtin-openai-${SUITE}" git fetch origin main --depth=1 - git show "origin/main:evals/results/${AGENT_NAME}-latest.json" > /tmp/baseline-results.json 2>/dev/null || true + git show "origin/main:evals/results/${RESULT_NAME}-latest.json" > /tmp/baseline-results.json 2>/dev/null || true - name: Check diff prerequisites id: diff-check @@ -413,7 +437,7 @@ jobs: if: always() && needs.check-trigger.outputs.is-pr == 'true' uses: actions/upload-artifact@v7 with: - name: eval-context + name: eval-context-${{ matrix.suite }} path: eval-context/ retention-days: 1 @@ -421,8 +445,9 @@ jobs: commit-results: name: Commit Evaluation Results needs: [check-trigger, run-evaluation] - # Only commit results on scheduled runs, not manual dispatch or PR comments - if: always() && github.event_name == 'schedule' && needs.run-evaluation.result == 'success' + # Only commit results on scheduled runs. Use != 'cancelled' so partial + # results from passing suites are still committed when some matrix jobs fail. + if: always() && github.event_name == 'schedule' && needs.run-evaluation.result != 'cancelled' runs-on: ubuntu-latest permissions: contents: write @@ -436,24 +461,31 @@ jobs: - name: Download mcpchecker results uses: actions/download-artifact@v8 with: - name: mcpchecker-results + pattern: mcpchecker-results-* path: mcpchecker-results/ - name: Copy results to evals/results - env: - EVAL_CONFIG: 'evals/openai-agent/eval.yaml' run: | - # Extract agent name from eval-config path - AGENT_NAME=$(echo "$EVAL_CONFIG" | sed 's|evals/||; s|/eval\.yaml||') - mkdir -p evals/results - # Copy the most recent mcpchecker results file with agent-specific name - RESULTS_FILE=$(ls -t mcpchecker-results/mcpchecker-*-out.json 2>/dev/null | head -1) - if [ -z "$RESULTS_FILE" ]; then - echo "Error: No mcpchecker results file found" - exit 1 - fi - cp "$RESULTS_FILE" "evals/results/${AGENT_NAME}-latest.json" + + # Each matrix job uploaded its own artifact (e.g. mcpchecker-results-core). + # Without merge-multiple, each lands in its own subdirectory. + for SUITE_DIR in mcpchecker-results/mcpchecker-results-*; do + [ -d "$SUITE_DIR" ] || continue + + # Extract suite name: mcpchecker-results-core -> core + SUITE=$(basename "$SUITE_DIR" | sed 's/^mcpchecker-results-//') + RESULT_NAME="builtin-openai-${SUITE}" + + RESULTS_FILE=$(ls -t "${SUITE_DIR}"/mcpchecker-*-out.json 2>/dev/null | head -1) + if [ -z "$RESULTS_FILE" ]; then + echo "Warning: No results file found for suite ${SUITE}, skipping" + continue + fi + + cp "$RESULTS_FILE" "evals/results/${RESULT_NAME}-latest.json" + echo "Copied ${SUITE} results to evals/results/${RESULT_NAME}-latest.json" + done - name: Create Pull Request env: