ci: add MCP tool name validation against live servers

.github/workflows/mcp-tool-validation.yml

@@ -0,0 +1,111 @@
+name: MCP Tool Validation
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+    paths:
+      - '**/mcps.json'
+      - '**/skills/*/SKILL.md'
+  push:
+    branches: [main]
+    paths:
+      - '**/mcps.json'
+      - '**/skills/*/SKILL.md'
+  workflow_dispatch:
+    inputs:
+      pack:
+        description: 'Pack name to validate (empty = all packs)'
+        required: false
+        type: string
+
+permissions:
+  contents: read
+
+concurrency:
+  group: mcp-tool-validation-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  mcp-tool-check:
+    if: github.event.pull_request.draft == false || github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          fetch-depth: 0  # full history required for git diff in pack detection
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: '3.11'
+
+      - name: Install podman
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq podman
+
+      - name: Create dummy kubeconfig
+        run: |
+          mkdir -p "$HOME/.kube"
+          cat > "$HOME/.kube/config" <<'KUBECONFIG'
+          apiVersion: v1
+          kind: Config
+          clusters:
+          - cluster:
+              server: https://localhost:6443
+            name: mcp-validation
+          contexts:
+          - context:
+              cluster: mcp-validation
+              user: mcp-validation
+            name: mcp-validation
+          current-context: mcp-validation
+          users:
+          - name: mcp-validation
+            user:
+              token: dummy-token-for-tool-listing
+          KUBECONFIG
+          echo "KUBECONFIG=$HOME/.kube/config" >> "$GITHUB_ENV"
+
+      - name: Detect changed packs
+        if: github.event_name == 'pull_request'
+        id: detect
+        env:
+          BASE_REF: ${{ github.base_ref }}
+        run: |
+          CHANGED=$(git diff --name-only "origin/$BASE_REF...HEAD" \
+            | grep -oP '^[^/]+(?=/(?:mcps\.json|skills/))' \
+            | sort -u \
+            | tr '\n' ' ')
+          if [ -z "$CHANGED" ]; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+            echo "packs=$CHANGED" >> "$GITHUB_OUTPUT"
+            echo "Changed packs: $CHANGED"
+          fi
+
+      - name: Validate MCP tools
+        env:
+          KUBECONFIG: ${{ env.KUBECONFIG }}
+          EVENT_NAME: ${{ github.event_name }}
+          INPUT_PACK: ${{ inputs.pack }}
+          CHANGED_PACKS: ${{ steps.detect.outputs.packs }}
+          PACKS_CHANGED: ${{ steps.detect.outputs.changed }}
+        run: |
+          # Dispatch: manual with pack > PR changed packs > all packs
+          if [ "$EVENT_NAME" = "workflow_dispatch" ] && [ -n "$INPUT_PACK" ]; then
+            echo "Manual dispatch: validating pack '$INPUT_PACK'"
+            python scripts/validate_mcp_tools.py "$INPUT_PACK"
+          elif [ "$EVENT_NAME" = "pull_request" ] && [ "$PACKS_CHANGED" != "true" ]; then
+            echo "No packs changed in this PR, skipping"
+          elif [ "$EVENT_NAME" = "pull_request" ]; then
+            echo "Validating changed packs: $CHANGED_PACKS"
+            python scripts/validate_mcp_tools.py $CHANGED_PACKS
+          else
+            echo "Validating all packs..."
+            python scripts/validate_mcp_tools.py
+          fi

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help install validate validate-collection-schema validate-collection-compliance catalog-mirror-json validate-skill-design validate-skill-design-changed generate serve clean test test-full check-uv
+.PHONY: help install validate validate-collection-schema validate-collection-compliance catalog-mirror-json validate-skill-design validate-skill-design-changed validate-mcp-tools generate serve clean test test-full check-uv
 
 help:
 	@echo "agentic-collections Documentation Generator"
@@ -11,6 +11,7 @@ help:
 	@echo "  catalog-mirror-json           - Regenerate all .catalog/collection.json from YAML"
 	@echo "  validate-skill-design         - Validate all skills (use PACK=rh-sre for a specific pack)"
 	@echo "  validate-skill-design-changed - Validate only changed skills (staged + unstaged, for local dev)"
+	@echo "  validate-mcp-tools            - Validate allowed-tools against live MCP servers (requires podman)"
 	@echo "  generate    - Generate docs/data.json"
 	@echo "  serve       - Start local server on http://localhost:8000"
 	@echo "  test        - Quick test (validate + generate + verify)"
@@ -42,7 +43,9 @@ validate: check-uv
 	@uv run python scripts/validate_structure.py
 	@echo "Validating collection compliance (.catalog/)..."
 	@uv run python scripts/validate_collection_compliance.py
-	@echo "✓ Validation passed!"
+	@echo "Validating MCP tool references (skips gracefully without podman)..."
+	@uv run python scripts/validate_mcp_tools.py
+	@echo "✓ Validation complete!"
 
 validate-collection-schema: check-uv
 	@uv run python scripts/validate_collection_schema.py
@@ -59,6 +62,11 @@ validate-skill-design: check-uv
 validate-skill-design-changed: check-uv
 	@VALIDATE_INCLUDE_UNCOMMITTED=1 ./scripts/ci-validate-changed-skills.sh
 
+validate-mcp-tools: check-uv
+	@echo "Validating MCP tool references against live servers..."
+	@uv run python scripts/validate_mcp_tools.py $(if $(PACK),$(PACK))
+	@echo "✓ MCP tool validation complete!"
+
 generate: check-uv
 	@echo "Generating documentation..."
 	@uv run python scripts/build_website.py