diff --git a/.claude/skills/federation-request/SKILL.md b/.claude/skills/federation-request/SKILL.md index c4ca3c49..bcee4b7b 100644 --- a/.claude/skills/federation-request/SKILL.md +++ b/.claude/skills/federation-request/SKILL.md @@ -1,7 +1,8 @@ --- name: federation-request description: | - Guide users step-by-step through creating a federation PR to register an external agentic pack in the marketplace. + Guide users through creating a federation PR to register an external agentic pack in the marketplace. + Collects only repository URL and pack path, infers other metadata from the repo, and confirms before proceeding. Use when: - "I want to federate my pack" @@ -20,7 +21,7 @@ allowed-tools: Read Edit Write Bash Glob Grep Skill # Federation Request -Guide users through creating a complete federation PR — from gathering module metadata to opening the pull request with the `federation` label. Assumes the user has no prior knowledge of the federation process. +Guide users through creating a complete federation PR — from discovering pack metadata in an external repo to opening the pull request with the `federation` label. Only repository URL and pack path are asked upfront; other fields are inferred and confirmed before any writes. ## Prerequisites @@ -59,66 +60,89 @@ Do NOT use when: ## Workflow -### Phase 1: Gather Module Data +### Phase 1: Discover and confirm module metadata -Ask the user for each field, one at a time. Explain what each field is and provide examples. Validate each answer before moving to the next question. +Collect only two inputs from the user, then infer the rest from the repository. -**Fields to collect:** +#### Step 1 — Ask for inputs (only these two) -1. **name** (required) - - Ask: "What is the module name? Use kebab-case (e.g., `partner-network-tools`). This will be the identifier in the marketplace." - - Validate: kebab-case, 1-64 chars, `a-z0-9-`, no consecutive `--`, no leading/trailing `-` - - Validate uniqueness: check it does not already exist in `marketplace/rh-agentic-collection.yml` +1. **repository** — Git repository URL (e.g., `https://github.com/org/repo`) + - Validate: valid URL; must NOT be `https://github.com/RHEcosystemAppEng/agentic-collections` (direct contribution, not federation) -2. **description** (required) - - Ask: "Describe the module in 1-2 sentences. What does it do and who is it for?" - - Validate: non-empty, under 200 characters +2. **path** — Path to the skill pack inside the repo (e.g., `.`, `plugins/claude-code-setup`, `my-pack`) + - Validate: non-empty; use `.` only when the pack root is the repo root -3. **version** (required) - - Ask: "What version is the module? Use semver format (e.g., `1.0.0`, `0.1.0`)." - - Validate: matches semver pattern `X.Y.Z` +Do **not** ask for name, description, version, ref, tags, or maturity individually — infer them in Step 2. -4. **repository** (required) - - Ask: "What is the Git repository URL? (e.g., `https://github.com/org/repo`)" - - Validate: valid URL, must NOT be `https://github.com/RHEcosystemAppEng/agentic-collections` (that would be a direct contribution, not federation) +#### Step 2 — Clone and infer fields automatically -5. **license** (required) - - Ask: "What license does the repository use? Must be compatible with Apache 2.0. Compatible licenses: Apache-2.0, MIT, BSD-2-Clause, BSD-3-Clause." - - Validate: must be one of the compatible licenses +After the user provides repository and path: -6. **ref** (optional) - - Ask: "Do you want to pin to a specific commit SHA or tag? Leave empty to use the default branch." - - If provided, validate format (40-char hex for SHA, or valid tag name) +1. **Clone** the repository to a temporary directory and resolve the default-branch HEAD commit SHA: -7. **path** (optional, default: `.`) - - Ask: "Where is the Lola pack inside the repository? Use `.` if it's at the repo root, or specify a subdirectory path (e.g., `my-pack`)." - - Default: `.` +```bash +TMP=/tmp/federation-discover-$$ +git clone --quiet --no-checkout "" "$TMP" +REF=$(git -C "$TMP" ls-remote origin HEAD | cut -f1) +git -C "$TMP" checkout --quiet "$REF" +PACK="$TMP/" +``` + +2. **Verify** the pack exists: + +```bash +test -d "$PACK/skills" && echo "✓ Pack found" || echo "✗ No skills/ directory at " +``` + +If verification fails, report the error and ask the user to correct the repository URL or path. Do not continue. + +3. **Infer each field** using this precedence (first match wins): -8. **tags** (required) - - Ask: "List tags for discoverability, comma-separated (e.g., `networking, sdn, troubleshooting`). The tag `federation` will be added automatically." - - Validate: at least 1 tag provided - - Always append `federation` tag if not already included +| Field | Inference rules | +|-------|-------------------| +| **name** | 1) `.claude-plugin/plugin.json` or `.cursor-plugin/plugin.json` → `name` · 2) basename of `path` (if not `.`) · 3) sole `skills//` folder name · Must be kebab-case, unique in `marketplace/rh-agentic-collection.yml` | +| **title** | Human-readable catalog display name · 1) `README.md` first `# ` heading · 2) title-case words from `name` (`claude-code-setup` → `Claude Code Setup`) · 3) short phrase from `plugin.json` `description` · Non-empty, ≤120 chars; used in `collection.yaml` `name:` and `docs/plugins.json` | +| **description** | 1) `plugin.json` → `description` · 2) `README.md` first substantive sentence · 3) first skill `SKILL.md` frontmatter `description` (flatten to one line, ≤200 chars) | +| **version** | 1) `plugin.json` → `version` (normalize to semver `X.Y.Z`) · 2) default `0.1.0` | +| **ref** | `REF` from clone above (40-char commit SHA of default branch HEAD) | +| **tags** | Derive 3–6 kebab-case tags from skill folder names, `plugin.json` keywords, README headings, and path segments; always append `federation` | +| **maturity** | Default **`ORANGE`** (recommended for new federations; not listed on public GitHub Pages until promoted to `GREEN`) | -**After collecting all fields**, present a summary table and ask for confirmation: +4. **Validate** inferred values before presenting: + - **name**: kebab-case, 1–64 chars, not already in `marketplace/rh-agentic-collection.yml` + - **title**: non-empty human-readable string; not identical to raw kebab-case `name` unless no better source exists + - **description**: non-empty, under 200 characters + - **version**: semver `X.Y.Z` + - **ref**: 40 hexadecimal characters (run `uv run python -c "import sys; sys.path.insert(0,'scripts'); import pack_registry as pr; print(pr.federation_ref_error('') or 'ok')"` to verify) + - **tags**: at least one tag besides `federation` + +If inference fails or produces ambiguous results (e.g., multiple skill dirs with no plugin.json and path is `.`), show what was found and ask the user to clarify **only the conflicting field(s)**, then re-infer. + +#### Step 3 — Present summary and wait for confirmation + +Show inferred values in one table. Include maturity and note that the user may override any field before confirming: ``` -## Module Summary - -| Field | Value | -|-------------|-----------------------------------------------| -| Name | | -| Description | | -| Version | | -| Repository | | -| License | | -| Ref | | -| Path | | -| Tags | , , ..., federation | - -Proceed? (yes/no) +## Inferred module metadata + +| Field | Value (inferred) | Source | +|-------------|-----------------------------------------------|---------------------| +| Name | | plugin.json / path | +| Title | | README / name | +| Description | | README / SKILL.md | +| Version | | plugin.json | +| Repository | | (user provided) | +| Ref | <40-character commit SHA> | default branch HEAD | +| Path | | (user provided) | +| Tags | , , ..., federation | skills / README | +| Maturity | ORANGE | default (new federation) | + +Reply **yes** to proceed, or tell me what to change (e.g., "use GREEN maturity", "title should be …", "name should be foo-bar"). ``` -Wait for explicit user confirmation before continuing. +Wait for explicit user confirmation before Phase 2. Apply any corrections the user requests, then confirm again if values changed materially. + +Keep the temporary clone at `$TMP` through Phase 2 if no corrections require re-cloning; otherwise re-clone at the confirmed `ref`. ### Phase 2: Create Marketplace Entry @@ -129,9 +153,8 @@ Wait for explicit user confirmation before continuing. - name: "" description: "" version: "" - license: "" repository: "" - ref: "" # omit this line if no ref was provided + ref: "" # required project extension: 40-character commit SHA path: "" tags: - "" @@ -141,17 +164,28 @@ Wait for explicit user confirmation before continuing. 3. **Output to user**: "Added module entry to `marketplace/rh-agentic-collection.yml`." +4. **Action**: Add a display-title entry to `docs/plugins.json`. Key **must** match the marketplace module `name` (not the repo path): + +```json +"": { + "title": "" +} +``` + + - Merge into existing `docs/plugins.json`; preserve JSON formatting (2-space indent, trailing newline). + - Skip if the key already exists — update `title` only when the user corrected it in Phase 1. + - **Output to user**: "Added `docs/plugins.json` entry for ``." + ### Phase 3: Generate Collection Files -1. **Action**: Clone the external repository to a temporary directory: +1. **Action**: Reuse the Phase 1 clone if still valid at the confirmed `ref`, or clone again: ```bash -git clone --quiet --depth 1 /tmp/federation- -# If ref was provided: -cd /tmp/federation- && git fetch --depth 1 origin && git checkout +git clone --quiet --no-checkout /tmp/federation- +cd /tmp/federation- && git checkout --quiet ``` -2. **Action**: Verify the pack exists at the declared path: +2. **Action**: Verify the pack exists at the declared path (skip if already verified in Phase 1): ```bash test -d /tmp/federation-//skills && echo "✓ Pack found" || echo "✗ No skills/ directory at " @@ -170,6 +204,9 @@ mkdir -p federation/modules/ Since `/create-collection` expects the pack to be a local directory registered in the marketplace, work as follows: - Point `/create-collection` to the cloned pack at `/tmp/federation-//` - After generation, copy the resulting `.catalog/` contents to `federation/modules//.catalog/` + - Set **`id:`** to the module `name` (kebab-case marketplace identifier) + - Set **`name:`** in `collection.yaml` to the confirmed **`title`** (must match `docs/plugins.json` → `title`) + - Set **`maturity:`** in `collection.yaml` to the value confirmed in Phase 1 (default `ORANGE`), then regenerate `collection.json` with `uv run python scripts/catalog_yaml_to_json.py --pack federation/modules/` ```bash mkdir -p federation/modules//.catalog @@ -198,7 +235,7 @@ git checkout -b feat/federate- 2. **Action**: Stage all changes: ```bash -git add marketplace/rh-agentic-collection.yml federation/modules// +git add marketplace/rh-agentic-collection.yml docs/plugins.json federation/modules// ``` 3. **Action**: Show the user what will be committed: @@ -235,19 +272,20 @@ Adds **** as a federated module from [](). | Field | Value | |-------------|------------------| | Name | | +| Title | | | Version | <version> | -| License | <license> | | Path | <path> | -| Ref | <ref or default> | +| Ref | <commit-sha> | ### What's Included - Module entry in `marketplace/rh-agentic-collection.yml` +- Display title in `docs/plugins.json` - Collection catalog at `federation/modules/<name>/.catalog/` ### Validation -CI will run automated federation validation (license, Tier 1, Tier 2, MCP pinning, credential scan) when the `federation` label is detected. +CI will run automated federation validation (repo LICENSE check, Tier 1, Tier 2, MCP pinning, credential scan) when the `federation` label is detected. EOF )" \ --label "federation" @@ -265,6 +303,7 @@ Present final summary: | Item | Status | |-----------------------|--------| | Marketplace entry | ✅ Added to marketplace/rh-agentic-collection.yml | +| plugins.json title | ✅ Added to docs/plugins.json | | Collection files | ✅ Generated at federation/modules/<name>/.catalog/ | | Pull request | ✅ <PR-URL> | | CI validation | ⏳ Will run automatically (federation label applied) | @@ -294,7 +333,7 @@ None — no MCP tools are invoked. ### Reference Documentation **Internal:** -- [Federation Review Guide](../../../docs/FEDERATION_REVIEW_GUIDE.md) — evaluation criteria for federated packs +- [Federation Review Guide](../../../FEDERATION_REVIEW_GUIDE.md) — evaluation criteria for federated packs - [COLLECTION_SPEC.md](../../../COLLECTION_SPEC.md) — collection catalog specification - [CONTRIBUTING.md](../../../CONTRIBUTING.md) — contribution paths overview @@ -302,14 +341,14 @@ None — no MCP tools are invoked. **MUST confirm before:** -1. **Module data** (Phase 1): Present summary table, wait for "yes" +1. **Inferred metadata** (Phase 1): Present summary table with all inferred fields (including maturity); wait for **yes** or apply user corrections first 2. **Commit** (Phase 4): Show staged changes and proposed message, wait for approval 3. **Push & PR** (Phase 4): Confirm before pushing and creating the PR **NEVER:** - Push without explicit approval - Create commits without confirmation -- Skip the summary confirmation in Phase 1 +- Skip the Phase 1 confirmation step - Proceed if the pack is not found at the declared path ## Example Usage @@ -317,60 +356,35 @@ None — no MCP tools are invoked. ``` User: /federation-request -Skill: Let's set up a federation request to add your external pack to the marketplace. - I'll guide you through each step. - - First — what is the module name? Use kebab-case (e.g., partner-network-tools). - -User: network-diagnostics - -Skill: Got it. Now describe the module in 1-2 sentences. - -User: Network troubleshooting skills for SDN and OVN diagnostics on OpenShift clusters. - -Skill: What version? Use semver (e.g., 1.0.0). - -User: 0.2.0 - -Skill: Repository URL? +Skill: I'll add your external pack to the marketplace. I only need two things: + 1. Git repository URL + 2. Path to the skill pack inside the repo (use "." if it's at the repo root) User: https://github.com/partner-org/net-diag-skills + plugins/network-diagnostics -Skill: License? Must be compatible with Apache 2.0 (Apache-2.0, MIT, BSD-2-Clause, BSD-3-Clause). - -User: MIT - -Skill: Pin to a specific commit or tag? Leave empty for default branch. - -User: v0.2.0 - -Skill: Path inside the repo where the pack lives? Use "." for repo root. - -User: . - -Skill: Tags for discoverability (comma-separated)? "federation" will be added automatically. +Skill: Cloning and inspecting the repository... -User: networking, sdn, ovn, troubleshooting + ## Inferred module metadata -Skill: ## Module Summary - | Field | Value | - |-------------|------------------------------------------------| - | Name | network-diagnostics | - | Description | Network troubleshooting skills for SDN and ... | - | Version | 0.2.0 | - | Repository | https://github.com/partner-org/net-diag-skills | - | License | MIT | - | Ref | v0.2.0 | - | Path | . | - | Tags | networking, sdn, ovn, troubleshooting, federation | + | Field | Value | Source | + |-------------|------------------------------------------------|---------------------| + | Name | network-diagnostics | plugin.json | + | Title | Network Diagnostics | README.md | + | Description | Network troubleshooting skills for SDN and ... | README.md | + | Version | 0.2.0 | plugin.json | + | Repository | https://github.com/partner-org/net-diag-skills | (user provided) | + | Ref | a1b2c3d4e5f6789012345678901234567890abcd | default branch HEAD | + | Path | plugins/network-diagnostics | (user provided) | + | Tags | networking, sdn, ovn, troubleshooting, federation | skills / README | + | Maturity | ORANGE | default | - Proceed? (yes/no) + Reply **yes** to proceed, or tell me what to change. User: yes Skill: ✅ Added module to marketplace/rh-agentic-collection.yml - ✅ Cloned repository, pack found at root + ✅ Added docs/plugins.json entry ✅ Generated collection files at federation/modules/network-diagnostics/.catalog/ - ✅ Created PR: https://github.com/RHEcosystemAppEng/agentic-collections/pull/42 - Label: federation — CI validation will run automatically. + ... ``` diff --git a/.claude/skills/federation-review/SKILL.md b/.claude/skills/federation-review/SKILL.md index c5452f8b..7965e6bf 100644 --- a/.claude/skills/federation-review/SKILL.md +++ b/.claude/skills/federation-review/SKILL.md @@ -41,18 +41,45 @@ Use this when reviewing a PR that adds or modifies a federated module in `market 2. **Extract:** - Module name - Repository URL - - Ref (if provided) + - Ref (required 40-character commit SHA) - Path within the repo - - License (if declared) -3. **Output to user:** Summary of the proposed federated module +3. **Validate ref format** before continuing: + - Must be exactly 40 hexadecimal characters + - Reject branch names, tags, or short SHAs +4. **Output to user:** Summary of the proposed federated module + +### Phase 1.5: Validate catalog against external repo + +Cross-check `federation/modules/<name>/.catalog/` against the external pack at the pinned `ref`, and verify `docs/plugins.json` / marketplace alignment. + +1. **Action:** Run the catalog cross-check script (skip if the PR does not add a federation catalog yet): + +```bash +uv run python scripts/validate_federation_catalog.py \ + --module-name <module-name> \ + --repo-url <repo-url> \ + --ref <commit-sha> \ + --pack-path <path> \ + --module-json '<module-json-from-marketplace-yaml>' +``` + +2. **Output to user:** Pass/fail for each check: + - Catalog present under `federation/modules/<name>/.catalog/` + - Collection compliance (schema, fragments, JSON mirror) + - **External skill roster** — catalog `contents.skills` matches `skills/` in the linked repo at `ref` + - `docs/plugins.json` title matches catalog `name:`; catalog `id:` matches module name + - Marketplace `version`, `description`, and `repository` align with catalog + +3. **If the catalog is missing** from the PR, report that cross-check was skipped and request the contributor add `federation/modules/<name>/.catalog/` before merge. ### Phase 2: License check -1. **Action:** Check the repository for a LICENSE file: +1. **Action:** Clone the repository at the pinned commit and check for a LICENSE file: ```bash -git clone --quiet --depth 1 <repo-url> /tmp/federation-review -cat /tmp/federation-review/LICENSE +git clone --quiet --no-checkout <repo-url> /tmp/federation-review +cd /tmp/federation-review && git checkout --quiet <commit-sha> +cat LICENSE ``` 2. **Evaluate** license compatibility with Apache 2.0: @@ -62,17 +89,15 @@ cat /tmp/federation-review/LICENSE ### Phase 3: Run automated validation -1. **Action:** Run the validation script: +1. **Action:** Run the validation script (``--ref`` is required and must be a commit SHA): ```bash -uv run python scripts/validate_federation.py <repo-url> --pack-path <path> --module-json '<module-json>' - -# With a specific ref -uv run python scripts/validate_federation.py <repo-url> --ref <ref> --pack-path <path> --module-json '<module-json>' +uv run python scripts/validate_federation.py <repo-url> --ref <commit-sha> --pack-path <path> --module-json '<module-json>' ``` 2. **Output to user:** The full validation report with pass/fail for each check: - - Clone and access + - Federation ref (40-character commit SHA) + - Clone and access at pinned commit - Lola module schema (name, description, version, repository) - Tier 1 (agentskills.io spec) - Tier 2 (design principles) @@ -84,9 +109,10 @@ uv run python scripts/validate_federation.py <repo-url> --ref <ref> --pack-path 1. **Action:** Verify the module is loadable by Lola using the PR branch: ```bash -lola market add test-federation https://raw.githubusercontent.com/<owner>/<repo>/<branch>/marketplace/rh-agentic-collection.yml -lola market ls test-federation -lola market rm test-federation +MARKET="federation-review-$(openssl rand -hex 4)" +lola market add "$MARKET" https://raw.githubusercontent.com/<owner>/<repo>/<branch>/marketplace/rh-agentic-collection.yml +lola market ls "$MARKET" +lola market rm "$MARKET" ``` 2. **Check:** The federated module appears in the module list with correct name, version, and description. @@ -98,6 +124,7 @@ Present a combined summary to the user: | Check | Result | |-------|--------| +| Catalog cross-check | Passed / Failed / Skipped (no catalog in PR) | | License | Compatible / Incompatible / Not found | | Automated validation | All passed / N checks failed | | Lola verification | Module visible / Not visible | @@ -121,12 +148,13 @@ None — no MCP tools are invoked. ### Reference Documentation **Internal:** -- [Federation Review Guide](../../../docs/FEDERATION_REVIEW_GUIDE.md) — full evaluation criteria +- [Federation Review Guide](../../../FEDERATION_REVIEW_GUIDE.md) — full evaluation criteria - [CONTRIBUTING.md](../../../CONTRIBUTING.md) — contribution paths overview - [SKILL_DESIGN_PRINCIPLES.md](../../../SKILL_DESIGN_PRINCIPLES.md) — Tier 2 design principles **Scripts:** -- `scripts/validate_federation.py` — automated validation checks +- `scripts/validate_federation.py` — external pack validation at pinned ref +- `scripts/validate_federation_catalog.py` — federation catalog cross-check vs external repo ## Human-in-the-Loop @@ -141,9 +169,14 @@ Skill: Reading PR diff... Found federated module: cursor-sdk Repository: https://github.com/cursor/plugins Path: cursor-sdk - Ref: (default branch) + Ref: a1b2c3d4e5f6789012345678901234567890abcd + + Checking catalog vs external repo... + uv run python scripts/validate_federation_catalog.py ... + ✅ external_skill_roster matches linked repo at ref + ✅ plugins.json title matches catalog - Checking license... + Checking license at pinned commit... LICENSE file: MIT MIT is compatible with Apache 2.0. Confirm? [Y/n] User: Y @@ -158,6 +191,7 @@ Skill: Running automated validation... ✅ Module "cursor-sdk" visible in marketplace Summary: + - Catalog cross-check: ✅ roster and metadata aligned - License: ✅ MIT (compatible) - Validation: ❌ Tier 1 failures - Lola: ✅ Module loadable diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 47243b05..504909b9 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -55,8 +55,8 @@ - **Repository URL:** <!-- https://github.com/org/repo --> - **Pack path:** <!-- Subdirectory within the repo, or "." for root --> -- **Ref:** <!-- Commit SHA or tag, or "default branch" --> -- **License:** <!-- Must be compatible with Apache 2.0 (Apache-2.0, MIT, BSD-2-Clause, BSD-3-Clause) --> +- **Ref:** <!-- Required 40-character commit SHA (not a branch or tag) --> +- **License:** <!-- Verified from repo LICENSE during review; must be compatible with Apache 2.0 --> - **Contact:** <!-- @github-handle or email of the pack owner --> ## Validation diff --git a/.github/workflows/README.md b/.github/workflows/README.md index e1c68c53..e2a5541f 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -160,6 +160,8 @@ The workflow will: - ✅ Required files presence (README.md, mcps.json, etc.) - ✅ Plugin metadata completeness - ✅ MCP server configurations +- ✅ Collection catalog compliance (`.catalog/` schema, fragments, JSON mirror) +- ✅ Federated catalog cross-check (clone external repos at pinned `ref`; roster and marketplace metadata) **Changed skills validation (`./scripts/ci-validate-changed-skills.sh`):** - ✅ Detects which skills were modified in the PR/push diff --git a/.github/workflows/federation-validation.yml b/.github/workflows/federation-validation.yml index fc3d5e9b..35e47bd4 100644 --- a/.github/workflows/federation-validation.yml +++ b/.github/workflows/federation-validation.yml @@ -103,10 +103,14 @@ jobs: name = mod.get('name', 'unknown') print(f'\n--- Validating: {name} ---') - cmd = [sys.executable, 'scripts/validate_federation.py', repo, '--pack-path', pack_path, '--json', + if not ref: + print(f'Module {name} missing required ref (40-character commit SHA)') + failed = True + all_results.append({'name': name, 'report': None, 'catalog_report': None}) + continue + + cmd = [sys.executable, 'scripts/validate_federation.py', repo, '--ref', ref, '--pack-path', pack_path, '--json', '--module-json', json.dumps(mod)] - if ref: - cmd.extend(['--ref', ref]) if skills: cmd.extend(['--skills'] + skills) @@ -117,13 +121,38 @@ jobs: try: report = json.loads(result.stdout) - all_results.append({'name': name, 'report': report}) except json.JSONDecodeError: - all_results.append({'name': name, 'report': None}) + report = None if result.returncode != 0: failed = True + cat_cmd = [ + sys.executable, 'scripts/validate_federation_catalog.py', + '--module-name', name, + '--repo-url', repo, + '--ref', ref, + '--pack-path', pack_path, + '--module-json', json.dumps(mod), + '--json', + ] + cat_result = subprocess.run(cat_cmd, capture_output=True, text=True) + print(cat_result.stdout) + if cat_result.stderr: + print(cat_result.stderr, file=sys.stderr) + try: + cat_report = json.loads(cat_result.stdout) + except json.JSONDecodeError: + cat_report = None + if cat_result.returncode != 0: + failed = True + + all_results.append({ + 'name': name, + 'report': report, + 'catalog_report': cat_report, + }) + with open('/tmp/federation-results.json', 'w') as f: json.dump(all_results, f) @@ -142,12 +171,18 @@ jobs: const timestamp = new Date().toISOString().replace('T', ' ').substring(0, 19) + ' UTC'; const checkLabels = { - clone: { name: 'Clone & access', desc: 'Repository cloned (pinned ref or default branch)' }, + clone: { name: 'Clone & access', desc: 'Repository cloned at pinned commit SHA' }, + federation_ref: { name: 'Commit SHA', desc: 'ref is a required 40-character commit SHA' }, lola_structure: { name: 'Lola module schema', desc: 'Required fields: name, description, version, repository' }, tier1_agentskills: { name: 'Tier 1', desc: 'agentskills.io spec compliance' }, tier2_design_principles: { name: 'Tier 2', desc: 'Design principles compliance (warning only)' }, mcp_version_pinning: { name: 'MCP pinning', desc: 'No `:latest` tags, no hardcoded credentials' }, gitleaks: { name: 'Credential scan', desc: 'gitleaks secret detection' }, + catalog_present: { name: 'Catalog present', desc: 'federation/modules/<name>/.catalog/collection.yaml exists' }, + catalog_compliance: { name: 'Catalog compliance', desc: 'Schema, fragments, JSON mirror' }, + external_skill_roster: { name: 'Catalog roster', desc: 'Catalog skills match external pack at ref' }, + plugins_json_title: { name: 'plugins.json title', desc: 'Display title matches catalog name/id' }, + marketplace_catalog_metadata: { name: 'Marketplace alignment', desc: 'Version, description, repository match catalog' }, }; // Checks that warn instead of blocking the merge const warnOnlyChecks = new Set(['tier2_design_principles']); @@ -165,12 +200,23 @@ jobs: } moduleSections += '| Check | Description | Status |\n'; moduleSections += '|-------|-------------|--------|\n'; - for (const check of report.checks) { + for check of report.checks) { const label = checkLabels[check.name] || { name: check.name, desc: '' }; const isWarnOnly = warnOnlyChecks.has(check.name); const statusIcon = check.skipped ? '⚠️' : (check.passed ? '✅' : (isWarnOnly ? '⚠️' : '❌')); moduleSections += `| ${label.name} | ${label.desc} | ${statusIcon} |\n`; } + const catReport = mod.catalog_report; + if (catReport && catReport.checks) { + moduleSections += '\n**Catalog cross-check**\n\n'; + moduleSections += '| Check | Description | Status |\n'; + moduleSections += '|-------|-------------|--------|\n'; + for (const check of catReport.checks) { + const label = checkLabels[check.name] || { name: check.name, desc: '' }; + const statusIcon = check.skipped ? '⚠️' : (check.passed ? '✅' : '❌'); + moduleSections += `| ${label.name} | ${label.desc} | ${statusIcon} |\n`; + } + } moduleSections += '\n'; } } catch (e) { diff --git a/CLAUDE.md b/CLAUDE.md index 018051ae..8a7d2e34 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,7 +45,7 @@ There are two ways to add skills to this project: **Direct Contribution** — Skills are added directly to this repository, inside an existing pack. The contributor opens a PR, skills are reviewed and merged, and maintainers own them from that point. Use `/agentic-contribution-skill` in Claude Code or follow [CONTRIBUTING.md](CONTRIBUTING.md). -**Federation** — An external repository containing a complete, independent Lola pack is referenced in our catalog. The code stays in the external repo; users install it directly via Lola. The external owner maintains their pack. To request federation, open a PR adding the module to `marketplace/rh-agentic-collection.yml` with the `federation` label. Maintainers evaluate the pack using the [Federation Review Guide](docs/FEDERATION_REVIEW_GUIDE.md) and the `/federation-review` skill. +**Federation** — An external repository containing a complete, independent Lola pack is referenced in our catalog. The code stays in the external repo; users install it directly via Lola. The external owner maintains their pack. To request federation, open a PR adding the module to `marketplace/rh-agentic-collection.yml` with the `federation` label. Maintainers evaluate the pack using the [Federation Review Guide](FEDERATION_REVIEW_GUIDE.md) and the `/federation-review` skill. ## Working with Agentic Collections diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 638eb788..b9810bf4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,7 +73,7 @@ Both tiers must pass before submitting a PR. ## Resources - [SKILL_DESIGN_PRINCIPLES.md](SKILL_DESIGN_PRINCIPLES.md) -- Design principles and templates -- [Federation Review Guide](docs/FEDERATION_REVIEW_GUIDE.md) -- How external packs are evaluated for federation +- [Federation Review Guide](FEDERATION_REVIEW_GUIDE.md) -- How external packs are evaluated for federation - [agentskills.io specification](https://agentskills.io/specification) -- Base skill standard - [Documentation site](https://rhecosystemappeng.github.io/agentic-collections) -- Browse all collections - [GitHub Issues](https://github.com/RHEcosystemAppEng/agentic-collections/issues) -- Report bugs or ask questions diff --git a/docs/FEDERATION_REVIEW_GUIDE.md b/FEDERATION_REVIEW_GUIDE.md similarity index 73% rename from docs/FEDERATION_REVIEW_GUIDE.md rename to FEDERATION_REVIEW_GUIDE.md index 87985def..834fcc8c 100644 --- a/docs/FEDERATION_REVIEW_GUIDE.md +++ b/FEDERATION_REVIEW_GUIDE.md @@ -20,23 +20,37 @@ CI also runs automated validation on any PR with the `federation` label (see [CI Steps 1–6 can be run with a single command from the agentic-collections repo root: ```bash -# Full pack at repo root (default branch) -uv run python scripts/validate_federation.py <repo-url> - -# At a specific ref -uv run python scripts/validate_federation.py <repo-url> --ref <ref> +# Full pack at repo root, pinned to a commit SHA +uv run python scripts/validate_federation.py <repo-url> --ref <40-character-commit-sha> # Pack in a subdirectory -uv run python scripts/validate_federation.py <repo-url> --pack-path <path> +uv run python scripts/validate_federation.py <repo-url> --ref <commit-sha> --pack-path <path> # Only specific skills -uv run python scripts/validate_federation.py <repo-url> --skills <skill1> <skill2> +uv run python scripts/validate_federation.py <repo-url> --ref <commit-sha> --skills <skill1> <skill2> # JSON output (for CI) -uv run python scripts/validate_federation.py <repo-url> --json +uv run python scripts/validate_federation.py <repo-url> --ref <commit-sha> --json ``` -The script checks: clone access, Lola module schema, Tier 1, Tier 2, MCP version pinning, and gitleaks. If all pass, the pack is ready for manual review (steps 7–8). +The script checks: commit SHA format, clone access at the pinned commit, Lola module schema, Tier 1, Tier 2, MCP version pinning, and gitleaks. + +### Catalog cross-check (in-repo federation artifacts) + +When the PR includes `federation/modules/<name>/.catalog/`, verify the catalog matches the external pack at `ref`: + +```bash +uv run python scripts/validate_federation_catalog.py \ + --module-name <name> \ + --repo-url <repo-url> \ + --ref <commit-sha> \ + --pack-path <path> \ + --module-json '<marketplace-module-json>' +``` + +This checks collection compliance, **skill roster parity** with the linked repo at `ref`, `docs/plugins.json` title alignment, and marketplace version/description/repository consistency. + +If all pass, the pack is ready for manual review (steps 7–8). --- @@ -45,14 +59,14 @@ The script checks: clone access, Lola module schema, Tier 1, Tier 2, MCP version ### Step 1: Verify access and basic info - [ ] Repository URL is reachable and public -- [ ] If a specific ref (SHA or tag) is declared, it exists: `git ls-remote <repo-url> <ref>` +- [ ] `ref` is a 40-character commit SHA (not a branch or tag) and exists: `git ls-remote <repo-url> <commit-sha>` - [ ] Owner/contact information is provided - [ ] License file exists in the repo and is compatible with Apache 2.0 (e.g., Apache-2.0, MIT, BSD-2-Clause, BSD-3-Clause) ```bash git clone --no-checkout <repo-url> /tmp/federation-review cd /tmp/federation-review -git checkout <ref> +git checkout <commit-sha> ``` ### Step 2: Verify Lola module schema @@ -65,6 +79,7 @@ The module entry in `marketplace/rh-agentic-collection.yml` must have the requir | `description` | Yes | Brief description | | `version` | Yes | Module version | | `repository` | Yes | Git URL to the external repo | +| `ref` | Yes (project extension) | 40-character commit SHA pinning the pack (not a Lola field; required by this repo) | If the request is for a **subset of skills** (not the full pack), verify only the listed skill paths exist. @@ -129,7 +144,7 @@ Verify the declared AI agent compatibility from the issue: | Minor issues | Comment on issue with specific fixes, label `federation/changes-requested` | | Major issues | Reject with explanation, label `federation/rejected`, close issue | -When approving, add the pack as a new entry in `modules` in `marketplace/rh-agentic-collection.yml` (include the `license` field from the issue, and use the external repository URL) and link back to the issue. +When approving, add the pack as a new entry in `modules` in `marketplace/rh-agentic-collection.yml` (use the external repository URL; license compatibility is verified from the repo's LICENSE file during review) and link back to the issue. --- @@ -150,14 +165,17 @@ The label-based trigger ensures validation only runs on federation PRs, not on e After merging a federation PR, verify the module is visible in the marketplace: ```bash +# Use a unique ephemeral market name for each verification run +MARKET="federation-review-$(openssl rand -hex 4)" + # Add the marketplace (use the raw YAML URL for a specific branch or main) -lola market add test-federation https://raw.githubusercontent.com/RHEcosystemAppEng/agentic-collections/main/marketplace/rh-agentic-collection.yml +lola market add "$MARKET" https://raw.githubusercontent.com/RHEcosystemAppEng/agentic-collections/main/marketplace/rh-agentic-collection.yml # List modules — the federated pack should appear alongside internal packs -lola market ls test-federation +lola market ls "$MARKET" # Clean up when done -lola market rm test-federation +lola market rm "$MARKET" ``` To test a PR branch before merging, replace `main` with the branch name in the URL. @@ -177,6 +195,7 @@ rm -rf /tmp/federation-review | Public access | Yes | Yes | `git ls-remote` | | License compatibility | Yes | No | Manual review | | Lola module schema | Yes | Yes | `scripts/validate_federation.py` | +| Federation catalog cross-check | Yes | Yes | `scripts/validate_federation_catalog.py` | | Tier 1 (agentskills.io) | Yes | Yes | `scripts/validate_federation.py` | | Tier 2 (design principles) | Yes | Yes | `scripts/validate_federation.py` | | MCP version pinning | Yes | Yes | `scripts/validate_federation.py` | diff --git a/Makefile b/Makefile index afb06769..e2fd2bf4 100644 --- a/Makefile +++ b/Makefile @@ -5,13 +5,13 @@ help: @echo "" @echo "Available targets:" @echo " install - Install Python dependencies (requires uv)" - @echo " validate - Pack structure + collection compliance (.catalog/)" + @echo " validate - Pack structure + collection compliance + federated catalog cross-check" @echo " validate-collection-schema - Schema + roster + banners (subset of compliance)" @echo " validate-collection-compliance - Full .catalog compliance (includes collection.json drift)" @echo " catalog-mirror-json - Regenerate all .catalog/collection.json from YAML" @echo " validate-skill-design - Validate all skills (use PACK=rh-sre for a specific pack)" @echo " validate-skill-design-changed - Validate only changed skills (staged + unstaged, for local dev)" - @echo " validate-federated - Validate federated modules from marketplace YAML" + @echo " validate-federated - Tier 1 skill lint on external federated packs (heavy; catalog cross-check is in validate)" @echo " validate-mcp-tools - Validate allowed-tools against live MCP servers (requires podman)" @echo " generate - Generate docs/data.json" @echo " serve - Start local server on http://localhost:8000" @@ -48,6 +48,8 @@ validate: check-uv @uv run python scripts/validate_docs_tree_links.py @echo "Validating collection compliance (.catalog/)..." @uv run python scripts/validate_collection_compliance.py + @echo "Validating federated catalog cross-check (clone external repos at pinned ref)..." + @uv run python scripts/validate_federation_catalog_all.py @echo "Validating MCP tool references (skips gracefully without podman)..." @uv run python scripts/validate_mcp_tools.py @echo "✓ Validation complete!" @@ -73,7 +75,7 @@ validate-mcp-tools: check-uv @echo "✓ MCP tool validation complete!" validate-federated: check-uv - @echo "Validating federated modules..." + @echo "Validating federated module skills (Tier 1, external clone)..." @uv run python scripts/fetch_federated_skills.py generate: check-uv diff --git a/README.md b/README.md index 5ff3694e..84d41da8 100644 --- a/README.md +++ b/README.md @@ -516,7 +516,7 @@ That CLI checks marketplace/plugin manifests for that workflow, including `plugi - **[CLAUDE.md](CLAUDE.md)**: Repository structure and development workflow - **[Skill Design Principles](SKILL_DESIGN_PRINCIPLES.md)**: Quality guidelines for skills - **[Federation Guide](docs/federation_guide.md)**: How to federate your external pack into the marketplace -- **[Federation Review Guide](docs/FEDERATION_REVIEW_GUIDE.md)**: How we evaluate external packs for federation +- **[Federation Review Guide](FEDERATION_REVIEW_GUIDE.md)**: How we evaluate external packs for federation - **[VALIDATION_REPORT.md](VALIDATION_REPORT.md)**: Marketplace compliance verification - **[Security Policy](SECURITY.md)**: Credential handling and vulnerability reporting diff --git a/marketplace/rh-agentic-collection.yml b/marketplace/rh-agentic-collection.yml index 4191ef3d..8911b545 100644 --- a/marketplace/rh-agentic-collection.yml +++ b/marketplace/rh-agentic-collection.yml @@ -105,35 +105,20 @@ modules: - "troubleshooting" - "execution" - - name: "rh-automation" - description: "Ansible Automation Platform governance, execution safety, and forensic troubleshooting tools for Red Hat automation engineers." - version: "0.1.0" - repository: "https://github.com/r2dedios/agentic-collections" - path: "rh-automation" - tags: - - "ansible" - - "aap" - - "automation" - - "red-hat" - - "governance" - - "troubleshooting" - - "execution" - - "federation" - # Federated modules — complete Lola packs hosted in external repositories. # Add them as regular module entries with a different repository URL. # The build identifies federated packs by comparing the repository field - # against this repo's URL. Extra fields (ref, license, skills) are - # project extensions that Lola silently ignores. + # against this repo's URL. Extra fields (ref, skills) are + # project extensions that Lola silently ignores. Federated modules + # MUST declare ref as a 40-character commit SHA. # To request federation, open a PR adding the module entry with the "federation" label. # # Example: # - name: "partner-network-tools" # description: "Network troubleshooting skills for SDN and OVN diagnostics" # version: "1.2.0" - # license: "Apache-2.0" # must be compatible with Apache 2.0 # repository: "https://github.com/partner-org/network-skills" - # ref: "a1b2c3d4e5f6..." # optional: pinned commit (default: default branch) + # ref: "a1b2c3d4e5f6789012345678901234567890abcd" # required: 40-character commit SHA # path: "." # path within the repo where the Lola pack lives (default: repo root) # skills: # optional: federate only these skills (omit to include all) # - "skills/sdn-diagnostics/SKILL.md" diff --git a/scripts/build_website.py b/scripts/build_website.py index 3f9cc621..b8c8e955 100644 --- a/scripts/build_website.py +++ b/scripts/build_website.py @@ -60,7 +60,8 @@ def build_website(): for pack in pack_data: pack_name = pack['name'] pack['icon'] = icons['packs'].get(pack_name, '') - cat_bundle, cat_warns = bundle_catalog_for_site(pack_name, root) + catalog_dir = pack.get('catalog_dir', pack_name) + cat_bundle, cat_warns = bundle_catalog_for_site(catalog_dir, root) for w in cat_warns: print(f"⚠️ {w}") if cat_bundle is not None: diff --git a/scripts/collection_validate_lib.py b/scripts/collection_validate_lib.py index dc41cb90..877df0c7 100644 --- a/scripts/collection_validate_lib.py +++ b/scripts/collection_validate_lib.py @@ -119,6 +119,127 @@ def read_yaml_catalog(pack_dir: str, root: Optional[Path] = None) -> Tuple[Optio return None, [f"{pack_dir}: failed to parse collection.yaml: {e}"] +def catalog_yaml_path(pack_dir: str, root: Optional[Path] = None) -> Path: + root = root or REPO_ROOT + return root / pack_dir / ".catalog" / "collection.yaml" + + +def _parse_yaml_scalar(value: str) -> str: + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + return value[1:-1] + return value + + +def _find_top_level_key_line(yaml_path: Path, key: str) -> Optional[int]: + pattern = re.compile(rf"^{re.escape(key)}:\s") + for line_no, line in enumerate(yaml_path.read_text(encoding="utf-8").splitlines(), 1): + if pattern.match(line): + return line_no + return None + + +def catalog_skill_name_line_map(yaml_path: Path) -> Dict[str, int]: + """Map skill name to 1-based line number in contents.skills / orchestration_skills.""" + lines = yaml_path.read_text(encoding="utf-8").splitlines() + section: Optional[str] = None + result: Dict[str, int] = {} + for line_no, line in enumerate(lines, 1): + if re.match(r"^ skills:\s*$", line): + section = "skills" + continue + if re.match(r"^ orchestration_skills:\s*$", line): + section = "orchestration_skills" + continue + if section and re.match(r"^ \S", line) and not line.startswith(" "): + section = None + match = re.match(r"^ - name:\s*(.+?)\s*$", line) + if match and section in {"skills", "orchestration_skills"}: + result[_parse_yaml_scalar(match.group(1))] = line_no + return result + + +def catalog_decision_guide_skill_line_map(yaml_path: Path) -> Dict[str, int]: + """Map skill_to_use value to 1-based line number in skills_decision_guide.""" + lines = yaml_path.read_text(encoding="utf-8").splitlines() + in_guide = False + result: Dict[str, int] = {} + for line_no, line in enumerate(lines, 1): + if re.match(r"^ skills_decision_guide:\s*$", line): + in_guide = True + continue + if in_guide and re.match(r"^ \S", line) and not line.startswith(" "): + in_guide = False + match = re.match(r"^ skill_to_use:\s*(.+?)\s*$", line) + if match and in_guide: + result[_parse_yaml_scalar(match.group(1))] = line_no + return result + + +def _yaml_loc(pack_dir: str, yaml_path: Path, line: Optional[int], root: Path) -> str: + rel = yaml_path.relative_to(root) + return f"{rel}:{line}" if line else str(rel) + + +def describe_json_mirror_drift( + pack_dir: str, yaml_data: Dict[str, Any], root: Optional[Path] = None, +) -> List[str]: + """Explain how collection.json differs from collection.yaml with field paths and line numbers.""" + root = root or REPO_ROOT + yaml_path = catalog_yaml_path(pack_dir, root) + json_path = root / pack_dir / ".catalog" / "collection.json" + try: + json_data = json.loads(json_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + return [f"{pack_dir}: .catalog/collection.json is invalid JSON: {exc}"] + + errs: List[str] = [] + skill_lines = catalog_skill_name_line_map(yaml_path) + reg_y, orch_y = catalog_skill_names(yaml_data) + reg_j, orch_j = catalog_skill_names(json_data) + + for group, ylist, jlist in ( + ("contents.skills", reg_y, reg_j), + ("contents.orchestration_skills", orch_y, orch_j), + ): + if ylist == jlist: + continue + max_len = max(len(ylist), len(jlist)) + for i in range(max_len): + yn = ylist[i] if i < len(ylist) else None + jn = jlist[i] if i < len(jlist) else None + if yn == jn: + continue + line = skill_lines.get(str(yn or "")) or skill_lines.get(str(jn or "")) + loc = _yaml_loc(pack_dir, yaml_path, line, root) + errs.append( + f"{pack_dir}: {loc} {group}[{i}].name out of sync: " + f"collection.yaml={yn!r}, collection.json={jn!r}" + ) + + for key in ("version", "description", "name", "id", "maturity"): + yv = yaml_data.get(key) + jv = json_data.get(key) + if yv != jv: + line = _find_top_level_key_line(yaml_path, key) + loc = _yaml_loc(pack_dir, yaml_path, line, root) + errs.append( + f"{pack_dir}: {loc} {key} out of sync: " + f"collection.yaml={yv!r}, collection.json={jv!r}" + ) + + if not errs: + errs.append( + f"{pack_dir}: .catalog/collection.json content differs from collection.yaml " + "(run diff or regenerate JSON to inspect)" + ) + errs.append( + f"{pack_dir}: regenerate mirror with: " + f"uv run python scripts/catalog_yaml_to_json.py --pack {pack_dir}" + ) + return errs + + def validate_yaml_banner(pack_dir: str, root: Optional[Path] = None) -> List[str]: root = root or REPO_ROOT p = root / pack_dir / ".catalog" / "collection.yaml" @@ -217,6 +338,94 @@ def list_disk_skill_names(pack_dir: str, root: Optional[Path] = None) -> List[st return names +def list_external_pack_skill_names( + pack_root: Path, + skill_subset: Optional[List[str]] = None, +) -> List[str]: + """Return skill directory names from an external pack checkout.""" + if skill_subset: + names: List[str] = [] + for sp in skill_subset: + p = Path(sp) + if sp.endswith("/SKILL.md"): + names.append(p.parent.name) + elif (pack_root / sp / "SKILL.md").is_file(): + names.append(Path(sp).name) + elif (pack_root / "skills" / sp / "SKILL.md").is_file(): + names.append(sp.strip("/").split("/")[-1]) + return sorted(set(names)) + + skills_dir = pack_root / "skills" + if not skills_dir.is_dir(): + return [] + return sorted( + p.name for p in skills_dir.iterdir() if p.is_dir() and (p / "SKILL.md").is_file() + ) + + +def validate_external_skill_roster( + label: str, + data: Dict[str, Any], + external_skills: Set[str], + yaml_path: Optional[Path] = None, + root: Optional[Path] = None, +) -> List[str]: + """Compare catalog YAML skill names to skills present in an external pack checkout.""" + root = root or REPO_ROOT + reg, orch = catalog_skill_names(data) + yaml_names = reg + orch + errs: List[str] = [] + skill_lines = catalog_skill_name_line_map(yaml_path) if yaml_path and yaml_path.is_file() else {} + guide_lines = ( + catalog_decision_guide_skill_line_map(yaml_path) if yaml_path and yaml_path.is_file() else {} + ) + + if len(yaml_names) != len(set(yaml_names)): + errs.append(f"{label}: duplicate skill name in contents.skills / orchestration_skills") + + seen = set(reg) | set(orch) + for n in reg + orch: + if n not in external_skills: + line = skill_lines.get(n) + loc = _yaml_loc(label, yaml_path, line, root) if yaml_path and line else label + expected = sorted(external_skills) + hint = f" (external pack skills at ref: {', '.join(expected)})" if expected else "" + errs.append( + f"{label}: {loc} lists skill {n!r} but external pack has no skills/{n}/SKILL.md at ref{hint}" + ) + + for d in external_skills: + if d not in seen: + errs.append( + f"{label}: external pack skill {d!r} missing from collection.yaml " + f"contents.skills / orchestration_skills" + ) + + contents = data.get("contents") or {} + guide = (contents or {}).get("skills_decision_guide") or [] + if not external_skills and guide: + errs.append(f"{label}: skills_decision_guide must be empty when the external pack has no skills/") + for i, row in enumerate(guide): + if not isinstance(row, dict): + continue + st = row.get("skill_to_use") + if external_skills and st and st not in external_skills: + line = guide_lines.get(str(st)) + loc = _yaml_loc(label, yaml_path, line, root) if yaml_path and line else label + errs.append( + f"{label}: {loc} skills_decision_guide[{i}] skill_to_use {st!r} " + f"not in external pack skills/ at ref" + ) + if external_skills and st and st not in seen: + line = guide_lines.get(str(st)) + loc = _yaml_loc(label, yaml_path, line, root) if yaml_path and line else label + errs.append( + f"{label}: {loc} skills_decision_guide[{i}] skill_to_use {st!r} " + f"not listed in contents.skills / orchestration_skills" + ) + return errs + + def catalog_skill_names(data: Dict[str, Any]) -> Tuple[List[str], List[str]]: contents = data.get("contents") or {} if not isinstance(contents, dict): @@ -260,10 +469,7 @@ def validate_json_mirror(pack_dir: str, data: Dict[str, Any], root: Optional[Pat expected = collection_json_dumps(data) actual = json_path.read_text(encoding="utf-8") if actual != expected: - return [ - f"{pack_dir}: .catalog/collection.json is out of sync with collection.yaml " - f"(run: uv run python scripts/catalog_yaml_to_json.py --pack {pack_dir})" - ] + return describe_json_mirror_drift(pack_dir, data, root) return [] @@ -372,7 +578,6 @@ def validate_pack_catalog_compliance_extra( errs.extend(validate_embedded_docs(pack_dir, data, root)) errs.extend(validate_catalog_inline_length(pack_dir, data)) - errs.extend(validate_json_mirror(pack_dir, data, root)) return errs diff --git a/scripts/fetch_federated_skills.py b/scripts/fetch_federated_skills.py index da630dd4..eb7788fa 100755 --- a/scripts/fetch_federated_skills.py +++ b/scripts/fetch_federated_skills.py @@ -17,7 +17,7 @@ import argparse import json -import os +import re import shutil import subprocess import sys @@ -49,23 +49,55 @@ class ModuleResult: clone_path: str = "" -def clone_at_ref(repository: str, ref: Optional[str], dest: Path) -> Optional[str]: - """Clone a repository and optionally checkout a pinned ref. Returns error string or None.""" +def _strip_ansi(text: str) -> str: + return re.sub(r"\x1b\[[0-9;]*m", "", text) + + +def format_linter_lines(output: str, marker: str) -> List[str]: + """Extract linter lines containing marker (e.g. [FAIL], [WARN]).""" + cleaned = _strip_ansi(output) + return [line.strip() for line in cleaned.splitlines() if marker in line] + + +def print_skill_validation_details(sr: SkillResult) -> None: + """Print Tier 1 linter details to the console (human-readable mode).""" + if sr.passed: + if sr.warnings: + for line in format_linter_lines(sr.output, "[WARN]"): + print(f" {line}") + return + + fail_lines = format_linter_lines(sr.output, "[FAIL]") + if fail_lines: + for line in fail_lines: + print(f" {line}") + return + + if sr.output.strip(): + for line in _strip_ansi(sr.output).splitlines(): + stripped = line.strip() + if stripped: + print(f" {stripped}") + return + + print(" (no linter output captured)") + + +def clone_at_ref(repository: str, ref: str, dest: Path) -> Optional[str]: + """Clone a repository and checkout a pinned commit SHA. Returns error string or None.""" + err = pack_registry.federation_ref_error(ref) + if err: + return err + sha = pack_registry.normalize_federation_ref(ref) try: - if ref: - subprocess.run( - ["git", "clone", "--quiet", "--no-checkout", repository, str(dest)], - check=True, capture_output=True, text=True, timeout=120, - ) - subprocess.run( - ["git", "checkout", "--quiet", ref], - check=True, capture_output=True, text=True, cwd=dest, timeout=30, - ) - else: - subprocess.run( - ["git", "clone", "--quiet", "--depth", "1", repository, str(dest)], - check=True, capture_output=True, text=True, timeout=120, - ) + subprocess.run( + ["git", "clone", "--quiet", "--no-checkout", repository, str(dest)], + check=True, capture_output=True, text=True, timeout=120, + ) + subprocess.run( + ["git", "checkout", "--quiet", sha], + check=True, capture_output=True, text=True, cwd=dest, timeout=30, + ) return None except subprocess.CalledProcessError as exc: return exc.stderr.strip() or str(exc) @@ -93,11 +125,14 @@ def validate_skill(skill_dir: Path, repo_root: Path) -> SkillResult: timeout=30, ) has_warnings = "[WARN]" in result.stdout + combined = result.stdout + if result.stderr.strip(): + combined = f"{combined}\n{result.stderr}".strip() return SkillResult( path=rel_path, passed=result.returncode == 0, warnings=has_warnings, - output=result.stdout.strip(), + output=combined.strip(), ) except subprocess.TimeoutExpired: return SkillResult(path=rel_path, passed=False, output="Linter timed out") @@ -121,6 +156,11 @@ def process_module( result.error = "Missing repository" return result + ref_err = pack_registry.federation_ref_error(ref) + if ref_err: + result.error = ref_err + return result + clone_dest = base_dir / name err = clone_at_ref(repository, ref, clone_dest) if err: @@ -190,7 +230,7 @@ def main() -> int: print(f"\n{'='*60}") print(f"Module: {mod.get('name', '?')}") print(f" Repository: {mod.get('repository', '?')}") - print(f" Ref: {mod.get('ref', '?')}") + print(f" Ref: {mod.get('ref') or '(missing — required 40-character commit SHA)'}") print(f" Pack path: {mod.get('path', '.')}") print(f"{'='*60}") @@ -213,6 +253,8 @@ def main() -> int: status = "PASS" if sr.passed else "FAIL" warn = " (warnings)" if sr.warnings else "" print(f" [{status}{warn}] {sr.path}") + if not sr.passed or sr.warnings: + print_skill_validation_details(sr) if not sr.passed: any_failure = True @@ -235,6 +277,8 @@ def main() -> int: passed = sum(1 for r in results for s in r.skills if s.passed) failed = sum(1 for r in results for s in r.skills if not s.passed) print(f"Summary: {passed}/{total_skills} skills passed, {failed} failed") + if failed: + print("See [FAIL] lines above for Tier 1 linter details (or re-run with --json).") return 1 if any_failure else 0 diff --git a/scripts/generate_pack_data.py b/scripts/generate_pack_data.py index e5724ac1..73a57bda 100644 --- a/scripts/generate_pack_data.py +++ b/scripts/generate_pack_data.py @@ -288,7 +288,34 @@ def parse_docs(pack_dir: str) -> List[Dict[str, Any]]: return sorted(docs, key=lambda d: (d['category'], d['title'])) -def load_federated_packs() -> List[Dict[str, Any]]: +def detect_repo_license(repo_root: Path, pack_path: str = ".") -> str: + """Best-effort SPDX identifier from LICENSE files in a cloned repository.""" + candidates = [ + repo_root / pack_path / "LICENSE", + repo_root / pack_path / "LICENSE.txt", + repo_root / "LICENSE", + repo_root / "LICENSE.txt", + ] + for path in candidates: + if not path.is_file(): + continue + try: + text = path.read_text(encoding="utf-8", errors="replace")[:8000] + except OSError: + continue + upper = text.upper() + if "APACHE LICENSE" in upper and "VERSION 2.0" in upper: + return "Apache-2.0" + if "MIT LICENSE" in upper or "PERMISSION IS HEREBY GRANTED, FREE OF CHARGE" in upper: + return "MIT" + if "BSD 3-CLAUSE" in upper or "REDISTRIBUTION AND USE IN SOURCE AND BINARY FORMS" in upper: + if "3-CLAUSE" in upper or "3 CLAUSE" in upper: + return "BSD-3-Clause" + return "BSD-2-Clause" + return "Unknown" + + +def load_federated_packs(plugin_titles: Dict[str, str] | None = None) -> List[Dict[str, Any]]: """ Fetch federated modules and return them as standalone pack entries. @@ -299,21 +326,32 @@ def load_federated_packs() -> List[Dict[str, Any]]: import subprocess import tempfile + titles = plugin_titles if plugin_titles is not None else load_plugin_titles() modules = pack_registry.load_federated_modules() if not modules: return [] + repo_root = Path(__file__).resolve().parent.parent packs: List[Dict[str, Any]] = [] tmp = Path(tempfile.mkdtemp(prefix="federated-build-")) try: for mod in modules: name = mod.get("name", "unknown") + fed_catalog_dir = f"federation/modules/{name}" + maturity = pack_registry.load_pack_maturity(fed_catalog_dir, repo_root) + if maturity != pack_registry.DOCS_MATURITY_PUBLISH: + label = maturity or "missing catalog" + print( + f" Skipping federated '{name}' for docs site: maturity {label} " + f"(only GREEN is published)" + ) + continue + repository = mod.get("repository", "") ref = mod.get("ref", "") description = mod.get("description", "") version = mod.get("version", "0.0.0") - license_id = mod.get("license", "Unknown") tags = mod.get("tags", []) pack_path = mod.get("path", ".") skill_subset = mod.get("skills") @@ -322,27 +360,27 @@ def load_federated_packs() -> List[Dict[str, Any]]: print(f" Warning: federated module '{name}' missing repository, skipping") continue + ref_err = pack_registry.federation_ref_error(ref) + if ref_err: + print(f" Warning: federated module '{name}' invalid ref: {ref_err}") + continue + clone_dest = tmp / name try: - if ref: - subprocess.run( - ["git", "clone", "--quiet", "--no-checkout", repository, str(clone_dest)], - check=True, capture_output=True, text=True, timeout=120, - ) - subprocess.run( - ["git", "checkout", "--quiet", ref], - check=True, capture_output=True, text=True, cwd=clone_dest, timeout=30, - ) - else: - subprocess.run( - ["git", "clone", "--quiet", "--depth", "1", repository, str(clone_dest)], - check=True, capture_output=True, text=True, timeout=120, - ) + subprocess.run( + ["git", "clone", "--quiet", "--no-checkout", repository, str(clone_dest)], + check=True, capture_output=True, text=True, timeout=120, + ) + subprocess.run( + ["git", "checkout", "--quiet", pack_registry.normalize_federation_ref(ref)], + check=True, capture_output=True, text=True, cwd=clone_dest, timeout=30, + ) except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc: print(f" Warning: failed to clone '{name}': {exc}") continue pack_dir = clone_dest / pack_path + license_id = detect_repo_license(clone_dest, pack_path) skills = [] if skill_subset: @@ -361,12 +399,13 @@ def load_federated_packs() -> List[Dict[str, Any]]: pack = { "name": name, "path": repository, + "catalog_dir": fed_catalog_dir, "source": "federated", "repository": repository, - "ref": ref[:12], + "ref": pack_registry.normalize_federation_ref(ref)[:12], "plugin": { "name": name, - "title": name, + "title": titles.get(name, name.replace("-", " ").title()), "version": version, "description": description, "author": {"name": "External"}, @@ -426,7 +465,7 @@ def generate_pack_data() -> List[Dict[str, Any]]: plugin_title = pack['plugin'].get('title', pack_dir) print(f"✓ Parsed {plugin_title}: {len(pack['skills'])} skills, {len(pack['agents'])} agents, {len(docs)} docs") - federated = load_federated_packs() + federated = load_federated_packs(plugin_titles) if federated: packs.extend(federated) print(f"✓ Added {len(federated)} federated pack(s)") diff --git a/scripts/pack_registry.py b/scripts/pack_registry.py index ceca9533..2157c8cd 100644 --- a/scripts/pack_registry.py +++ b/scripts/pack_registry.py @@ -8,6 +8,7 @@ from __future__ import annotations import json +import re from pathlib import Path from typing import Any, Dict, List, Optional, Set @@ -87,6 +88,37 @@ def load_marketplace_module_by_path( MAIN_REPO_URL = "https://github.com/RHEcosystemAppEng/agentic-collections" +FEDERATION_REF_SHA_RE = re.compile(r"^[0-9a-f]{40}$", re.IGNORECASE) + + +def federation_ref_error(ref: Any) -> Optional[str]: + """Return an error message when *ref* is missing or not a 40-character commit SHA.""" + if ref is None or not str(ref).strip(): + return "ref is required (40-character commit SHA; not a branch or tag name)" + value = str(ref).strip() + if not FEDERATION_REF_SHA_RE.fullmatch(value): + return ( + f"ref must be a 40-character commit SHA, not a branch or tag (got {value!r})" + ) + return None + + +def normalize_federation_ref(ref: Any) -> str: + """Return a lowercase 40-character commit SHA.""" + err = federation_ref_error(ref) + if err: + raise ValueError(err) + return str(ref).strip().lower() + + +def validate_federated_module_entry(module: Dict[str, Any]) -> List[str]: + """Return validation errors for a federated marketplace module entry.""" + name = module.get("name") or "<unknown>" + err = federation_ref_error(module.get("ref")) + if err: + return [f"{name}: {err}"] + return [] + def load_federated_modules( marketplace_path: Optional[Path] = None, @@ -171,3 +203,13 @@ def get_docs_pack_dirs( if load_pack_maturity(p, root) == DOCS_MATURITY_PUBLISH: out.append(p) return out + + +def get_docs_federation_module_dirs(repo_root: Optional[Path] = None) -> List[str]: + """Federation module dirs listed on GitHub Pages (catalog maturity GREEN only).""" + root = repo_root or _repo_root() + return [ + p + for p in get_federation_module_dirs(root) + if load_pack_maturity(p, root) == DOCS_MATURITY_PUBLISH + ] diff --git a/scripts/validate_federation.py b/scripts/validate_federation.py index c90d0f0b..72f8f680 100755 --- a/scripts/validate_federation.py +++ b/scripts/validate_federation.py @@ -2,30 +2,28 @@ """ Validate an external repository for federation into Red Hat Agentic Collections. -Automates the mechanical checks from docs/FEDERATION_REVIEW_GUIDE.md: - 1. Clone repository (at pinned ref if provided, default branch otherwise) - 2. Verify Lola pack structure - 3. Tier 1 validation (agentskills.io spec) - 4. Tier 2 validation (design principles) - 5. MCP version pinning (no :latest) - 6. Credential leak scan (gitleaks) +Automates the mechanical checks from FEDERATION_REVIEW_GUIDE.md: + 1. Validate pinned commit SHA (ref) + 2. Clone repository at that commit + 3. Verify Lola pack structure + 4. Tier 1 validation (agentskills.io spec) + 5. Tier 2 validation (design principles) + 6. MCP version pinning (no :latest) + 7. Credential leak scan (gitleaks) Usage: - python scripts/validate_federation.py <repo-url> [--ref <ref>] [--pack-path <path>] [--skills skill1 skill2] - python scripts/validate_federation.py <repo-url> --json + python scripts/validate_federation.py <repo-url> --ref <commit-sha> [--pack-path <path>] [--skills skill1 skill2] + python scripts/validate_federation.py <repo-url> --ref <commit-sha> --json Examples: - # Validate entire pack (default branch) - python scripts/validate_federation.py https://github.com/org/repo - - # Validate at a specific ref - python scripts/validate_federation.py https://github.com/org/repo --ref v1.0.0 + # Validate entire pack at a pinned commit + python scripts/validate_federation.py https://github.com/org/repo --ref a1b2c3d4e5f6789012345678901234567890abcd # Pack lives in a subdirectory - python scripts/validate_federation.py https://github.com/org/repo --pack-path my-pack + python scripts/validate_federation.py https://github.com/org/repo --ref a1b2c3... --pack-path my-pack # Validate only specific skills - python scripts/validate_federation.py https://github.com/org/repo --skills sdn-diagnostics ovn-trace + python scripts/validate_federation.py https://github.com/org/repo --ref a1b2c3... --skills sdn-diagnostics ovn-trace """ from __future__ import annotations @@ -39,6 +37,8 @@ from dataclasses import dataclass, field, asdict from pathlib import Path +import pack_registry + LOLA_REQUIRED_FIELDS = ["name", "description", "version", "repository"] REPO_ROOT = Path(__file__).resolve().parent.parent @@ -69,25 +69,32 @@ def all_passed(self) -> bool: ) -def clone_at_ref(repo_url: str, ref: str | None, dest: Path) -> CheckResult: +def check_federation_ref(ref: str | None) -> CheckResult: + check = CheckResult(name="federation_ref") + err = pack_registry.federation_ref_error(ref) + if err: + check.passed = False + check.details.append(err) + return check + normalized = pack_registry.normalize_federation_ref(ref) + check.passed = True + check.details.append(f"Pinned commit SHA: {normalized}") + return check + + +def clone_at_ref(repo_url: str, ref: str, dest: Path) -> CheckResult: check = CheckResult(name="clone") + sha = pack_registry.normalize_federation_ref(ref) try: - if ref: - subprocess.run( - ["git", "clone", "--quiet", "--no-checkout", repo_url, str(dest)], - check=True, capture_output=True, text=True, timeout=120, - ) - subprocess.run( - ["git", "checkout", "--quiet", ref], - check=True, capture_output=True, text=True, cwd=dest, timeout=30, - ) - check.details.append(f"Cloned and checked out {ref}") - else: - subprocess.run( - ["git", "clone", "--quiet", "--depth", "1", repo_url, str(dest)], - check=True, capture_output=True, text=True, timeout=120, - ) - check.details.append("Cloned default branch") + subprocess.run( + ["git", "clone", "--quiet", "--no-checkout", repo_url, str(dest)], + check=True, capture_output=True, text=True, timeout=120, + ) + subprocess.run( + ["git", "checkout", "--quiet", sha], + check=True, capture_output=True, text=True, cwd=dest, timeout=30, + ) + check.details.append(f"Cloned and checked out {sha}") check.passed = True except subprocess.CalledProcessError as exc: check.passed = False @@ -95,10 +102,13 @@ def clone_at_ref(repo_url: str, ref: str | None, dest: Path) -> CheckResult: except subprocess.TimeoutExpired: check.passed = False check.details.append("Git operation timed out") + except ValueError as exc: + check.passed = False + check.details.append(str(exc)) return check -def check_lola_module_schema(module_meta: dict | None) -> CheckResult: +def check_lola_module_schema(module_meta: dict | None, ref: str | None = None) -> CheckResult: check = CheckResult(name="lola_structure") if module_meta is None: check.passed = True @@ -107,12 +117,18 @@ def check_lola_module_schema(module_meta: dict | None) -> CheckResult: return check missing = [f for f in LOLA_REQUIRED_FIELDS if not module_meta.get(f, "")] + ref_value = ref if ref is not None else module_meta.get("ref") + ref_err = pack_registry.federation_ref_error(ref_value) if missing: - check.passed = False check.details.append(f"Missing required Lola fields: {', '.join(missing)}") - else: + if ref_err: + check.details.append(ref_err) + if not missing and not ref_err: check.passed = True - check.details.append(f"Module schema valid: {', '.join(LOLA_REQUIRED_FIELDS)} present") + check.details.append( + f"Module schema valid: {', '.join(LOLA_REQUIRED_FIELDS)} present; " + f"ref pinned to {pack_registry.normalize_federation_ref(ref_value)}" + ) return check @@ -272,7 +288,7 @@ def print_report(report: ValidationReport) -> None: print("=" * 60) print("Federation Validation Report") print(f" Repository: {report.repository}") - print(f" Ref: {report.ref or 'default branch'}") + print(f" Ref: {report.ref or '(missing)'}") print("=" * 60) for c in report.checks: @@ -308,7 +324,11 @@ def main() -> int: description="Validate an external repo for federation" ) parser.add_argument("repo_url", help="Public repository URL") - parser.add_argument("--ref", default=None, help="Commit SHA or release tag (default: default branch)") + parser.add_argument( + "--ref", + required=True, + help="Required 40-character commit SHA (not a branch or tag name)", + ) parser.add_argument("--pack-path", default=".", help="Path to the pack within the repo (default: repo root)") parser.add_argument("--skills", nargs="*", help="Validate only these skills (by directory name)") parser.add_argument("--json", action="store_true", help="Output as JSON") @@ -320,6 +340,15 @@ def main() -> int: tmp = Path(tempfile.mkdtemp(prefix="federation-review-")) try: + ref_check = check_federation_ref(args.ref) + report.checks.append(ref_check) + if not ref_check.passed: + if args.json: + print(json.dumps(asdict(report), indent=2)) + else: + print_report(report) + return 1 + # Step 1: Clone clone_result = clone_at_ref(args.repo_url, args.ref, tmp / "repo") report.checks.append(clone_result) @@ -332,9 +361,9 @@ def main() -> int: pack_dir = tmp / "repo" / args.pack_path - # Step 2: Lola module schema + # Step 2: Lola module schema (+ federation ref when module metadata provided) module_meta = json.loads(args.module_json) if args.module_json else None - report.checks.append(check_lola_module_schema(module_meta)) + report.checks.append(check_lola_module_schema(module_meta, ref=args.ref)) # Step 3: Tier 1 report.checks.append(run_tier1(pack_dir, args.skills)) diff --git a/scripts/validate_federation_catalog.py b/scripts/validate_federation_catalog.py new file mode 100644 index 00000000..40f486f6 --- /dev/null +++ b/scripts/validate_federation_catalog.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +""" +Cross-check federation catalog artifacts against the linked external pack at a pinned ref. + +Validates that federation/modules/<name>/.catalog/ in agentic-collections matches the +skills and marketplace metadata of the external repository declared in +marketplace/rh-agentic-collection.yml. + +Usage: + uv run python scripts/validate_federation_catalog.py \\ + --module-name claude-code-setup \\ + --repo-url https://github.com/org/repo \\ + --ref <40-char-sha> \\ + --pack-path plugins/claude-code-setup \\ + [--module-json '<marketplace-module-json>'] \\ + [--json] +""" + +from __future__ import annotations + +import argparse +import json +import re +import shutil +import sys +import tempfile +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Set + +import collection_validate_lib as cvl +import pack_registry + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +@dataclass +class CheckResult: + name: str + passed: bool = False + skipped: bool = False + details: list[str] = field(default_factory=list) + + +@dataclass +class CatalogReport: + module_name: str + repository: str + ref: str + checks: list[CheckResult] = field(default_factory=list) + + @property + def all_passed(self) -> bool: + return all(c.passed for c in self.checks) + + +def _norm_text(value: Any) -> str: + return re.sub(r"\s+", " ", str(value or "").strip().lower()) + + +def _token_set(text: str) -> Set[str]: + return {t for t in re.findall(r"[a-z0-9]+", _norm_text(text)) if len(t) > 2} + + +def _descriptions_align(a: str, b: str) -> bool: + na, nb = _norm_text(a), _norm_text(b) + if not na or not nb: + return True + if na in nb or nb in na: + return True + ta, tb = _token_set(a), _token_set(b) + if not ta or not tb: + return True + overlap = len(ta & tb) / min(len(ta), len(tb)) + return overlap >= 0.5 + + +def federation_catalog_dir(module_name: str, repo_root: Optional[Path] = None) -> str: + return f"federation/modules/{module_name}" + + +def check_catalog_present(module_name: str, repo_root: Optional[Path] = None) -> CheckResult: + check = CheckResult(name="catalog_present") + root = repo_root or REPO_ROOT + cat_dir = federation_catalog_dir(module_name) + cat_yaml = root / cat_dir / ".catalog" / "collection.yaml" + if not cat_yaml.is_file(): + check.details.append(f"Missing {cat_yaml.relative_to(root)}") + return check + check.passed = True + check.details.append(f"Found {cat_yaml.relative_to(root)}") + return check + + +def check_catalog_compliance(module_name: str, repo_root: Optional[Path] = None) -> CheckResult: + check = CheckResult(name="catalog_compliance") + cat_dir = federation_catalog_dir(module_name) + errs = cvl.validate_pack_iteration5(cat_dir, repo_root, is_federated=True) + if errs: + check.details.extend(errs[:20]) + if len(errs) > 20: + check.details.append(f"... and {len(errs) - 20} more") + return check + check.passed = True + check.details.append("collection.yaml passes schema, fragments, and JSON mirror checks") + return check + + +def check_external_skill_roster( + module_name: str, + external_pack_root: Path, + skill_subset: Optional[List[str]] = None, + repo_root: Optional[Path] = None, +) -> CheckResult: + check = CheckResult(name="external_skill_roster") + root = repo_root or REPO_ROOT + cat_dir = federation_catalog_dir(module_name) + data, errs = cvl.read_yaml_catalog(cat_dir, root) + if errs or data is None: + check.details.extend(errs) + return check + + external = set(cvl.list_external_pack_skill_names(external_pack_root, skill_subset)) + yaml_path = root / cat_dir / ".catalog" / "collection.yaml" + roster_errs = cvl.validate_external_skill_roster( + cat_dir, data, external, yaml_path=yaml_path, root=root, + ) + if roster_errs: + check.details.extend(roster_errs) + return check + + check.passed = True + check.details.append( + f"Catalog roster matches external pack ({len(external)} skill(s) at ref)" + ) + return check + + +def check_plugins_json_title(module_name: str, repo_root: Optional[Path] = None) -> CheckResult: + check = CheckResult(name="plugins_json_title") + root = repo_root or REPO_ROOT + plugins_path = root / "docs" / "plugins.json" + cat_dir = federation_catalog_dir(module_name) + + if not plugins_path.is_file(): + check.details.append("Missing docs/plugins.json") + return check + + data, errs = cvl.read_yaml_catalog(cat_dir, root) + if errs or data is None: + check.details.extend(errs) + return check + + try: + plugins = json.loads(plugins_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + check.details.append(f"Invalid docs/plugins.json: {exc}") + return check + + entry = plugins.get(module_name) + title = entry.get("title") if isinstance(entry, dict) else None + if not str(title or "").strip(): + check.details.append(f"docs/plugins.json missing title for {module_name!r}") + return check + + if data.get("id") != module_name: + check.details.append( + f"collection id {data.get('id')!r} must equal module name {module_name!r}" + ) + return check + + if data.get("name") != title: + check.details.append( + f"collection name {data.get('name')!r} must equal plugins.json title {title!r}" + ) + return check + + check.passed = True + check.details.append(f"plugins.json title matches catalog name ({title!r})") + return check + + +def check_marketplace_catalog_metadata( + module_name: str, + module_meta: Dict[str, Any], + repo_root: Optional[Path] = None, +) -> CheckResult: + check = CheckResult(name="marketplace_catalog_metadata") + root = repo_root or REPO_ROOT + cat_dir = federation_catalog_dir(module_name) + data, errs = cvl.read_yaml_catalog(cat_dir, root) + if errs or data is None: + check.details.extend(errs) + return check + + mp_version = str(module_meta.get("version") or "").strip() + cat_version = str(data.get("version") or "").strip() + if mp_version and cat_version and mp_version != cat_version: + check.details.append( + f"marketplace version {mp_version!r} != catalog version {cat_version!r}" + ) + return check + + mp_desc = _norm_text(module_meta.get("description")) + cat_desc = _norm_text(data.get("description")) + if mp_desc and cat_desc and not _descriptions_align( + str(module_meta.get("description") or ""), + str(data.get("description") or ""), + ): + check.details.append( + "marketplace description and catalog description appear unrelated " + f"(marketplace={module_meta.get('description')!r}, catalog={data.get('description')!r})" + ) + return check + + mp_repo = str(module_meta.get("repository") or "").rstrip("/") + cat_repo = str(data.get("repository") or "").rstrip("/") + if mp_repo and cat_repo and mp_repo != cat_repo: + check.details.append( + f"marketplace repository {mp_repo!r} != catalog repository {cat_repo!r}" + ) + return check + + check.passed = True + check.details.append("Marketplace version, description, and repository align with catalog") + return check + + +def run_catalog_validation( + module_name: str, + repo_url: str, + ref: str, + pack_path: str, + module_meta: Optional[Dict[str, Any]] = None, + repo_root: Optional[Path] = None, + external_pack_root: Optional[Path] = None, + cleanup_clone: bool = True, +) -> CatalogReport: + from validate_federation import check_federation_ref, clone_at_ref + + root = repo_root or REPO_ROOT + report = CatalogReport(module_name=module_name, repository=repo_url, ref=ref) + + present = check_catalog_present(module_name, root) + report.checks.append(present) + if not present.passed: + return report + + ref_check = check_federation_ref(ref) + report.checks.append(ref_check) + if not ref_check.passed: + return report + + tmp: Optional[Path] = None + pack_root = external_pack_root + if pack_root is None: + tmp = Path(tempfile.mkdtemp(prefix="federation-catalog-")) + clone = clone_at_ref(repo_url, ref, tmp / "repo") + report.checks.append(clone) + if not clone.passed: + if cleanup_clone and tmp: + shutil.rmtree(tmp, ignore_errors=True) + return report + pack_root = tmp / "repo" / pack_path + + if not (pack_root / "skills").is_dir(): + fail = CheckResult(name="external_pack_path") + fail.details.append(f"No skills/ directory at external pack path {pack_path!r}") + report.checks.append(fail) + if cleanup_clone and tmp: + shutil.rmtree(tmp, ignore_errors=True) + return report + + skill_subset = None + if module_meta and module_meta.get("skills"): + skill_subset = module_meta.get("skills") + + report.checks.append(check_catalog_compliance(module_name, root)) + report.checks.append( + check_external_skill_roster(module_name, pack_root, skill_subset, root) + ) + report.checks.append(check_plugins_json_title(module_name, root)) + if module_meta: + report.checks.append(check_marketplace_catalog_metadata(module_name, module_meta, root)) + + if cleanup_clone and tmp: + shutil.rmtree(tmp, ignore_errors=True) + + return report + + +def print_report(report: CatalogReport) -> None: + print() + print("=" * 60) + print("Federation Catalog Cross-Check") + print(f" Module: {report.module_name}") + print(f" Repository: {report.repository}") + print(f" Ref: {report.ref or '(missing)'}") + print("=" * 60) + + for c in report.checks: + if c.skipped: + status, icon = "SKIP", "⚠️" + elif c.passed: + status, icon = "PASS", "✅" + else: + status, icon = "FAIL", "❌" + print(f"\n{icon} [{status}] {c.name}") + for d in c.details: + print(f" {d}") + + print() + print("=" * 60) + if report.all_passed: + print("✅ CATALOG CROSS-CHECK PASSED") + else: + failed = [c.name for c in report.checks if not c.passed] + print(f"❌ FAILED CHECKS: {', '.join(failed)}") + print("=" * 60) + print() + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Cross-check federation catalog against external pack at pinned ref" + ) + parser.add_argument("--module-name", required=True, help="Federated module name (marketplace name)") + parser.add_argument("--repo-url", required=True, help="External repository URL") + parser.add_argument("--ref", required=True, help="40-character commit SHA") + parser.add_argument("--pack-path", default=".", help="Path to pack within external repo") + parser.add_argument("--module-json", default=None, help="Marketplace module entry as JSON") + parser.add_argument("--json", action="store_true", help="Output report as JSON") + args = parser.parse_args() + + module_meta = json.loads(args.module_json) if args.module_json else None + report = run_catalog_validation( + module_name=args.module_name, + repo_url=args.repo_url, + ref=args.ref, + pack_path=args.pack_path, + module_meta=module_meta, + ) + + if args.json: + print(json.dumps(asdict(report), indent=2)) + else: + print_report(report) + + return 0 if report.all_passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/validate_federation_catalog_all.py b/scripts/validate_federation_catalog_all.py new file mode 100644 index 00000000..c4017017 --- /dev/null +++ b/scripts/validate_federation_catalog_all.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Run validate_federation_catalog.py for every federated module with a local catalog.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pack_registry +from validate_federation_catalog import print_report, run_catalog_validation + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def main() -> int: + modules = pack_registry.load_federated_modules() + if not modules: + print("No federated modules configured in marketplace YAML.") + return 0 + + failed = False + for mod in modules: + name = mod.get("name", "") + repository = mod.get("repository", "") + ref = mod.get("ref", "") + pack_path = mod.get("path", ".") + cat_dir = REPO_ROOT / "federation" / "modules" / name / ".catalog" / "collection.yaml" + if not cat_dir.is_file(): + print(f"Skipping {name}: no federation catalog at {cat_dir.relative_to(REPO_ROOT)}") + continue + + ref_err = pack_registry.federation_ref_error(ref) + if ref_err: + print(f"FAIL {name}: {ref_err}") + failed = True + continue + + print(f"\n--- Catalog cross-check: {name} ---") + report = run_catalog_validation( + module_name=name, + repo_url=repository, + ref=ref, + pack_path=pack_path, + module_meta=mod, + repo_root=REPO_ROOT, + ) + print_report(report) + if not report.all_passed: + failed = True + + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main())