diff --git a/AGENTS.md b/AGENTS.md index 42916ba..66de982 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,6 +40,8 @@ agentic-collections-catalog/ The clones are ephemeral (deleted after each build). Nothing from external repos is committed here. +> **Eval data lives in source repos, not here.** `eval_site_enrichment.py` reads `eval///report.json` from the *cloned* source repo (e.g. `agentic-collections-skills/eval/rh-sre/remediation/report.json`). There is no `eval/` directory in this catalog repo — looking for eval files here will always find nothing. + ## Marketplace File `marketplace/rh-agentic-collection.yml` is the **only** place that controls which packs appear on the site. Each module entry carries: diff --git a/README.md b/README.md index c11cbff..a76fbc6 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ Unified marketplace and website for Red Hat agentic skill collections. This repo **This is not a skills development repository.** Skills are authored in source repositories like [agentic-collections-skills](https://github.com/RHEcosystemAppEng/agentic-collections-skills). An internal process fetches, evaluates, and assembles the catalog automatically. +> **Note:** Evaluation reports (`eval///report.json`) live in the **source skills repos**, not here. This catalog repo contains no `eval/` directory — eval data is read from the temporary clones at build time. + [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) [![Validate Catalog](https://github.com/RHEcosystemAppEng/agentic-collections-catalog/actions/workflows/validate.yml/badge.svg)](https://github.com/RHEcosystemAppEng/agentic-collections-catalog/actions/workflows/validate.yml) [![Deploy GitHub Pages](https://github.com/RHEcosystemAppEng/agentic-collections-catalog/actions/workflows/deploy-pages.yml/badge.svg)](https://github.com/RHEcosystemAppEng/agentic-collections-catalog/actions/workflows/deploy-pages.yml) diff --git a/scripts/build_website.py b/scripts/build_website.py index a41d858..dec3762 100644 --- a/scripts/build_website.py +++ b/scripts/build_website.py @@ -11,7 +11,6 @@ # Import our data generators from catalog_site_bundle import bundle_catalog_for_site -from eval_site_enrichment import apply_eval_enrichment from generate_collection_pages import generate_collection_pages from generate_pack_data import generate_pack_data from generate_mcp_data import generate_mcp_data @@ -44,10 +43,6 @@ def build_website(): # Keep pack cards deterministic and alphabetically ordered. pack_data = sorted(pack_data, key=lambda p: p['name']) - print("📊 Enriching packs with eval/reports (eval///report.json)...") - apply_eval_enrichment(pack_data, root) - print() - # Generate MCP server data print("🔌 Parsing MCP servers...") mcp_data = generate_mcp_data(pack_data) diff --git a/scripts/eval_site_enrichment.py b/scripts/eval_site_enrichment.py index 580ba39..87455bf 100644 --- a/scripts/eval_site_enrichment.py +++ b/scripts/eval_site_enrichment.py @@ -299,14 +299,21 @@ def apply_eval_enrichment(packs: List[Dict[str, Any]], root: Path) -> None: """ Mutate each pack: attach skill['evaluation'] for catalog-listed skills when eval///report.json exists; set pack['evaluation_summary'] rollup. + + Eval reports live in the SOURCE repo (read from the clone at build time), not in + the catalog repo. The root argument must point to the cloned source repo root. + + Falls back to pack['skills'] (from SKILL.md frontmatter) when no catalog is present. """ root = root.resolve() for pack in packs: coll = pack.get("collection") - if not isinstance(coll, dict): - continue + if isinstance(coll, dict): + skills_ref = _iter_catalog_skills(coll) + else: + # No catalog: use raw skills parsed from SKILL.md frontmatter + skills_ref = [s for s in (pack.get("skills") or []) if isinstance(s, dict)] - skills_ref = _iter_catalog_skills(coll) catalog_skill_count = len(skills_ref) attached: List[Optional[Dict[str, Any]]] = []