Make catalog validation real and guard against link rot

.github/workflows/catalog-health.yaml

@@ -0,0 +1,53 @@
+name: Catalog health
+
+# The catalog points at release assets that live in other repositories, which
+# can be deleted, retagged, or re-cut after a plugin lands here. PR validation
+# only catches breakage at merge time, so this re-verifies every plugin's
+# download links and checksums on a schedule and opens an issue if the catalog
+# has rotted — before a user hits it during `datumctl plugin install`.
+on:
+  schedule:
+    - cron: "0 12 * * 1" # Mondays, 12:00 UTC
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  verify-assets:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      - name: Install tooling
+        run: pip install --quiet "jsonschema[format-nongpl]>=4" "PyYAML>=6"
+
+      - name: Verify every plugin's download links and checksums
+        run: python3 scripts/verify_manifests.py
+
+      - name: Open an issue if the catalog is unhealthy
+        if: failure()
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          gh label create catalog-health \
+            --color b60205 \
+            --description "A plugin download link or checksum is broken" 2>/dev/null || true
+
+          body=$(printf 'The scheduled catalog health check failed: one or more plugin download URLs no longer resolve, or their checksums no longer match the published release assets.\n\nUsers may be unable to install or verify the affected plugin(s) with datumctl until this is fixed. See the run log for the specific plugin and platform:\n\n%s\n' "$RUN_URL")
+
+          number=$(gh issue list --state open --label catalog-health --json number --jq '.[0].number // empty')
+          if [ -n "$number" ]; then
+            gh issue comment "$number" --body "$body"
+          else
+            gh issue create \
+              --title "Catalog health check is failing" \
+              --label catalog-health \
+              --body "$body"
+          fi

.github/workflows/validate.yaml

@@ -0,0 +1,38 @@
+name: Validate catalog
+
+# Runs on every PR and on pushes to main. The catalog is small, so we validate
+# and verify the whole thing each time rather than trying to diff which plugins
+# changed — that keeps the check honest (a manifest can break for reasons
+# unrelated to the files a PR touched) and always exercises the real assets.
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      - name: Install tooling
+        run: pip install --quiet "jsonschema[format-nongpl]>=4" "PyYAML>=6"
+
+      # index.yaml is what datumctl reads, and it is fully derived from
+      # plugins/*.yaml. Fail if the committed index doesn't match what the
+      # generator produces, so main's index can never drift from the manifests.
+      - name: Check index.yaml is in sync with plugins/
+        run: python3 scripts/generate_index.py --check
+
+      # Schema-validate every manifest and prove every advertised download
+      # actually resolves and matches its checksum.
+      - name: Validate manifests and verify release assets
+        run: python3 scripts/verify_manifests.py

README.md

@@ -39,8 +39,14 @@ datumctl ipam pool list
 
 1. Add the `datumctl-plugin` topic to your plugin's GitHub repository.
 2. Open a pull request adding `plugins/<your-plugin-name>.yaml`, following the [schema](schema/plugin-v1alpha1.json).
-3. CI validates your manifest: schema conformance, that every download URL resolves, and that each SHA256 matches the published archive.
-4. Once merged, [`index.yaml`](index.yaml) — the file `datumctl` actually reads — is regenerated automatically.
+3. Regenerate the index and commit it alongside your manifest:
+
+   ```sh
+   python3 scripts/generate_index.py
+   ```
+
+   [`index.yaml`](index.yaml) — the file `datumctl` actually reads — is derived entirely from `plugins/*.yaml`, and CI fails if the two drift.
+4. CI validates every manifest: schema conformance, that each download URL resolves, and that each SHA256 matches the published archive. A weekly [health check](.github/workflows/catalog-health.yaml) re-verifies the whole catalog so a plugin's links can't rot unnoticed.
 
 ## Plugin manifest format
 

scripts/generate_index.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+"""Generate index.yaml from the per-plugin manifests in plugins/.
+
+index.yaml is the single file datumctl reads to discover plugins, their
+versions, and per-platform download locations. It is fully derived from
+plugins/*.yaml, so it is regenerated deterministically here and checked for
+drift in CI (`--check`) rather than hand-edited.
+
+Usage:
+  scripts/generate_index.py           # write index.yaml
+  scripts/generate_index.py --check   # fail if index.yaml is out of date
+"""
+import difflib
+import os
+import sys
+
+import yaml
+
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+PLUGINS_DIR = os.path.join(REPO_ROOT, "plugins")
+INDEX_PATH = os.path.join(REPO_ROOT, "index.yaml")
+
+# Catalog identity header, surfaced in `datumctl plugin index list` and
+# `datumctl plugin browse`, followed by one entry per plugin manifest.
+HEADER = {
+    "apiVersion": "datumctl.datum.net/v1alpha1",
+    "kind": "PluginList",
+    "name": "milo-os",
+    "description": "Portable CLI plugins for the Milo platform",
+    "owner": "milo-os",
+    "homepage": "https://github.com/milo-os/cli-plugins",
+}
+
+
+def render() -> tuple[str, int]:
+    index = dict(HEADER)
+    index["items"] = []
+    for filename in sorted(os.listdir(PLUGINS_DIR)):
+        if not filename.endswith(".yaml"):
+            continue
+        with open(os.path.join(PLUGINS_DIR, filename)) as fh:
+            index["items"].append(yaml.safe_load(fh))
+    text = yaml.dump(
+        index, default_flow_style=False, allow_unicode=True, sort_keys=False
+    )
+    return text, len(index["items"])
+
+
+def main() -> None:
+    expected, count = render()
+
+    if "--check" in sys.argv[1:]:
+        current = ""
+        if os.path.exists(INDEX_PATH):
+            with open(INDEX_PATH) as fh:
+                current = fh.read()
+        if current != expected:
+            print(
+                "index.yaml is out of sync with plugins/.\n"
+                "Run `python3 scripts/generate_index.py` and commit the result.\n",
+                file=sys.stderr,
+            )
+            diff = difflib.unified_diff(
+                current.splitlines(),
+                expected.splitlines(),
+                fromfile="index.yaml (committed)",
+                tofile="index.yaml (expected)",
+                lineterm="",
+            )
+            print("\n".join(diff), file=sys.stderr)
+            sys.exit(1)
+        print(f"index.yaml is in sync ({count} plugin(s)).")
+        return
+
+    with open(INDEX_PATH, "w") as fh:
+        fh.write(expected)
+    print(f"Wrote index.yaml ({count} plugin(s)).")
+
+
+if __name__ == "__main__":
+    main()