Update Registry And Add Reconcilation

.github/workflows/reconcile.yml

@@ -0,0 +1,78 @@
+name: Reconcile
+
+# Manually triggered pin reconciliation workflow. A maintainer runs this
+# workflow; it reads each catalogued benchmark's public release tags (via
+# `git ls-remote`) and opens a PR bumping any pin whose latest release tag
+# differs from what is committed.
+on:
+  workflow_dispatch:
+    inputs:
+      benchmark:
+        description: "Bump only this benchmark (leave blank to check all)."
+        required: false
+        type: string
+
+permissions:
+  contents: write
+  pull-requests: write
+  issues: write
+
+jobs:
+  reconcile:
+    name: reconcile catalog pins
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          cache-dependency-glob: |
+            **/pyproject.toml
+            **/uv.lock
+
+      - name: Create venv
+        run: uv venv --python 3.11
+
+      - name: Install from uv.lock (dev extras)
+        run: uv sync --locked --extra dev
+
+      - name: Run reconcile
+        id: reconcile
+        env:
+          NAME: ${{ inputs.benchmark }}
+        run: |
+          if [ -n "$NAME" ]; then
+            uv run python scripts/reconcile.py --benchmark "$NAME"
+          else
+            uv run python scripts/reconcile.py
+          fi
+
+      - name: Open bump PR
+        if: steps.reconcile.outputs.bumped == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          NAME: ${{ steps.reconcile.outputs.name }}
+          TITLE: ${{ steps.reconcile.outputs.title }}
+          BODY: ${{ steps.reconcile.outputs.body }}
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          BRANCH="reconcile/$NAME"
+          git checkout -b "$BRANCH"
+          git add src/worldevals/register/
+          git commit -m "$TITLE"
+          git push -u origin "$BRANCH"
+          gh pr create --title "$TITLE" --body "$BODY" --base main --head "$BRANCH"
+
+      - name: Open failure issue
+        if: steps.reconcile.outputs.failed == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          NAME: ${{ steps.reconcile.outputs.name }}
+          REASON: ${{ steps.reconcile.outputs.reason }}
+        run: |
+          gh issue create \
+            --title "reconcile: failed to extract $NAME" \
+            --body "Reconcile could not extract task keys for **$NAME**: $REASON"

pyproject.toml

@@ -22,7 +22,7 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Artificial Intelligence",
     "Typing :: Typed",
 ]
-dependencies = ["inspect-robots>=0.3"]
+dependencies = ["inspect-robots>=0.3", "pyyaml>=6.0", "tomli>=2.0; python_version < '3.11'"]
 
 [project.optional-dependencies]
 dev = [
@@ -31,6 +31,8 @@ dev = [
     "ruff>=0.6",
     "mypy>=1.11",
     "pre-commit>=3.5",
+    "tomli>=2.0",
+    "types-PyYAML>=6.0",
 ]
 docs = [
     "mkdocs>=1.6",

scripts/gen_task_keys.py

@@ -0,0 +1,27 @@
+"""Regenerate register/<name>/task_keys.yaml from each benchmark's pinned source.
+
+Reads benchmark.yaml, fetches the pinned commit's pyproject.toml, and writes the
+extracted inspect_robots.tasks entry-point names.
+"""
+
+from __future__ import annotations
+
+import yaml
+
+from worldevals._extract import fetch_task_keys
+from worldevals.catalog import _REGISTER_DIR
+
+
+def main() -> int:
+    for benchmark_yaml in sorted(_REGISTER_DIR.glob("*/benchmark.yaml")):
+        meta = yaml.safe_load(benchmark_yaml.read_text())
+        src = meta["source"]
+        keys = fetch_task_keys(src["repository_url"], src["repository_commit"])
+        out = benchmark_yaml.parent / "task_keys.yaml"
+        out.write_text(yaml.safe_dump({"task_keys": list(keys)}, sort_keys=False))
+        print(f"{meta['name']}: {len(keys)} task keys")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/reconcile.py

@@ -0,0 +1,139 @@
+"""Reconcile catalog pins against upstream releases.
+
+For each benchmark (or a single named one), resolve the latest semver tag via
+``git ls-remote --tags``, extract task keys at that SHA, and decide whether to
+bump. When bumping, rewrite the register/ YAML files and emit step outputs for
+the GitHub Actions workflow to open a PR or an issue on failure.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import subprocess
+import sys
+
+import yaml
+
+from worldevals._extract import ExtractError, fetch_task_keys
+from worldevals._reconcile import decide_bump, format_pr_body, select_latest_tag
+from worldevals.catalog import _REGISTER_DIR, _load_catalog
+
+
+def _write_github_output(pairs: dict[str, str]) -> None:
+    """Append key=value pairs to $GITHUB_OUTPUT; print if unset."""
+    output_file = os.environ.get("GITHUB_OUTPUT")
+    if output_file:
+        with open(output_file, "a") as fh:
+            for key, value in pairs.items():
+                if "\n" in value:
+                    # Multi-line: use heredoc form
+                    fh.write(f"{key}<<EOF\n{value}\nEOF\n")
+                else:
+                    fh.write(f"{key}={value}\n")
+    else:
+        for key, value in pairs.items():
+            print(f"{key}={value}")
+
+
+def _resolve_latest_tag(repository_url: str) -> tuple[str, str] | None:
+    """Run git ls-remote --tags and pick the highest semver tag."""
+    result = subprocess.run(
+        ["git", "ls-remote", "--tags", repository_url],
+        capture_output=True,
+        text=True,
+        timeout=60,
+    )
+    if result.returncode != 0:
+        return None
+    lines = result.stdout.strip().splitlines()
+    return select_latest_tag(lines)
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Entry point: reconcile catalog pins against upstream releases."""
+    parser = argparse.ArgumentParser(description="Reconcile catalog pins")
+    parser.add_argument(
+        "--benchmark",
+        type=str,
+        default=None,
+        help="Process only the named benchmark (default: all)",
+    )
+    args = parser.parse_args(argv)
+
+    benchmarks = _load_catalog()
+    if args.benchmark:
+        benchmarks = tuple(b for b in benchmarks if b.name == args.benchmark)
+        if not benchmarks:
+            print(f"error: no benchmark named {args.benchmark!r}", file=sys.stderr)
+            return 1
+
+    for benchmark in benchmarks:
+        name = benchmark.name
+        print(f"[{name}] checking...")
+
+        tag_result = _resolve_latest_tag(benchmark.source.repository_url)
+        if tag_result is None:
+            print(f"[{name}] no semver tags found; skipping")
+            continue
+
+        latest_tag, latest_sha = tag_result
+
+        try:
+            latest_keys = fetch_task_keys(benchmark.source.repository_url, latest_sha)
+        except ExtractError as exc:
+            print(f"[{name}] extract error: {exc}", file=sys.stderr)
+            _write_github_output(
+                {
+                    "failed": "true",
+                    "name": name,
+                    "reason": str(exc),
+                }
+            )
+            continue
+
+        bump = decide_bump(
+            name=name,
+            current_sha=benchmark.source.repository_commit,
+            current_tag=benchmark.source.tag,
+            current_keys=benchmark.task_keys,
+            latest_sha=latest_sha,
+            latest_tag=latest_tag,
+            latest_keys=latest_keys,
+        )
+
+        if bump is None:
+            print(f"[{name}] up to date")
+            continue
+
+        # Rewrite register/<name>/benchmark.yaml (source section only)
+        benchmark_yaml_path = _REGISTER_DIR / name / "benchmark.yaml"
+        meta = yaml.safe_load(benchmark_yaml_path.read_text())
+        meta["source"]["repository_commit"] = bump.new_sha
+        meta["source"]["tag"] = bump.new_tag
+        benchmark_yaml_path.write_text(yaml.safe_dump(meta, sort_keys=False))
+
+        # Rewrite register/<name>/task_keys.yaml
+        task_keys_path = _REGISTER_DIR / name / "task_keys.yaml"
+        task_keys_path.write_text(
+            yaml.safe_dump({"task_keys": list(bump.new_keys)}, sort_keys=False)
+        )
+
+        title = f"bump({name}): {bump.old_tag} to {bump.new_tag}"
+        body = format_pr_body(bump)
+        print(f"[{name}] {title}")
+
+        _write_github_output(
+            {
+                "bumped": "true",
+                "name": name,
+                "title": title,
+                "body": body,
+            }
+        )
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/worldevals/_extract.py

@@ -0,0 +1,78 @@
+"""Read a benchmark's task entry points from its pinned pyproject.toml.
+
+Fetch only pyproject.toml at a pinned commit and read the
+`[project.entry-points."inspect_robots.tasks"]` table.
+"""
+
+from __future__ import annotations
+
+import sys
+import urllib.error
+import urllib.request
+from collections.abc import Callable
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:  # pragma: no cover - stdlib tomllib backport for 3.10
+    import tomli as tomllib
+
+_ENTRY_POINT_GROUP = "inspect_robots.tasks"
+
+
+class ExtractError(Exception):
+    """A benchmark's task keys could not be extracted."""
+
+
+class NoTaskEntryPointsError(ExtractError):
+    """The pinned pyproject.toml declares no inspect_robots.tasks entry points."""
+
+
+def parse_task_keys(pyproject_bytes: bytes) -> tuple[str, ...]:
+    """Return sorted task entry-point names from pyproject.toml bytes.
+
+    Raises ``NoTaskEntryPointsError`` if the entry-point table is missing or
+    empty, and ``ExtractError`` if the TOML cannot be parsed.
+    """
+    try:
+        data = tomllib.loads(pyproject_bytes.decode("utf-8"))
+    except (tomllib.TOMLDecodeError, UnicodeDecodeError) as exc:
+        raise ExtractError(f"malformed pyproject.toml: {exc}") from exc
+    table = data.get("project", {}).get("entry-points", {}).get(_ENTRY_POINT_GROUP, {})
+    if not table:
+        raise NoTaskEntryPointsError(
+            f'no [project.entry-points."{_ENTRY_POINT_GROUP}"] table found'
+        )
+    return tuple(sorted(table))
+
+
+_RAW_URL = "https://raw.githubusercontent.com/{owner}/{repo}/{sha}/pyproject.toml"
+
+
+def parse_owner_repo(repository_url: str) -> tuple[str, str]:
+    """Split ``https://github.com/<owner>/<repo>`` into ``(owner, repo)``."""
+    prefix = "https://github.com/"
+    if not repository_url.startswith(prefix):
+        raise ExtractError(f"not a github.com URL: {repository_url}")
+    parts = repository_url[len(prefix) :].strip("/").split("/")
+    if len(parts) != 2 or not all(parts):
+        raise ExtractError(f"cannot parse owner/repo from: {repository_url}")
+    return parts[0], parts[1]
+
+
+def _fetch_pyproject(owner: str, repo: str, sha: str) -> bytes:  # pragma: no cover
+    url = _RAW_URL.format(owner=owner, repo=repo, sha=sha)
+    try:
+        with urllib.request.urlopen(url, timeout=30) as resp:
+            data: bytes = resp.read()
+            return data
+    except urllib.error.URLError as exc:
+        raise ExtractError(f"failed to fetch {url}: {exc}") from exc
+
+
+_FETCH: Callable[[str, str, str], bytes] = _fetch_pyproject
+
+
+def fetch_task_keys(repository_url: str, sha: str) -> tuple[str, ...]:
+    """Fetch the pinned pyproject.toml and return its sorted task entry-point names."""
+    owner, repo = parse_owner_repo(repository_url)
+    return parse_task_keys(_FETCH(owner, repo, sha))