From 9ce158ff6847c7aaddd3c4cf2dec7eec8a312d3d Mon Sep 17 00:00:00 2001 From: Kudo Chien Date: Mon, 29 Jun 2026 22:31:47 +0800 Subject: [PATCH] add expo-plugin-eval --- .claude/eval-shared/scripts/clean-fixture.sh | 56 ++ .claude/eval-shared/scripts/make-fixture.sh | 110 ++++ .claude/skills/expo-plugin-eval/SKILL.md | 250 +++++++++ .../expo-plugin-eval/agents/plugin-grader.md | 73 +++ .../references/design-rubric.md | 46 ++ .../references/example-apps.md | 77 +++ .../expo-plugin-eval/scripts/check-static.sh | 1 + .../expo-plugin-eval/scripts/clean-fixture.sh | 1 + .../scripts/discover_routes.py | 145 +++++ .../scripts/generate_viewer.py | 494 ++++++++++++++++++ .../expo-plugin-eval/scripts/latest-sdk.sh | 1 + .../expo-plugin-eval/scripts/make-fixture.sh | 1 + .../scripts/make-workspace.sh | 1 + .../scripts/snapshot-routes-android.sh | 207 ++++++++ .../scripts/snapshot-routes-ios.sh | 169 ++++++ .../scripts/snapshot-routes-web.sh | 80 +++ .../expo-skill-eval/scripts/check-static.sh | 31 +- .../expo-skill-eval/scripts/clean-fixture.sh | 57 +- .../scripts/generate_viewer.py | 41 +- .../expo-skill-eval/scripts/make-fixture.sh | 96 +--- 20 files changed, 1771 insertions(+), 166 deletions(-) create mode 100755 .claude/eval-shared/scripts/clean-fixture.sh create mode 100755 .claude/eval-shared/scripts/make-fixture.sh create mode 100644 .claude/skills/expo-plugin-eval/SKILL.md create mode 100644 .claude/skills/expo-plugin-eval/agents/plugin-grader.md create mode 100644 .claude/skills/expo-plugin-eval/references/design-rubric.md create mode 100644 .claude/skills/expo-plugin-eval/references/example-apps.md create mode 120000 .claude/skills/expo-plugin-eval/scripts/check-static.sh create mode 120000 .claude/skills/expo-plugin-eval/scripts/clean-fixture.sh create mode 100755 .claude/skills/expo-plugin-eval/scripts/discover_routes.py create mode 100644 .claude/skills/expo-plugin-eval/scripts/generate_viewer.py create mode 120000 .claude/skills/expo-plugin-eval/scripts/latest-sdk.sh create mode 120000 .claude/skills/expo-plugin-eval/scripts/make-fixture.sh create mode 120000 .claude/skills/expo-plugin-eval/scripts/make-workspace.sh create mode 100755 .claude/skills/expo-plugin-eval/scripts/snapshot-routes-android.sh create mode 100755 .claude/skills/expo-plugin-eval/scripts/snapshot-routes-ios.sh create mode 100755 .claude/skills/expo-plugin-eval/scripts/snapshot-routes-web.sh mode change 100755 => 120000 .claude/skills/expo-skill-eval/scripts/clean-fixture.sh mode change 100755 => 120000 .claude/skills/expo-skill-eval/scripts/make-fixture.sh diff --git a/.claude/eval-shared/scripts/clean-fixture.sh b/.claude/eval-shared/scripts/clean-fixture.sh new file mode 100755 index 0000000..7f4141b --- /dev/null +++ b/.claude/eval-shared/scripts/clean-fixture.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Reclaim disk after a fixture's screenshots are captured. Essential for +# dev-build runs, where each `expo run:` leaves multi-GB native build +# output (iOS Pods + DerivedData, Android Gradle build). Removes the heavy, +# regenerable artifacts and keeps the app source + git history, so the grader's +# `git diff` still works. +# +# Usage: clean-fixture.sh +# +# Env: EXPO_SKILL_EVAL_KEEP_DERIVEDDATA=1 skips the iOS DerivedData sweep (set it +# only if you keep a real Xcode project literally named "fixture"). +set -uo pipefail + +APP="${1:?usage: clean-fixture.sh }" + +# Safety guard: only ever clean inside an expo-skill-eval / expo-plugin-eval workspace. +case "$APP" in + *expo-skill-eval-*|*expo-plugin-eval-*) : ;; + *) echo "clean-fixture: refusing to clean '$APP' (not an expo eval fixture)" >&2; exit 1 ;; +esac +[[ -d "$APP" ]] || { echo "clean-fixture: $APP not found, skipping"; exit 0; } + +# Stop any Metro / Expo dev server that may still be running for this fixture. +# The snapshot scripts' EXIT trap normally handles this, but cleaning up here +# catches leftover processes from crashes or interrupted runs. +# Ports 8081 (iOS) and 8082 (Android) are the two ports the eval harness +# reserves — freeing them here is safe because clean-fixture.sh only runs after +# all screenshots for this fixture have been captured. +# +# `-sTCP:LISTEN` is REQUIRED, not a refinement: without it, `lsof -ti tcp:8082` +# also matches the *established* connection the adb daemon holds from the +# `adb reverse tcp:8082` tunnel, and SIGKILLing the adb daemon crashes the +# emulator with a gfxstream gRPC SIGABRT (std::bad_function_call). Keep the flag +# so only the Metro listener is killed. +lsof -ti tcp:8081 -sTCP:LISTEN 2>/dev/null | xargs kill -9 2>/dev/null || true +lsof -ti tcp:8082 -sTCP:LISTEN 2>/dev/null | xargs kill -9 2>/dev/null || true + +# Per-fixture heavy dirs — all gitignored / regenerable (node_modules, the +# prebuilt native projects incl. iOS Pods and Android Gradle output, caches). +rm -rf \ + "$APP/node_modules" \ + "$APP/ios" \ + "$APP/android" \ + "$APP/.expo" \ + "$APP/dist" \ + "$APP/web-build" 2>/dev/null || true + +# iOS DerivedData for fixture builds. create-expo-app names the project +# "fixture", so its build output lives under DerivedData/fixture-. This is +# a cache (worst case a rebuild), so it's safe to drop between fixtures. +if [[ "${EXPO_SKILL_EVAL_KEEP_DERIVEDDATA:-0}" != "1" ]]; then + DD="$HOME/Library/Developer/Xcode/DerivedData" + [[ -d "$DD" ]] && rm -rf "$DD"/fixture-* 2>/dev/null || true +fi + +echo "cleaned fixture: $APP" diff --git a/.claude/eval-shared/scripts/make-fixture.sh b/.claude/eval-shared/scripts/make-fixture.sh new file mode 100755 index 0000000..36a65a3 --- /dev/null +++ b/.claude/eval-shared/scripts/make-fixture.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Create an Expo app fixture for an eval run. +# +# Usage: make-fixture.sh [sdk-version] [variant] +# project-path Where the app should end up (must not exist yet) +# sdk-version Expo SDK major version, e.g. "56". Omit for the latest template. +# variant "clean" (default) runs the template's reset-project script so +# executors start from a blank app instead of the example +# screens. "full" keeps the example tabs (Home/Explore) - use +# it for evals whose prompt assumes an existing app. +# +# One pristine app is created per SDK version + variant with +# `bunx create-expo-app -t default@sdk-` and cached under +# $EXPO_SKILL_EVAL_CACHE (default ~/.cache/expo-skill-eval/fixtures). +# Each call clones the cache with APFS copy-on-write, so only the first +# call per SDK version pays the install cost. +set -euo pipefail + +PROJECT_PATH="${1:?usage: make-fixture.sh [sdk-version] [variant]}" +SDK_VERSION="${2:-}" +VARIANT="${3:-clean}" +if [[ "$VARIANT" != "clean" && "$VARIANT" != "full" ]]; then + echo "error: variant must be 'clean' or 'full', got '$VARIANT'" >&2 + exit 1 +fi +CACHE_ROOT="${EXPO_SKILL_EVAL_CACHE:-$HOME/.cache/expo-skill-eval/fixtures}" + +if [[ -e "$PROJECT_PATH" ]]; then + echo "error: $PROJECT_PATH already exists" >&2 + exit 1 +fi + +TEMPLATE="default" +CACHE_KEY="latest" +if [[ -n "$SDK_VERSION" ]]; then + TEMPLATE="default@sdk-${SDK_VERSION}" + CACHE_KEY="sdk-${SDK_VERSION}" +fi +if [[ "$VARIANT" == "clean" ]]; then + CACHE_KEY="${CACHE_KEY}-clean" +fi +CACHE_DIR="$CACHE_ROOT/$CACHE_KEY" + +if [[ ! -d "$CACHE_DIR" ]]; then + echo "building fixture cache for template '$TEMPLATE'..." >&2 + mkdir -p "$CACHE_ROOT" + TMP_DIR="$(mktemp -d)" + trap 'rm -rf "$TMP_DIR"' EXIT + (cd "$TMP_DIR" && bunx create-expo-app --yes --template "$TEMPLATE" fixture) >&2 + if [[ "$VARIANT" == "clean" ]]; then + # Strip the example screens so executors start from a blank app. + # Answer "n" to the "move to /example?" prompt - delete them instead. + (cd "$TMP_DIR/fixture" && echo n | bun run reset-project) >&2 + fi + # Pin a stable iOS bundle identifier and Android package. The default template + # leaves both unset, so a dev-build run (`expo run:ios`/`expo run:android`) + # would prompt for them - which fails in the snapshot scripts' non-interactive + # mode - and the snapshot scripts need a known id to relaunch the app by. + (cd "$TMP_DIR/fixture" && bun -e ' + const f = "app.json"; + const j = await Bun.file(f).json(); + j.expo ??= {}; + j.expo.ios ??= {}; + j.expo.android ??= {}; + j.expo.ios.bundleIdentifier ??= "com.exposkilleval.fixture"; + j.expo.android.package ??= "com.exposkilleval.fixture"; + // A custom scheme lets a dev-build deep-link to specific routes + // (://); harmless for Expo Go, which uses exp:// instead. + j.expo.scheme ??= "exposkilleval"; + await Bun.write(f, JSON.stringify(j, null, 2) + "\n"); + ') >&2 + # The CLI normally generates expo-env.d.ts on first `expo start`; the static + # gate runs tsc before any start, so create it up front (it provides the + # CSS-module and Metro type declarations from expo/types). + printf '/// \n' >"$TMP_DIR/fixture/expo-env.d.ts" + # expo-dev-client is required for dev-build snapshots: it registers the + # expo-development-client URL scheme and enables custom Metro port connections + # (expo run:ios/android --port N opens the app via a deep link that the dev + # client handles). Without it, a custom port deep link may not open the app. + (cd "$TMP_DIR/fixture" && bunx expo install expo-dev-client) >&2 + # First `expo lint` self-installs eslint + eslint-config-expo and writes + # eslint.config.js if missing; it may exit non-zero on that bootstrap run. + (cd "$TMP_DIR/fixture" && CI=1 bunx expo lint) >&2 || true + printf '.eval-static/\n' >>"$TMP_DIR/fixture/.gitignore" + mv "$TMP_DIR/fixture" "$CACHE_DIR" +fi + +mkdir -p "$(dirname "$PROJECT_PATH")" +# -c uses APFS clonefile (instant, no extra disk); fall back to a plain copy. +cp -Rc "$CACHE_DIR" "$PROJECT_PATH" 2>/dev/null || cp -R "$CACHE_DIR" "$PROJECT_PATH" + +# Disable the *published* expo plugin for any `claude -p` run inside this fixture, +# so it can't collide with a locally --plugin-dir'd copy (the model could otherwise +# trigger the published skill and we'd silently score it). A *project*-level +# settings file is the reliable lever: project settings override the user's +# ~/.claude/settings.json, so disabling there alone doesn't take effect. Written +# per-clone (not baked into the shared cache) so an existing cache still picks it +# up. The id is the published plugin's @; override per env. +PUBLISHED_PLUGIN_ID="${EXPO_SKILL_EVAL_PUBLISHED_PLUGIN_ID:-expo@claude-plugins-official}" +mkdir -p "$PROJECT_PATH/.claude" +printf '{\n "enabledPlugins": {\n "%s": false\n }\n}\n' "$PUBLISHED_PLUGIN_ID" \ + >"$PROJECT_PATH/.claude/settings.local.json" + +# Fresh git history so `git diff` in the app shows exactly what the executor changed. +rm -rf "$PROJECT_PATH/.git" +git -C "$PROJECT_PATH" init -q +git -C "$PROJECT_PATH" add -A +git -C "$PROJECT_PATH" commit -qm "fixture: pristine $TEMPLATE template" --no-verify + +echo "$PROJECT_PATH" diff --git a/.claude/skills/expo-plugin-eval/SKILL.md b/.claude/skills/expo-plugin-eval/SKILL.md new file mode 100644 index 0000000..6ff211b --- /dev/null +++ b/.claude/skills/expo-plugin-eval/SKILL.md @@ -0,0 +1,250 @@ +--- +name: expo-plugin-eval +description: Evaluate the whole Expo plugin (all skills + the Expo MCP) end-to-end by having it build a complete multi-screen app from a prompt or a target screenshot, then deep-linking to every Expo Router route and screenshotting each on iOS/Android/web via Expo Go or a dev build. Reports which skills and MCP tools the plugin actually used. Use when the user wants to eval the Expo plugin as a whole, benchmark the plugin's app-building quality, test that the plugin produces a working multi-screen app, or compare apps built with vs without the plugin. +version: 1.0.0 +license: MIT +allowed-tools: "Read(~/.cache/expo-skill-eval/**), Read(/tmp/expo-plugin-eval-*/**), Read(/private/tmp/expo-plugin-eval-*/**), Write(/tmp/expo-plugin-eval-*/**), Write(/private/tmp/expo-plugin-eval-*/**), Edit(/tmp/expo-plugin-eval-*/**), Edit(/private/tmp/expo-plugin-eval-*/**), Bash(python3 /tmp/expo-plugin-eval-*), Bash(python3 /private/tmp/expo-plugin-eval-*), Bash(python3 *expo-plugin-eval/scripts/*), Bash(tee /tmp/expo-plugin-eval-*), Bash(tee /private/tmp/expo-plugin-eval-*), Bash(bash *expo-plugin-eval/scripts/*)" +--- + +# Expo Plugin Eval + +Evaluates the **whole `expo` plugin** — all skills auto-triggering by description **plus the Expo MCP server** — by giving it a full-app prompt (or a target screenshot to reproduce), letting it build a complete app, then **deep-linking to every Expo Router route and screenshotting each one** on the chosen platforms. It also **reports which skills and MCP tools the plugin used** while building. This is the plugin-level companion to `expo-skill-eval` (which evaluates one skill at a time); it reuses that skill's debugged fixture/static/clean scripts via symlinks. + +Requirements: macOS with Xcode (iOS simulators), Android SDK with at least one AVD, and `bun`. No other device tooling is assumed. + +Workspace root: `/private/tmp/expo-plugin-eval-/iteration-N/` (e.g. `/private/tmp/expo-plugin-eval-recipe-app/iteration-1/`). + +## Before starting — clarify scope + +**Confirm all of the following up front, before any pipeline work — don't skip any** (only skip a given item if the request already states that choice). Batch them into `AskUserQuestion` calls of ≤4 questions each, in this order: + +1. **Prompts** — which app(s) to build. Built-in example apps (from `references/example-apps.md`) are **pre-selected**; drop any, add a custom typed prompt, or **build from an uploaded screenshot** (a target UI to reproduce). See **Prompts** below. +2. **Baseline** — **A/B vs a no-plugin baseline (recommended default)** or plugin-only. See **Configs** below. +3. **Skill loading** — **force-invoke a preset (recommended; Dev/UI)** or routing-style (interactive simulation). Recommend by intent — see **Force-invoking skills** below. +4. **Expo SDK** — latest (default, auto-detected) or a pinned version. +5. **Runner** — Expo Go (default) or development build. +6. **Platforms** — iOS / Android / web (always offer all three). +7. **Permission flag** for `claude -p` — skip-permissions (default) or accept-edits. +8. **Viewer delivery** — local only (default) or publish a shareable Artifact. + +Items 4–6 (SDK, runner, platforms) fit naturally in one `AskUserQuestion` call. **You no longer ask the user to disable the published `expo` plugin** — the fixture handles it (see the next paragraph); it's documentation + a post-run check, not a question. + +**Loading the plugin under test — and the published-`expo` collision (handled by the fixture).** Executors load the **local, in-repo** plugin via `--plugin-dir /plugins/expo` so every skill can auto-trigger from its description, and the plugin's `.mcp.json` brings in the Expo MCP. A `claude -p` subprocess **inherits the user's global plugin/MCP config**, so if the *published* `expo` plugin is installed it would collide with `--plugin-dir plugins/expo` (the model may trigger the published `expo:expo-ui` instead of your local edits, and usage detection only sees the tool-call name — you'd silently score the published copy), and it would contaminate the no-plugin baseline. + +**This is now handled automatically:** `make-fixture.sh` writes `.claude/settings.local.json` into every fixture with `{"enabledPlugins": {"expo@claude-plugins-official": false}}`. The executor runs with the fixture as its cwd, so that **project-level** setting disables the published plugin for the subprocess — and it has to be project-level: **project settings override the user's `~/.claude/settings.json`, so disabling there alone does *not* take effect** (the exact trap seen in practice). The locally `--plugin-dir`'d copy is unaffected (it's loaded by the CLI flag, not the marketplace `enabledPlugins` registry). Two caveats: (1) the default id is `expo@claude-plugins-official` — if your environment installed the published plugin under a different `@`, set `EXPO_SKILL_EVAL_PUBLISHED_PLUGIN_ID` so `make-fixture.sh` disables the right one; (2) it's still worth a glance at the usage panel — if a `without_plugin` baseline shows `expo:*` skills used, the published copy leaked in (wrong id), so fix the id and re-run. `expo-plugin-eval` itself is a standalone project skill (not part of the `expo` plugin), so none of this disables the harness. + +**Prompts — built-in, custom, or a target screenshot.** The prompt is what the plugin builds; it's separate from the per-route grading. Confirm with `AskUserQuestion` (skip if the request already names one), as a **multi-select**: + +- **Built-in example apps** — multi-screen apps from `references/example-apps.md` (recipe app, fitness tracker, notes, weather, social profile). They're designed to exercise navigation (so route capture has work to do) and a spread of skills. **Pre-select all** so the default run covers the standard cases; let the user deselect. +- **Custom typed prompt** — don't spend an option slot on this; `AskUserQuestion`'s auto "Type something"/Other entry covers it. Anything typed becomes a custom case. +- **Build from an uploaded screenshot** — **always reserve a slot for this** (it must never be the option dropped when the 4-slot cap fills — collapse the built-ins into one pre-selected "All example apps" option if needed). The user gives the path to a **target screenshot**; the executor opens it with its Read tool and builds a matching app; the case records the path as `reference_image` and grading compares the result to the target (step 6). + +Each selected prompt (built-in, typed, or image) becomes one eval case. When the prompt is a screenshot, the run captures the generated app and the grader scores it against the target. + +**Phrase prompts at user-intent level — over-specified prompts suppress skill triggering.** Plugin skills auto-trigger when the model recognizes it needs domain guidance; a prompt that already dictates the implementation ("use Expo Router file-based routing", "each screen its own route", "use a data-fetching approach with loading and error states") hands the model everything and it builds the app directly **without ever invoking a skill** — so `with_plugin` collapses toward the baseline and the eval measures nothing. Write each prompt the way a real user would: say **what the app does and which screens it has**, not **how to build it**. The built-in prompts in `references/example-apps.md` are phrased this way deliberately. If a skill-specific behavior is the point (NativeWind styling, live data fetching), name that *domain* naturally ("style it with Tailwind", "use real weather data") — that's a real user signal, not implementation dictation. Note the asymmetry: the **eval prompt** (what we measure) stays natural, while the **executor scaffolding** (emit `routes.json`, don't start servers — see step 2) is separate harness instruction that doesn't affect triggering. **Zero skill usage with the plugin loaded is a finding, not a harness bug** — it means the plugin added no lift for that prompt (either the prompt was too prescriptive, or the skills' descriptions don't recall on that phrasing). The A/B delta and the usage panel surface it; investigate trigger recall in the affected skill's `SKILL.md` description, and cross-check with `expo-skill-eval`'s trigger eval, which exercises recall directly with short natural queries. + +**Configs — A/B (recommended default) or plugin-only.** Default to **A/B**: two configs, `with_plugin` and `without_plugin` (a plain `claude -p` with no `--plugin-dir`/MCP). The viewer then shows the plugin's *lift* per config — the most informative result, and the only way to tell whether loading the plugin actually changed the output (a `with_plugin` run that triggered no skills should look ~identical to its baseline; the delta makes that obvious). Offer **plugin-only** (`with_plugin` alone, graded on its own merits) as the cheaper option when build time / token cost matters — A/B doubles fixture count and build time. Pass the config set to `make-workspace.sh` (`"with_plugin without_plugin"` for A/B, or `"with_plugin"`). + +**Force-invoking skills — presets.** This question picks what an eval run *measures*, so recommend by intent rather than a fixed default: + +- **Recommend a force preset (Dev/UI) by default** when the goal is to evaluate or improve the skills' output — the common case for this tool. Forcing guarantees the skills actually run, so the build exercises their *content*; without it a natural run often triggers **nothing** and `with_plugin` collapses to the baseline (uninformative). Recall — "does the skill fire on a realistic prompt?" — is measured more directly and cheaply by `expo-skill-eval`'s trigger eval, so it's usually not worth spending a whole-app build on here. +- **Recommend routing-style** when the question is "does `expo-overview`'s dispatch chain pick the right leaf skill?" — the closest simulation of what interactive Claude Code does. A single generic directive tells the executor to find and invoke the right skill itself; the model chooses which one, replicating the interactive routing layer. Skills that fire are model-chosen (not pre-specified), so a routing-style run is more informative than force-invoke for testing dispatch quality. + +Ask with `AskUserQuestion` (single-select preset; the auto "Other" entry takes a custom comma-separated list); skip if the request already says. Presets: + +- **[Force] Dev / UI skills (recommended default).** Instructs the executor to invoke each named skill via the `Skill` tool before writing code — the skills *will* run, guaranteed. The set: `building-native-ui`, `expo-dev-client`, `expo-examples`, `expo-tailwind-setup`, `expo-ui`, `native-data-fetching`, `use-dom`. Excludes ops/release skills (`expo-cicd-workflows`, `expo-deployment`, `eas-update-insights`), native-packaging skills (`expo-module`, `add-app-clip`, `expo-brownfield`), `expo-api-routes`, `upgrading-expo`, and `expo-observe` — forcing those into an app build just adds noise. +- **[Force] This case's `expected_skills`.** Force exactly the skills the eval case already lists — the most targeted option; each app forces only what it's "supposed" to use. +- **[Auto] Interactive simulation.** Gives the executor a single hint: *"Before writing any code, call the Skill tool with the most relevant plugin skill for this task (start with `expo:expo-overview` if unsure) and follow its guidance before building anything."* The model decides which skill to invoke — replicating the interactive Claude Code routing layer. Skills that fire are model-chosen, not pre-specified. Set `routing_style: true` on the eval case. +- **[Neutral] No directive.** No instruction about skills is added to the executor prompt. Claude may or may not invoke a skill depending on natural recall. Use this when the question is whether the plugin provides any lift at all without guidance — expect `with_plugin` to collapse toward the baseline unless a skill's description fires on the prompt. +- **Custom.** A comma-separated skill list typed into "Other" (treated as force-invoke). + +How it works (see step 2): when a force set is chosen, the `with_plugin` executor prompt is prefixed with a directive to **use the `Skill` tool to invoke each named skill** (`expo:`) and follow its guidance before building. Only `with_plugin` is forced — the `without_plugin` baseline has no plugin loaded, so it stays natural. Write the set to each case's `force_skills`, and the usage parser marks `skill_usage.json` `"forced": true`. **Forcing changes what an A/B delta means** — it becomes the *content* lift of those skills (recall is bypassed), not "does the plugin naturally help"; the viewer labels a forced config so the two aren't confused. Don't force *all 16* skills: half the plugin is orthogonal to an app build and only adds conflicting context. To exercise the whole plugin instead, run one case per skill (that's `expo-skill-eval`'s job), not one mega-build. + +**Expo SDK — once, up front.** Detect the latest with `bash /abs/path/expo-plugin-eval/scripts/latest-sdk.sh` (prints the major, e.g. `56`; it uses `bun` internally and is covered by the bash-scripts rule — don't run the registry query inline). Default to that latest SDK (it stays compatible with the Expo Go that `expo start` installs; pinning an SDK *older* than the device's Expo Go makes `expo start` try to prompt and every snapshot fails in non-interactive mode). Write the chosen version into each case's `runtime.sdk` and pass it to `make-fixture.sh`. + +**Runner — Expo Go (default) or a development build.** Expo Go runs anything Expo Go bundles (incl. `@expo/ui` on SDK 56+); it's fast and is the **fully-supported multi-route path**. A development build (`expo run:ios/android`) is for output needing custom native code; it's much slower, multi-GB per fixture, and route deep-linking is **best-effort** (see **Deep-link mechanics**). Prefer fewer cases + a single platform for dev-build runs and keep a few GB free. Pass the choice via `EXPO_SKILL_EVAL_RUNNER` (`expo-go` default, or `dev-build`) and reflect it in each case's `runtime.mode`. + +**Platforms — always ask.** Offer iOS / Android / web (multi-select); default iOS + Android, but always offer web (NativeWind/Tailwind, `use-dom`, API routes, plain RN, and `@expo/ui`'s *universal* components all render on web; only platform-specific `@expo/ui/swift-ui` / `jetpack-compose` trees render blank, which is itself a signal). Web runs via `snapshot-routes-web.sh` (`expo start --web` + Playwright) **regardless of runner** (`expo run` is native-only). Write the set into each case's `runtime.platforms`. + +**Permission flag for `claude -p` — once, before starting.** Either `--dangerously-skip-permissions` (recommended; each executor runs unattended in a throwaway fixture) or `--permission-mode acceptEdits` (Bash/installs auto-denied with no TTY, so some evals may be partial). A bare `claude -p` with neither can't write files. Apply the same answer to every subprocess this run. + +**Viewer delivery — once, up front.** Local only (default; `generate_viewer.py` writes `viewer.html` and opens it) or also **publish a shareable Artifact** at the end (outward-facing — only if the user opts in). See **Viewer**. + +## Eval case schema + +You generate the run's eval cases — one per chosen prompt — and write them to `/iteration-N/evals.json` (the viewer reads them). Each case: + +```json +{ + "id": 1, + "prompt": "Build a recipe app where I can browse a feed of recipes, tap one to see its details with ingredients and steps, keep favorites, and open settings to turn on dark mode.", + "reference_image": "/abs/path/to/target.png", + "expectations": ["Uses Expo Router file-based routing", "TypeScript compiles with no errors"], + "runtime": { "mode": "expo-go", "platforms": ["ios", "android"], "sdk": "56" }, + "routes": [ + {"path": "/", "label": "Home feed"}, + {"path": "/recipe/1", "label": "Recipe detail"}, + {"path": "/settings", "label": "Settings"} + ], + "visual_expectations": [ + "Home shows a scrollable list of recipe cards", + "Settings shows a labeled dark-mode toggle" + ], + "expected_skills": ["building-native-ui", "expo-ui"], + "force_skills": ["building-native-ui", "expo-ui"], + "routing_style": false +} +``` + +- `runtime.mode`: `"expo-go"` (default), `"dev-build"`, or `"static-only"` (stop after the static gate — no device; useful for a quick CI-style check that the plugin produces a building app). +- `runtime.platforms`: subset of `ios`, `android`, `web`. +- `runtime.sdk`: SDK major chosen up front (omit to use the latest template). +- `routes` (optional **hint**): expected navigable routes with a deep-link path; sample params filled in for dynamic routes (e.g. `/recipe/1`). **Not authoritative** — at capture time the executor's emitted `routes.json` wins, then `discover_routes.py`, then this hint. Always include `/` first. +- `reference_image` (optional — **image prompt**): absolute path to a target screenshot the app must reproduce. The executor opens it and builds a match; the grader scores fidelity (step 6). +- `expected_skills` (optional): skills you'd expect the plugin to use — reported as coverage, not a pass/fail gate. +- `force_skills` (optional): skills the `with_plugin` executor is **instructed to invoke** (`expo:`) before building — set from the **Skill loading** preset chosen up front. Omit (or `[]`) for natural triggering. Only applies to `with_plugin`. See step 2. +- `routing_style` (optional, default `false`): when `true`, the executor receives the generic routing directive instead of a named-skill force list. Mutually exclusive with a non-empty `force_skills`. The usage parser records `routing_style: true` in `skill_usage.json` (and leaves `forced: false`) so the viewer labels the run as routing-style, not forced. + +Pull `prompt`/`routes`/`visual_expectations`/`expected_skills` for built-in cases straight from `references/example-apps.md`. + +## Pipeline per eval case + +**Orchestration model — on the main thread you run `python3 ` and almost nothing else.** Every phase is driven by a small Python orchestrator you `Write` into the workspace and run with `python3 /private/tmp/expo-plugin-eval-/.py` (covered by the `python3` rule). The orchestrators are the **only** place the `scripts/*.sh` files are invoked — always via `subprocess.run(["bash", "/.sh", …])`, which runs as a child of `python3` and needs no rule of its own — and the only place parallelism, logging, and directory creation live. On the main thread you only: **Write** orchestrators, **run** them with `python3`, **inspect** outputs with `Read`/`Glob`/`Grep`, and **spawn the grader subagent**. Never chain/background/pipe shell constructs (each `|`/`&&`/`&`/`wait`/`echo` segment has no rule and prompts; the one allowed pipe is `… 2>&1 | tee /…log`). **Run each orchestrator in the foreground** (it parallelizes within a phase). **Expect one permission prompt at the very start** — the first `Write` into the workspace; choose **"allow all edits in this directory for the session"** and every later write goes through silently (`allowed-tools` suppresses `Bash`/`Read` but not `Write`/`Edit`). `` is this skill's `scripts/` dir; its `make-fixture.sh`/`clean-fixture.sh`/`latest-sdk.sh`/`make-workspace.sh`/`check-static.sh` are symlinks (to `eval-shared/` and `expo-skill-eval/`) and run identically. + +### 0. Workspace setup + +```bash +bash /abs/path/expo-plugin-eval/scripts/make-workspace.sh /private/tmp/expo-plugin-eval- iteration-N "with_plugin" +``` + +Use `"with_plugin without_plugin"` for A/B. This creates `trigger-evals/scratch` (unused here, harmless) and `iteration-N/eval-//outputs` per eval. Per-platform/route output dirs (`outputs//`) are created later by the snapshot scripts' `mkdir`. Never use ad-hoc `mkdir`. + +### 1. Fixtures + +Each executor run gets a fresh Expo app from `scripts/make-fixture.sh `: + +```bash +scripts/make-fixture.sh /iteration-N/eval-X//app +``` + +It clones an APFS copy-on-write cache under `~/.cache/expo-skill-eval/fixtures/` (shared with `expo-skill-eval` — fixtures aren't rebuilt across the two skills), runs the template's `reset-project` (blank app, so every screen is the plugin's), pins iOS/Android ids **and an app `scheme`** (for dev-build deep links), pre-installs `expo-dev-client`, and resets git so `git diff` shows exactly what the executor changed. **Build all fixtures sequentially in a plain Python loop first** (concurrent creation races the shared bun cache — `EEXIST`), then fan out executors. Only the first fixture per SDK+variant pays the install cost; the rest are ~1s clones. + +### 2. Generate (executor subprocesses) + +Write `run_executors.py`. **First** create every run's fixture sequentially (step 1). **Then** run the executors in parallel via a `ThreadPoolExecutor`, each a `claude -p` subprocess (not the `Agent` tool — that prompts for in-fixture edits). Each: + +- Strip `CLAUDECODE` from the env (`{k:v for k,v in os.environ.items() if k!="CLAUDECODE"}`) — otherwise `claude -p` hangs when nested in a running session. +- The permission flag chosen up front (`--dangerously-skip-permissions` or `--permission-mode acceptEdits`). +- `--output-format=stream-json --verbose --include-partial-messages` and capture the stream to a log next to the fixture (for usage + token parsing). +- **`with_plugin` only:** `--plugin-dir /plugins/expo`. This loads all skills (auto-triggering) **and** the plugin's `.mcp.json` (the Expo MCP). If the usage parser later shows the MCP never connected, add `--mcp-config /plugins/expo/.mcp.json` as a fallback. `` must be the absolute path to the in-repo `plugins/expo` (the dir with `.claude-plugin/plugin.json`) — the subprocess runs from the fixture cwd, so a relative path won't resolve. +- **`without_plugin`:** no `--plugin-dir`/`--mcp-config` — the no-plugin baseline. +- Timeout 1200s (a full multi-screen app build regularly takes 5–10 min; add another 3–5 min when forced skills are in play since each skill invocation consumes context before any code is written — 900s is too tight for the Dev/UI preset of 7 skills). + +Executor prompt must include: +- The eval prompt, and for **image cases** the target path + "Open the reference screenshot at `` with your Read tool and build an app whose UI matches it as closely as you can — layout, components, spacing, and colors." If the user uploaded the screenshot inline (no file path), describe the UI in detail in the prompt instead — tab structure, screen layouts, components, and visual style — and omit `reference_image` from the eval case. Pass the same description to the grader. +- **Relay any user hint about a specific library or UI style into the `with_plugin` executor prompt.** When the user says something like "try to use `@expo/ui`" or "style it with Tailwind", that signal belongs in the `with_plugin` executor prompt verbatim — not just in your own skill-preset selection. Keep it at domain-intent level ("use `@expo/ui` components where they fit naturally") rather than a hard dictate, but make sure it reaches the executor. The `without_plugin` baseline should not receive library-specific hints (it has no plugin to guide correct usage). +- **Force-invoke directive (`with_plugin` only, when the case has a non-empty `force_skills`):** prefix the prompt with "Before writing any code, use the Skill tool to invoke each of these plugin skills and follow their guidance while building: `expo:`, `expo:`, … Invoke every one." Build the list from `force_skills` (skip entirely for natural-triggering runs and for the `without_plugin` baseline, which has no plugin to invoke from). +- **Routing-style directive (`with_plugin` only, when the case has `routing_style: true`):** instead of naming specific skills, prefix the prompt with "Before writing any code, call the Skill tool with the most relevant plugin skill for this task (start with `expo:expo-overview` if unsure) and follow its guidance before building anything." The model decides which skill to invoke — replicating what interactive Claude Code's routing layer does. Do NOT set `forced: true` in `skill_usage.json` for routing-style runs; set `routing_style: true` so the viewer labels them correctly. +- **Note:** force-invoking a skill guarantees it is *read*, not that its components are *used* — the executor may read the `expo-ui` skill and still fall back to standard RN components. When the goal is to test a specific skill's component output (e.g. `@expo/ui` adoption), pair the force-invoke directive with an explicit domain hint in the prompt (e.g. "use `@expo/ui` components where they fit") so the executor is nudged toward the skill's guidance, not just exposed to it. +- "Make your changes inside ``. The project already exists with dependencies installed. Use absolute paths." +- "Before writing files, inspect the layout — `ls`, read `package.json`/`app.json` — to find the routes dir. Recent SDK templates put Expo Router routes in `src/app/`; older ones in `app/`." +- **"When done, write `/routes.json`: a JSON array of every navigable screen as `{\"path\": \"/deep/link\", \"label\": \"Human name\"}`. Use real sample values for dynamic segments (e.g. `/recipe/1`, not `/recipe/[id]`). List `/` first. This drives per-screen screenshots."** +- "Do NOT start the dev server, boot simulators, or take screenshots — the harness does that." +- Where to save a short build summary. + +After each executor, immediately parse its stream log and write: +- `skill_usage.json` — `{skills:[…], mcp_tools:[…], mcp_available:bool, forced:bool, forced_skills:[…]}`. See **Skill/MCP usage parsing**. Set `forced:true` and `forced_skills` to the case's `force_skills` when the run used the force-invoke directive — so the viewer/grader read the skills as *forced*, not as natural recall. +- `timing.json` — tokens (from `message_start`/`message_delta` events) + elapsed seconds. + +### 3. Static gate + +Write `run_static.py`. For each eval/config app, `subprocess.run(["bash","/check-static.sh", app, ""])` across a `ThreadPoolExecutor`, writing `eval-//static.json` (`{"exit_code":…, "output":…}`). `check-static.sh` runs `tsc --noEmit`, diff-aware `eslint`, and `expo export` for the platforms — a failing export short-circuits the device stage with a clean FAIL; record it and have the snapshot orchestrator skip that app. **Lint and TypeScript errors alone do not block the device stage** — `check-static.sh` returns non-zero for any of the three checks, so do not use `exit_code != 0` as the skip signal. Instead check whether `"[PASS] export"` is present in `static.json["output"]`; skip only if it is absent. + +**Auto-install missing native deps before giving up on the export.** When the export fails with "Unable to resolve module X", the executor correctly decided to use a package that isn't pre-installed in the SDK template (e.g. `react-native-maps`, `react-native-worklets`). `check-static.sh` handles this automatically: it detects "Unable to resolve module" lines, runs `bunx expo install ` for each, and retries the export once. The `run_static.py` orchestrator needs no special handling — just call `check-static.sh` as usual and read `[PASS] export` from the output. +### 4. Routes + screenshots (serial across apps) + +Write `run_snapshots.py`. Simulators/emulators are shared, so this runs **serially** (no thread pool). For each app that passed the static gate: + +1. **Resolve routes:** prefer `/routes.json` (executor-emitted); else `python3 /discover_routes.py ` (static Expo Router enumeration); else the eval case's `routes`; else `[{"path":"/"}]`. Ensure `/` is first. Build a comma-separated `route-csv` of the `path`s. **Immediately write the resolved list to `eval-//routes.json`** (the config directory, not the app directory) — the viewer's `routes_for()` reads from that path; if the file is absent it falls back to the eval-case hint routes, whose slugs may not match the actual screenshot filenames, producing "no shot" entries for every captured route. +2. **Per platform**, call the route-capture script (booted via `subprocess.run(["bash", …], env={**os.environ, "EXPO_SKILL_EVAL_RUNNER": runner})`): + + ``` + bash /snapshot-routes-.sh //outputs/ "" + ``` + + Port `8081` for iOS/web, `8082` for Android. Each script **boots Metro once**, deep-links to each route in turn, settles, and screenshots into `outputs//.png` (slug: `/`→`index`, `/recipe/1`→`recipe-1`), writing one shared `outputs//metro.log`. It frees its port on startup and tears Metro down on exit — never run `lsof`/`kill` yourself. +3. **Reclaim disk:** after all platforms for an app are captured (and before the next fixture), `subprocess.run(["bash","/clean-fixture.sh", app])`. Essential for dev-build (multi-GB per fixture); harmless for Expo Go. It keeps the app source + git so the grader's `git diff` works. + +Then generate the viewer: + +```bash +python3 /abs/path/expo-plugin-eval/scripts/generate_viewer.py /private/tmp/expo-plugin-eval- +``` + +It writes `viewer.html` one level above `iteration-N/` and opens it (`webbrowser.open`). + +### 5. Grade + +Spawn a grader subagent in the foreground per config. Its prompt must include: +- The eval prompt, `expectations`, `visual_expectations`, `routes`, `expected_skills`. +- The instructions in `agents/plugin-grader.md`. +- The inputs: every `outputs//.png`, the `outputs//metro.log`s, `routes.json`, `static.json`, and `skill_usage.json`. +- **Image cases:** also the `reference_image`, `references/design-rubric.md`, and the fixture's `git diff`. + +The grader writes `grading.json` next to the config's outputs (shape in `agents/plugin-grader.md`): an `expectations` array (case expectations + each visual_expectation, mapped to the right route, PASS/FAIL with cited evidence naming the screenshot file), and — **only for image cases or when a quality grade is requested** — `reference_match` + `quality` blocks. Usage is context, not a gate. + +## Deep-link mechanics (how each route is captured) + +The snapshot scripts boot Metro once and navigate by deep link: +- **Expo Go iOS:** `xcrun simctl openurl booted "exp://127.0.0.1:/--/"` (the standard Expo deep-link form; `exp://` openurl raises no dialog). +- **Expo Go Android:** `adb shell am start -a android.intent.action.VIEW -d "exp://127.0.0.1:/--/"`. +- **Web:** Playwright screenshots `http://localhost:/` directly. +- **dev build:** the app's `://` (scheme set by `make-fixture.sh`). **Best-effort:** the iOS Simulator can raise an "Open in ?" dialog on the first custom-scheme `openurl` (the script launches root via `simctl launch` to avoid it, but sub-routes use `openurl`, so the first sub-route may capture the dialog — re-run if so). **Expo Go is the fully-supported multi-route path; use it unless the app needs native code.** + +The first (cold) route pays the bundle wait; later routes settle for `EXPO_SKILL_EVAL_NAV_SETTLE` (default 5s) — bump it if a navigation transition isn't done when the screenshot fires. **A deep link that doesn't navigate captures the wrong screen** — the grader treats a route showing the home feed instead of its own content as a FAIL, so design prompts so each screen has a real, stable route. + +## Skill/MCP usage parsing + +From each executor's `--output-format=stream-json` log, build `skill_usage.json`: +- **skills**: collect `tool_use` blocks for the `Skill` tool and `Read` tool calls whose path ends in a `plugins/expo/skills//SKILL.md` or a `references/` file under it (the model often reads a skill before/instead of the `Skill` invocation). Record the de-duplicated skill names. +- **mcp_tools**: collect `tool_use` blocks whose name starts with `mcp__` (the Expo MCP surfaces as `mcp__expo__*`). Record the de-duplicated tool names. +- **Event structure:** `--output-format=stream-json` does NOT emit top-level `{"type":"tool_use",…}` lines. Tool uses appear as blocks inside `type=="assistant"` envelope events: `event["message"]["content"]` is a list; iterate it and look for `{"type":"tool_use","name":"Skill",…}` items. Checking `etype == "tool_use"` at the top level never matches and produces an empty skill list. The correct pattern: `for ev in log: if ev["type"]=="assistant": for block in ev["message"]["content"]: if block["type"]=="tool_use": …`. +- **mcp_available**: true if any `mcp__*` tool was called, or if the stream's init/`system` event lists the `expo` MCP server as connected; false if the server appears with an error or never connects. If indeterminate, set it to `null`. This matters because the Expo MCP is a **remote HTTP server** (`https://mcp.expo.dev/mcp`) that may require OAuth — a non-interactive `claude -p` may not authenticate it, so a build can legitimately run skills-only. The viewer surfaces this so a result isn't misread. +- **forced / forced_skills**: when the run used the **force-invoke directive** (the case had a non-empty `force_skills`), set `forced:true` and `forced_skills` to that list. The skills then appearing in `skills` reflect *instructed* invocation, not natural recall — the viewer labels the panel "forced" so a forced run isn't read as a recall win. Only `with_plugin` is ever forced. +- **routing_style**: when the run used the **routing-style directive** (the case had `routing_style: true`), set `routing_style: true` and leave `forced: false`. The skills appearing in `skills` are model-chosen, not pre-specified — the viewer labels the panel "routing-style (model-chosen)". Mutually exclusive with `forced: true`. + +**Baseline-contamination check (replaces the old manual "disable the published plugin" step).** After parsing, if any `without_plugin` config shows `expo:*` skills used, the fixture's `enabledPlugins` disable didn't match your environment's published-plugin id — the published copy leaked into the baseline. **Stop and tell the user** to set `EXPO_SKILL_EVAL_PUBLISHED_PLUGIN_ID` to the right `@` and re-run; don't report a contaminated A/B delta. This is the only residual guard now that the disable is automatic. + +## Practical notes + +- **Temp locations & permissions.** All workspaces go under `/private/tmp/expo-plugin-eval-/`. The `allowed-tools` frontmatter mirrors `expo-skill-eval`'s proven rules (path-scoped, no broad interpreters): `Bash(python3 …/expo-plugin-eval-*)` runs your orchestrators, `Bash(python3 *expo-plugin-eval/scripts/*)` runs the checked-in `discover_routes.py`/`generate_viewer.py`, `Bash(bash *expo-plugin-eval/scripts/*)` runs a single standalone script (e.g. re-running one flaky snapshot), and the scoped `tee` rule covers `python3 … 2>&1 | tee /…log`. The same caveats apply: a Bash rule is a gitignore-style glob **over the command string** (so it matches the symlinked-script path fine), `**` is matched literally (never use it), compound commands are checked per segment, and **`Write`/`Edit` rules don't suppress the prompt** — approve the workspace dir once at the start. After editing this frontmatter, **fully restart Claude Code** (not `/reload-skills`) so the rules reload. +- **Inspect outputs with tools, not shell.** Find files with **Glob** (`/private/tmp/expo-plugin-eval-/iteration-N/**/index.png`), view them with **Read** (renders PNGs — use it to confirm each route's screenshot), search with **Grep**. Never `find`/`ls`/`cat` (they prompt; `find -exec` is deliberately disallowed). +- **Skills that can't run in Expo Go** (expo-module, App Clips, brownfield native code) won't appear in a renderable app — if the plugin reaches for them for a given prompt, expect a `static-only` outcome or a dev build. Prefer prompts whose screens render in Expo Go. +- **Timing data** isn't recoverable later — capture tokens + duration into `timing.json` right after each executor. +- **First-launch dialogs:** Expo Go occasionally shows a one-time prompt on a fresh simulator; if a screenshot captures it, re-run that platform's snapshot script. + +## Viewer + +After screenshots, always generate and open the HTML viewer: + +```bash +python3 /abs/path/expo-plugin-eval/scripts/generate_viewer.py /private/tmp/expo-plugin-eval- +``` + +It writes a self-contained `viewer.html` (screenshots embedded as base64) and opens it. It renders, per iteration tab: a summary bar (per-config %, A/B delta, eval count); per eval case the prompt and (image cases) the target screenshot; and per config a **skill/MCP usage panel** (skills used with expected-coverage marks, MCP tools, MCP-reachable badge) above a **per-route screenshot gallery** (one row per route, a screenshot per platform — click to zoom), then the PASS/FAIL expectations, and (image cases) `reference_match` + the design-quality rubric. + +### Publishing the viewer (only if opted in up front) + +When the user chose "Publish a shareable Artifact", additionally run with `--artifact`: + +```bash +python3 /abs/path/expo-plugin-eval/scripts/generate_viewer.py /private/tmp/expo-plugin-eval- --artifact +``` + +It writes `viewer_artifact.html` (page-content only — no `///`, no browser open). Pass that file to the `Artifact` tool (`favicon: "📊"`). It's already self-contained (base64 images, inline CSS/JS), satisfying the Artifact CSP. + +## References + +- `references/example-apps.md` — built-in multi-screen example app prompts (the defaults). +- `references/design-rubric.md` — design-quality rubric for image-prompt / quality grading. +- `agents/plugin-grader.md` — multi-route, usage-aware grader instructions for the grader subagent. diff --git a/.claude/skills/expo-plugin-eval/agents/plugin-grader.md b/.claude/skills/expo-plugin-eval/agents/plugin-grader.md new file mode 100644 index 0000000..5081297 --- /dev/null +++ b/.claude/skills/expo-plugin-eval/agents/plugin-grader.md @@ -0,0 +1,73 @@ +# Plugin Grader Instructions + +Grading instructions for an `expo-plugin-eval` run: a whole-app build, captured +**route by route**, produced by loading the entire `expo` plugin. You receive the +eval prompt, its `expectations` / `visual_expectations` / `routes` / `expected_skills`, +and the run's outputs, and you write `grading.json` next to the outputs in the +shape defined by the skill's **Grade** step. + +Grade every expectation **PASS/FAIL on cited, concrete evidence** — never on what +the executor's transcript *claims* it built, only on the actual outputs (the +screenshots and the diff). Quote or name the evidence for each verdict. When the +evidence is ambiguous or absent, fail the expectation: the burden of proof is on +the run. + +## Inputs (per config — with_plugin, and without_plugin if present) + +- `outputs//.png` — one screenshot **per route per platform**. + Slugs: `/`→`index`, `/recipe/1`→`recipe-1`, `/settings`→`settings`. +- `outputs//metro.log` — the Metro/dev-server log for that platform (shared across routes). +- `routes.json` — the routes that were captured (executor-emitted or discovered). +- `static.json` — the static-gate result (`exit_code` + captured output). +- `skill_usage.json` — `{skills, mcp_tools, mcp_available}` parsed from the build stream. +- Image-prompt cases also get `reference_image`, `references/design-rubric.md`, and the fixture's `git diff`. + +## Process + +1. **Read every route screenshot with the Read tool.** Never grade a visual expectation from the transcript — only from the pixels. A multi-screen app is judged across all its captured routes. + +2. **Check failure signatures on every screenshot**, regardless of the listed expectations: + - A red error screen / redbox, or "Something went wrong" page + - The Expo Go home/project-list screen (the app never opened) + - A blank white/black screen (bundle loaded but nothing rendered) + - The wrong screen (a deep link that didn't navigate — e.g. `/settings` still shows the home feed) + - A system permission dialog or first-launch prompt covering the app + If any appear, every visual expectation that depends on that route fails, with the signature as evidence. A covering dialog is grounds to flag the run for re-capture (`user_notes_summary.needs_review`) rather than failing outright. + +3. **Map each `visual_expectation` to the route that should satisfy it**, then grade it on that route's screenshot(s). E.g. "Settings shows a dark-mode toggle" is graded on `outputs//settings.png`, not the home screen. If a needed route's screenshot is missing, fail the expectation and cite the gap. + +4. **Grade each expectation per platform.** If `runtime.platforms` lists ios and android, a per-route expectation must hold on **both** that route's screenshots to pass. Evidence names the file and what is visible, e.g. `with_plugin/outputs/ios/recipe-1.png: recipe detail with an ingredients list and step-by-step instructions`. + +5. **Cross-check the Metro log.** Scan for `ERROR`, `Unable to resolve`, missing-module warnings, unhandled rejections. A clean-looking screenshot with runtime errors in the log is suspect — fail expectations the errors plausibly affect and cite the line. + +6. **Static gate is upstream of visuals.** If `static.json` failed and the device stage was skipped, mark all visual expectations failed with evidence pointing at the gate, and don't speculate about what would have rendered. + +7. **Use `skill_usage.json` as context, not as a pass/fail gate.** It records which plugin skills the build actually used and whether the Expo MCP was reachable. Don't fail a build merely because an `expected_skill` wasn't triggered — grade the *output*. But when the output is weak in an area a relevant skill covers (e.g. unstyled UI while `building-native-ui` never triggered, or a wrong `@expo/ui` import while `expo-ui` wasn't used), note that link in `user_notes_summary.notes`. If `mcp_available` is false, note it (the run couldn't reach the Expo MCP) so a low score isn't misread. **If `forced` is true**, the listed skills were *instructed*, not naturally recalled — don't read their presence as a recall win, and don't note "expected skill not used" coverage gaps (they were forced); a forced run is testing skill *content*, so weak output despite a forced skill is the more interesting note. + +## Judgment calibration + +- Platform-appropriate rendering differences (status bar, fonts, safe areas, scroll-bar styling) are not failures. +- Be strict about substance: "a list of recipe cards" means several visible cards, not one placeholder row. +- When a screenshot is ambiguous (mid-animation, partially loaded), say so and fail the expectation — the harness can re-capture with a longer settle. + +## Output: `grading.json` + +```json +{ + "score": 8.5, + "max_score": 9, + "expectations": [ + {"text": "Home shows a scrollable list of recipe cards", "passed": true, "evidence": "with_plugin/outputs/ios/index.png: 6 recipe cards with images/titles in a scroll view"} + ], + "reference_match": { "score": 7, "max": 10, "evidence": "..." }, + "quality": { + "dimensions": [{"name": "Layout & hierarchy", "score": 2, "max": 3, "evidence": "..."}], + "subtotal": 17, "max": 24, "summary": "..." + }, + "user_notes_summary": {"needs_review": false, "notes": ""} +} +``` + +- Put **all** expectations (the case's `expectations` plus each `visual_expectation`) into the `expectations` array with cited evidence. `score` is the count/severity-weighted pass total; `max_score` is the number graded. +- Emit `reference_match` (0–10, generated vs the target screenshot) and `quality` (the `references/design-rubric.md` dimensions, incl. the code-quality dimension from `git diff`) **only for image-prompt cases** (`reference_image` present) or when a quality grade is explicitly requested. For image-prompt cases, take the **worst** route/platform per quality dimension and let a failure signature cap that dimension at 0. Omit both blocks for plain text-prompt runs. +- `quality.subtotal` is the sum of dimension scores; `quality.max` is the sum of their maxes (24 for the built-ins, +3 per extra `visual_expectation` graded as a quality dimension). diff --git a/.claude/skills/expo-plugin-eval/references/design-rubric.md b/.claude/skills/expo-plugin-eval/references/design-rubric.md new file mode 100644 index 0000000..142e35f --- /dev/null +++ b/.claude/skills/expo-plugin-eval/references/design-rubric.md @@ -0,0 +1,46 @@ +# Design-quality rubric + +Applied by the grader when scoring an app's visual quality — primarily for **image-prompt cases** (`reference_image`), and any time a quality grade is requested — on top of the pass/fail expectations, not instead of them. It answers "how *good* is this app?", scored from the captured screenshots and the implementation diff. Combine it with the failure-signature checks in `agents/plugin-grader.md`: a redbox, blank screen, or wrong-screen capture caps every visual dimension at 0 for that route/platform regardless of intent. + +## How to score + +- **Judge visuals only from the pixels.** Read every screenshot with the Read tool; never infer a visual score from the transcript or from what the code *intended* to render. +- **Grade per platform, then take the worst.** If the case ran ios + android, a dimension's score is the lower of the two — unless the gap is a legitimate platform difference (see *Platform conventions*). +- **For a multi-screen app, grade the app as a whole** across the captured routes — a polished home with a broken detail screen is not a polished app. Cite the weakest screens. +- Each dimension is **0–3**. The seven visual dimensions sum to a max of 21; with the code-quality dimension (also 0–3) the built-in rubric max is **24**. +- If a route/platform screenshot is **missing**, score only what's present, note the gap, and do **not** penalize for the absence. +- Cite concrete evidence per dimension, naming the file: e.g. `with_plugin/outputs/ios/settings.png: 16px gutters, consistent 12px row spacing, large-title nav bar`. + +### Score anchors (apply to every dimension) + +- **0** — broken/absent: the dimension fails outright (overlapping or clipped text, unreadable contrast, no hierarchy, content off-screen). +- **1** — below bar: present but with clear problems a designer would reject. +- **2** — solid: clean and conventional, no notable problems; what a competent dev ships. +- **3** — polished: deliberate and refined; App-Store quality for this dimension. + +## Visual dimensions + +1. **Layout & hierarchy** — clear primary/secondary structure, alignment, logical grouping, balanced composition; nothing clipped or pushed off-screen. +2. **Spacing & density** — consistent padding/margins and vertical rhythm; not cramped, not sparse; respects safe areas, the notch, and the home indicator. +3. **Typography** — readable sizes, a sensible type scale (title vs body vs caption), consistent weights, no truncation/overflow/orphaned text. +4. **Color & contrast** — coherent, limited palette; sufficient text/background contrast for legibility; sensible light/dark handling. +5. **Platform conventions** — feels native to the platform it's shown on: + - **iOS** — large titles / nav bars, grouped/inset lists, SF-style controls, standard tab bars, system spacing. + - **Android** — Material surfaces, app bar, FAB where appropriate, ripple/elevation cues, Material controls. + - **web** — sensible responsive layout, real cursor/hover affordances, no mobile-only chrome stranded on a wide canvas. + A genuine platform difference (status bar, system fonts, control styling, safe areas) is **not** a defect; shipping an iOS-only pattern verbatim on Android **is**. +6. **Polish & states** — pixel alignment, crisp icons/images at the correct density, no stray debug text or placeholder lorem, sensible empty/loading states where the screen implies them. +7. **Visual accessibility** — tap targets look ≥44pt, text isn't clipped at default scale, contrast meets a rough WCAG-AA bar, interactive elements are visually distinguishable from static content. + +## Code-quality dimension (0–3, from the diff) + +Scored from the fixture's `git diff`, not the screenshots: + +- Idiomatic Expo/RN: correct import paths and current APIs (e.g. the `@expo/ui` `Host` wrapper, Expo Router file conventions), no deprecated patterns. +- Reasonable structure: componentized where it helps, no copy-paste duplication, styles organized (not a wall of inline objects), TypeScript types where the template uses them. +- No dead code, leftover `console.log` noise, or unused scaffolding left from the template. +- Anchors: **0** broken/anti-patterns, **1** works but messy, **2** clean and idiomatic, **3** exemplary. + +## Extra criteria + +If the eval case lists `visual_expectations`, grade each as an **additional 0–3 dimension** with the same anchors, appended to the `dimensions` array and folded into `subtotal`/`max` (`max` grows by 3 per extra criterion). Cite evidence for each just like the built-ins. diff --git a/.claude/skills/expo-plugin-eval/references/example-apps.md b/.claude/skills/expo-plugin-eval/references/example-apps.md new file mode 100644 index 0000000..6819d88 --- /dev/null +++ b/.claude/skills/expo-plugin-eval/references/example-apps.md @@ -0,0 +1,77 @@ +# Built-in example apps + +The default prompts for an `expo-plugin-eval` run. Each is a **multi-screen app** +chosen to exercise navigation (so the route-by-route screenshot capture has work +to do) and a spread of the plugin's skills. Surface these in the up-front +"Prompts" `AskUserQuestion` (pre-selected), alongside the custom-prompt and +upload-a-screenshot options. Each selected example becomes one eval case. + +**The prompts are intentionally phrased at user-intent level** — what the app +does and which screens it has, *not* how to build it. Over-specified prompts +("use Expo Router file-based routing", "each screen its own route", "use a +data-fetching approach with loading/error states") hand the model the whole +implementation, so it builds the app directly and **never triggers a skill** — +which makes `with_plugin` collapse toward the baseline and the eval measure +nothing. Keep them natural. Where a skill-specific behavior is the point (Tailwind +styling, live data), name that *domain* the way a real user would, not the +mechanism. See **Phrase prompts at user-intent level** in `SKILL.md`. + +Turn an example into an eval case by copying its `prompt`. The `routes`, +`visual_expectations`, and `expected_skills` are **harness/grading metadata** — +they are NOT dictated to the executor. `routes` is only a capture hint (the +executor's emitted `routes.json` or `discover_routes.py` is authoritative, with +sample params filled in for dynamic routes); `expected_skills` is aspirational +coverage (if a skill doesn't trigger, that's a recorded finding, not a failure). + +Keep custom prompts deterministic too: an app whose screens render from local +state screenshots more reliably than one waiting on a flaky network (the weather +example is the deliberate exception). + +--- + +## 1. Recipe app (lists + detail + toggle) +**prompt:** "Build a recipe app where I can browse a feed of recipes, tap one to see its full details with the ingredients and steps, keep a list of favorites, and open a settings screen to turn on dark mode." +- **routes (capture hint):** `/` (Home feed), `/recipe/1` (Recipe detail), `/favorites` (Favorites), `/settings` (Settings) +- **visual_expectations:** + - "Home shows a vertically scrollable list of recipe cards with images and titles" + - "Recipe detail shows a single recipe with an ingredients/steps layout" + - "Settings shows a labeled dark-mode toggle control" +- **expected_skills:** `building-native-ui`, `expo-ui` + +## 2. Fitness tracker (dashboard + stats) +**prompt:** "Build a fitness tracker app. I want a dashboard showing my daily activity — steps, calories, and active minutes — with a weekly trend, a list of my workouts I can tap into to see the details, and a profile screen with my info and goals." +- **routes (capture hint):** `/` (Dashboard), `/workouts` (Workouts list), `/workout/1` (Workout detail), `/profile` (Profile) +- **visual_expectations:** + - "Dashboard shows summary stat cards and a simple chart/visualization" + - "Workouts list shows multiple workout rows" + - "Profile shows user info laid out cleanly" +- **expected_skills:** `building-native-ui`, `expo-ui` + +## 3. Notes app (list + editor) +**prompt:** "Build a notes app where I can see all my notes, open one to read and edit it, and adjust preferences like sort order and theme in settings." +- **routes (capture hint):** `/` (Notes list), `/note/1` (Note editor), `/settings` (Settings) +- **visual_expectations:** + - "Notes list shows multiple note rows with title and preview text" + - "Note editor shows an editable text area with the note content" + - "Settings shows labeled option rows" +- **expected_skills:** `building-native-ui` + +## 4. Weather app (data fetching) +**prompt:** "Build a weather app that shows the current conditions and the next few hours for my city, a 7-day forecast, a list of my saved locations, and a settings screen to switch between Celsius and Fahrenheit. Use real weather data from a free source." +- **routes (capture hint):** `/` (Current weather), `/forecast` (7-day forecast), `/locations` (Saved locations), `/settings` (Settings) +- **visual_expectations:** + - "Home shows current temperature/conditions and an hourly strip" + - "Forecast shows a multi-day list with per-day high/low" + - "No error/empty state is shown on Home once data loads" +- **expected_skills:** `native-data-fetching`, `building-native-ui` +- **note:** the only example that hits a live endpoint (the "real weather data" cue should lead it to a free API like Open-Meteo); allow extra settle for the fetch. + +## 5. Social profile (NativeWind/Tailwind styling) +**prompt:** "Build a social app with a scrollable feed of posts (avatar, name, text, like and comment counts), a profile screen for a user showing their header, bio, stats, and a grid of their posts, and a settings screen. Give it a clean, modern look using Tailwind styling." +- **routes (capture hint):** `/` (Feed), `/profile/1` (User profile), `/settings` (Settings) +- **visual_expectations:** + - "Feed shows post cards with avatar, author, and content" + - "Profile shows a header, avatar, bio, and a posts grid" + - "Styling is visibly applied (spacing, colors, rounded elements) — not unstyled defaults" +- **expected_skills:** `expo-tailwind-setup`, `building-native-ui` +- **note:** good **web** candidate — NativeWind renders on web too. The "Tailwind styling" cue is the natural user signal that should trigger `expo-tailwind-setup`; if it doesn't, that's a recorded recall finding. diff --git a/.claude/skills/expo-plugin-eval/scripts/check-static.sh b/.claude/skills/expo-plugin-eval/scripts/check-static.sh new file mode 120000 index 0000000..3c3a086 --- /dev/null +++ b/.claude/skills/expo-plugin-eval/scripts/check-static.sh @@ -0,0 +1 @@ +../../expo-skill-eval/scripts/check-static.sh \ No newline at end of file diff --git a/.claude/skills/expo-plugin-eval/scripts/clean-fixture.sh b/.claude/skills/expo-plugin-eval/scripts/clean-fixture.sh new file mode 120000 index 0000000..a6585a0 --- /dev/null +++ b/.claude/skills/expo-plugin-eval/scripts/clean-fixture.sh @@ -0,0 +1 @@ +../../../eval-shared/scripts/clean-fixture.sh \ No newline at end of file diff --git a/.claude/skills/expo-plugin-eval/scripts/discover_routes.py b/.claude/skills/expo-plugin-eval/scripts/discover_routes.py new file mode 100755 index 0000000..572e6c8 --- /dev/null +++ b/.claude/skills/expo-plugin-eval/scripts/discover_routes.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Statically enumerate an Expo Router app's navigable routes (fallback). + +Usage: discover_routes.py [--pretty] + The generated Expo app. Routes are read from app/ or src/app/. + --pretty Indent the JSON output. + +Prints a JSON array of {"path","label","file"} to stdout, e.g. + [{"path":"/","label":"Home","file":"app/(tabs)/index.tsx"}, ...] + +This is the FALLBACK for when the executor did not emit its own routes.json. +The executor manifest is always preferred — it knows real sample values for +dynamic ([id]) routes, whereas this derives them by convention. Routes are +mapped with standard Expo Router rules: + - index.* -> the directory's route ("/" at the root) + - (group)/ -> group dirs are transparent (stripped from the URL) + - [param].* -> dynamic; a sample value ("1") is substituted + - [...rest].* -> catch-all; a single sample segment is substituted + - _layout.* -> layout, not a route (skipped) + - +not-found / +html / +api / +native-intent / any "+"-prefixed file -> skipped +Only .tsx/.ts/.jsx/.js files are considered. API route handlers (+api) and +non-route files are skipped so every emitted path renders a screen. +""" + +import json +import sys +from pathlib import Path + +ROUTE_EXTS = {".tsx", ".ts", ".jsx", ".js"} +SAMPLE_PARAM = "1" + + +def find_routes_dir(project): + for candidate in (project / "app", project / "src" / "app"): + if candidate.is_dir(): + return candidate + return None + + +def is_route_file(p): + if p.suffix not in ROUTE_EXTS: + return False + name = p.name + stem = p.stem # name without the final extension + # Expo Router supports platform-suffixed files (index.ios.tsx); strip a + # trailing .ios/.android/.web/.native so they collapse onto one route. + base = stem + for plat in (".ios", ".android", ".web", ".native"): + if base.endswith(plat): + base = base[: -len(plat)] + break + if base.startswith("+"): # +not-found, +html, +api, +native-intent + return False + if base == "_layout": + return False + # +api route handlers (server endpoints) render no screen. + if base.endswith("+api"): + return False + return True + + +def stem_base(p): + """Route stem with any platform suffix and trailing extension removed.""" + base = p.stem + for plat in (".ios", ".android", ".web", ".native"): + if base.endswith(plat): + return base[: -len(plat)] + return base + + +def segment_to_url(seg): + """Map one path segment to its URL form, or None to drop it.""" + if seg.startswith("(") and seg.endswith(")"): + return None # group dir — transparent to the URL + if seg.startswith("[...") and seg.endswith("]"): + return SAMPLE_PARAM # catch-all + if seg.startswith("[") and seg.endswith("]"): + return SAMPLE_PARAM # dynamic segment + return seg + + +def humanize(route, segments): + if route == "/": + return "Home" + # Prefer the last static (non-sampled) segment for a friendly label. + for seg in reversed(segments): + if seg.startswith("[") or seg.startswith("(") or seg == SAMPLE_PARAM: + continue + return seg.replace("-", " ").replace("_", " ").strip().title() + # All-dynamic route: name it after the nearest static parent. + return route.strip("/").replace("/", " ").title() or "Home" + + +def discover(project): + routes_dir = find_routes_dir(project) + if not routes_dir: + return [] + + seen = {} + for p in sorted(routes_dir.rglob("*")): + if not p.is_file() or not is_route_file(p): + continue + rel = p.relative_to(routes_dir) + raw_segments = list(rel.parts[:-1]) + [stem_base(p)] + if raw_segments and raw_segments[-1] == "index": + raw_segments = raw_segments[:-1] # index represents its parent dir + + url_segments = [] + ok = True + for raw in raw_segments: + mapped = segment_to_url(raw) + if mapped is None: + continue # dropped group dir + url_segments.append(mapped) + + route = "/" + "/".join(url_segments) + route = "/" if route == "/" else route.rstrip("/") + if not ok: + continue + if route not in seen: + seen[route] = { + "path": route, + "label": humanize(route, raw_segments), + "file": str(p.relative_to(project)), + } + + # "/" first, then alphabetical — a stable order the snapshot loop relies on. + routes = list(seen.values()) + routes.sort(key=lambda r: (r["path"] != "/", r["path"])) + return routes + + +def main(): + args = [a for a in sys.argv[1:] if not a.startswith("--")] + pretty = "--pretty" in sys.argv + if not args: + print("usage: discover_routes.py [--pretty]", file=sys.stderr) + sys.exit(1) + project = Path(args[0]).resolve() + routes = discover(project) + print(json.dumps(routes, indent=2 if pretty else None)) + + +if __name__ == "__main__": + main() diff --git a/.claude/skills/expo-plugin-eval/scripts/generate_viewer.py b/.claude/skills/expo-plugin-eval/scripts/generate_viewer.py new file mode 100644 index 0000000..6fda962 --- /dev/null +++ b/.claude/skills/expo-plugin-eval/scripts/generate_viewer.py @@ -0,0 +1,494 @@ +#!/usr/bin/env python3 +"""Generate the self-contained HTML viewer for an expo-plugin-eval run. + +Usage: generate_viewer.py [--artifact] + Run root, e.g. /private/tmp/expo-plugin-eval-recipe-app. + Iteration dirs (iteration-*) live under it; viewer.html is + written there and opened in the browser. + --artifact Emit viewer_artifact.html (page-content only, no + ///) for the Artifact tool, and + do NOT open a browser. + +Unlike the single-skill viewer, this renders the WHOLE-PLUGIN eval: per config +(with_plugin / optionally without_plugin) it shows a per-route screenshot +gallery (one row per route, a screenshot per platform) plus a skill/MCP usage +panel parsed from the executor stream. Reads per eval: + eval-//{grading.json,static.json,skill_usage.json,routes.json, + outputs//.png} +Screenshots are embedded as base64 data: URIs so the file is self-contained. +""" + +import base64 +import html +import json +import sys +import webbrowser +from pathlib import Path + +CONFIG_ORDER = ["with_plugin", "without_plugin"] +CONFIG_LABEL = {"with_plugin": "With Plugin", "without_plugin": "Without Plugin"} + + +def esc(value): + """HTML-escape an untrusted string before interpolating it into element text. + + Route labels/paths, grading evidence/summaries, skill and MCP tool names, + and the eval prompt all come from model-generated JSON (routes.json, + grading.json, skill_usage.json, evals.json), so they must be escaped to + keep injected markup inert in the local viewer and any published Artifact. + """ + return html.escape("" if value is None else str(value)) + + +def b64_img(path): + if not path: + return None + p = Path(path) + if not p.exists() or not p.is_file(): + return None + with open(p, "rb") as f: + data = base64.b64encode(f.read()).decode() + ext = p.suffix.lower().lstrip(".") + mime = {"png": "image/png", "jpg": "image/jpeg", "jpeg": "image/jpeg"}.get(ext, "image/png") + return f"data:{mime};base64,{data}" + + +def score_color(score, max_score): + if not max_score: + return "#888" + pct = score / max_score + if pct >= 0.85: + return "#4ade80" + elif pct >= 0.65: + return "#fbbf24" + return "#f87171" + + +def load_json(path, default=None): + p = Path(path) + if not p.exists(): + return default + try: + with open(p) as f: + return json.load(f) + except Exception: + return default + + +def route_slug(path): + r = (path or "/").lstrip("/").replace("/", "-").rstrip("-") + return r or "index" + + +def routes_for(cfg_dir, eval_case): + """Routes captured for a config: executor manifest first, then the eval + case hint, then root-only.""" + manifest = load_json(cfg_dir / "routes.json") + if isinstance(manifest, dict): + manifest = manifest.get("routes") + if isinstance(manifest, list) and manifest: + return manifest + if eval_case.get("routes"): + return eval_case["routes"] + return [{"path": "/", "label": "Home"}] + + +def render_expectations(expectations): + if not expectations: + return "" + html = '
    ' + for exp in expectations: + if isinstance(exp, dict): + passed = exp.get("passed", None) + text = exp.get("text", str(exp)) + evidence = exp.get("evidence", "") + if passed is True: + badge = 'PASS' + elif passed is False: + badge = 'FAIL' + else: + badge = '?' + ev_html = f'
    {esc(evidence)}
    ' if evidence else "" + html += f'
  • {badge} {esc(text)}{ev_html}
    • ' + else: + html += f'
  • ? {esc(exp)}
    • ' + html += "
" + return html + + +def render_quality(quality): + if not quality: + return "" + dims = quality.get("dimensions", []) + subtotal = quality.get("subtotal", 0) + max_total = quality.get("max", 0) + summary = quality.get("summary", "") + html = '
' + html += f'
Design Quality: {subtotal}/{max_total}
' + for d in dims: + name = d.get("name", "") + score = d.get("score", 0) + mx = d.get("max", 3) + evidence = d.get("evidence", "") + pct = (score / mx * 100) if mx else 0 + color = score_color(score, mx) + html += f'''
+
{esc(name)} {score}/{mx}
+
+ {f'
{esc(evidence)}
' if evidence else ""} +
''' + if summary: + html += f'
{esc(summary)}
' + html += "
" + return html + + +def render_usage(usage, expected_skills): + """Skill/MCP usage panel from skill_usage.json.""" + if not usage and not expected_skills: + return "" + skills = usage.get("skills", []) if usage else [] + mcp_tools = usage.get("mcp_tools", []) if usage else [] + mcp_available = usage.get("mcp_available") if usage else None + forced = usage.get("forced") if usage else False + + html = '
' + + # MCP availability + if mcp_available is True: + html += '
MCPreachable
' + elif mcp_available is False: + html += '
MCPunavailable
' + + # Skills used (mark expected coverage). When forced, the skills reflect an + # instructed invocation, not natural recall — label the row so it isn't misread. + expected = set(expected_skills or []) + used = set(skills) + skills_key = "Skills (forced)" if forced else "Skills" + if skills or expected: + html += f'
{skills_key}' + for s in sorted(used | expected): + short = s.split(":")[-1] + cls = "chip used" if s in used or short in {u.split(":")[-1] for u in used} else "chip miss" + mark = "" if cls == "chip used" else " (not used)" + html += f'{esc(short)}{mark}' + if not (used | expected): + html += 'none' + html += '
' + + # MCP tools called + if mcp_tools: + html += '
MCP tools' + for t in sorted(set(mcp_tools)): + html += f'{esc(t.split("__")[-1])}' + html += '
' + + html += '
' + return html + + +def render_route_gallery(cfg_dir, eval_case, routes): + platforms = eval_case.get("runtime", {}).get("platforms", ["ios"]) + html = '
' + for r in routes: + path = r.get("path", "/") + label = r.get("label", path) + slug = route_slug(path) + html += '
' + html += f'
{esc(label)} {esc(path)}
' + html += '
' + for plat in platforms: + img = b64_img(cfg_dir / "outputs" / plat / f"{slug}.png") + if img: + html += ( + f'
{plat}
' + f'
' + ) + else: + html += ( + f'
{plat}
' + f'
no shot
' + ) + html += '
' + html += '
' + return html + + +def render_config_card(iter_dir, eval_id, config, eval_case): + cfg_dir = iter_dir / f"eval-{eval_id}" / config + grading = load_json(cfg_dir / "grading.json") + static_result = load_json(cfg_dir / "static.json") + usage = load_json(cfg_dir / "skill_usage.json") + routes = routes_for(cfg_dir, eval_case) + + static_pass = static_result and static_result.get("exit_code") == 0 + static_badge = 'BUILD OK' if static_pass else 'BUILD FAIL' + + score = grading.get("score") if grading else None + max_score = grading.get("max_score", 1) if grading else 1 + score_html = "" + if score is not None: + score_html = f'
{score}/{max_score}
' + + klass = "with" if config == "with_plugin" else "without" + html = f'
' + html += ( + f'
' + f'{CONFIG_LABEL.get(config, config)}{static_badge}{score_html}
' + ) + html += render_usage(usage, eval_case.get("expected_skills", [])) + html += render_route_gallery(cfg_dir, eval_case, routes) + + if grading: + html += render_expectations(grading.get("expectations", [])) + ref_match = grading.get("reference_match") + if ref_match: + rm_score = ref_match.get("score", 0) + rm_max = ref_match.get("max", 10) + html += ( + f'
Reference Match: ' + f'{rm_score}/{rm_max}' + ) + if ref_match.get("evidence"): + html += f'
{esc(ref_match["evidence"])}
' + html += '
' + html += render_quality(grading.get("quality")) + notes = grading.get("user_notes_summary", {}) + if notes and notes.get("notes"): + html += f'
Notes: {esc(notes["notes"])}
' + + html += '
' + return html + + +def present_configs(iter_dir, eval_id): + present = [] + for cfg in CONFIG_ORDER: + if (iter_dir / f"eval-{eval_id}" / cfg).is_dir(): + present.append(cfg) + return present or ["with_plugin"] + + +def render_iteration(iter_dir): + evals = load_json(iter_dir / "evals.json") + if not evals: + return f"

No evals.json found in {iter_dir}

" + + totals = {c: [0, 0] for c in CONFIG_ORDER} # [scored, max] + body = "" + total_evals = 0 + + for ev in evals: + eid = ev["id"] + total_evals += 1 + configs = present_configs(iter_dir, eid) + + for cfg in configs: + g = load_json(iter_dir / f"eval-{eid}" / cfg / "grading.json") + if g and g.get("max_score"): + totals[cfg][0] += g.get("score", 0) + totals[cfg][1] += g.get("max_score", 0) + + ref_img = b64_img(ev.get("reference_image", "")) + body += '
' + body += f'
Eval #{eid}
' + body += f'
{esc(ev.get("prompt", "")[:400])}
' + if ref_img: + body += '
Target Reference
' + body += f'
' + + cols = "1fr 1fr" if len(configs) > 1 else "1fr" + body += f'
' + for cfg in configs: + body += render_config_card(iter_dir, eid, cfg, ev) + body += '
' + + # Summary + items = "" + pcts = {} + for cfg in CONFIG_ORDER: + scored, mx = totals[cfg] + if mx: + pct = scored / mx * 100 + pcts[cfg] = pct + items += ( + f'
{CONFIG_LABEL[cfg]}: ' + f'{pct:.0f}%
' + ) + if "with_plugin" in pcts and "without_plugin" in pcts: + delta = pcts["with_plugin"] - pcts["without_plugin"] + dcolor = "#4ade80" if delta > 0 else "#f87171" if delta < 0 else "#888" + items += f'
Delta: {delta:+.0f}pp
' + items += f'
Evals: {total_evals}
' + summary = f'
{items}
' + + return summary + body + + +CSS = """\ +:root { + --ground:#090C14; --surface:#0E1320; --surface-hi:#131929; --border:#1A2238; + --text:#D9E2F5; --text-muted:#6B7A9E; --text-dim:#3A4560; --accent:#7B6FD3; + --accent-glow:rgba(123,111,211,0.12); --pass:#4DB87C; --pass-bg:rgba(77,184,124,0.10); + --fail:#E05555; --fail-bg:rgba(224,85,85,0.10); --score-with:#9B8EE8; --score-without:#5A6480; + --mcp:#5FB8D6; +} +* { box-sizing:border-box; margin:0; padding:0; } +body { background:var(--ground); color:var(--text); font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif; font-size:14px; line-height:1.5; padding:20px 24px; } +h1 { font-size:15px; font-weight:500; letter-spacing:0.06em; text-transform:uppercase; color:var(--text-muted); margin-bottom:20px; } + +.tabs { display:flex; gap:6px; margin-bottom:20px; } +.tab { background:var(--surface); border:1px solid var(--border); color:var(--text-muted); padding:5px 14px; border-radius:5px; cursor:pointer; font-size:12px; font-weight:500; } +.tab:hover { color:var(--text); border-color:#2A3558; } +.tab.active { background:var(--surface-hi); border-color:var(--accent); color:var(--text); } + +.summary-bar { display:flex; gap:0; background:var(--surface); border:1px solid var(--border); border-radius:8px; overflow:hidden; margin-bottom:20px; } +.summary-item { flex:1; padding:12px 16px; border-right:1px solid var(--border); font-size:11px; letter-spacing:0.04em; text-transform:uppercase; color:var(--text-muted); } +.summary-item:last-child { border-right:none; } +.summary-item span { display:block; font-size:22px; font-weight:700; font-variant-numeric:tabular-nums; letter-spacing:-0.02em; color:var(--text); margin-top:2px; } + +.eval-case { background:var(--surface); border:1px solid var(--border); border-radius:10px; margin-bottom:24px; overflow:hidden; } +.eval-header { padding:10px 16px; font-size:11px; font-weight:600; letter-spacing:0.07em; text-transform:uppercase; color:var(--text-muted); border-bottom:1px solid var(--border); } +.eval-prompt { padding:12px 16px; font-size:13px; color:var(--text); border-bottom:1px solid var(--border); line-height:1.55; } + +.ref-image-wrap { padding:16px; background:#07090F; border-bottom:1px solid var(--border); } +.ref-label { font-size:10px; font-weight:600; letter-spacing:0.09em; text-transform:uppercase; color:var(--text-dim); margin-bottom:8px; } +.ref-image { max-height:260px; width:auto; border-radius:10px; border:1px solid #1E2840; cursor:zoom-in; display:block; } +.ref-image.zoomed { max-height:none; cursor:zoom-out; } + +.configs-row { display:grid; } +.config-card { padding:16px; border-right:1px solid var(--border); position:relative; } +.config-card:last-child { border-right:none; } +.config-card.with::before { content:""; position:absolute; top:0; left:0; right:0; height:180px; background:radial-gradient(ellipse 80% 120px at 50% 0,var(--accent-glow),transparent 70%); pointer-events:none; } +.config-header { display:flex; align-items:center; gap:8px; margin-bottom:14px; flex-wrap:wrap; } +.config-label { font-size:10px; font-weight:700; letter-spacing:0.07em; text-transform:uppercase; padding:3px 8px; border-radius:4px; } +.config-label.with { background:rgba(123,111,211,0.15); color:var(--accent); border:1px solid rgba(123,111,211,0.25); } +.config-label.without { background:var(--surface-hi); color:var(--text-muted); border:1px solid var(--border); } +.score { font-size:22px; font-weight:700; font-variant-numeric:tabular-nums; margin-left:auto; } +.config-card.with .score { color:var(--score-with); } +.config-card.without .score { color:var(--score-without); } + +.badge { display:inline-flex; align-items:center; font-size:9px; font-weight:700; padding:2px 7px; border-radius:3px; letter-spacing:0.05em; text-transform:uppercase; } +.badge.pass { background:var(--pass-bg); color:var(--pass); border:1px solid rgba(77,184,124,0.2); } +.badge.fail { background:var(--fail-bg); color:var(--fail); border:1px solid rgba(224,85,85,0.2); } +.badge.unknown { background:var(--surface-hi); color:var(--text-dim); border:1px solid var(--border); } + +/* Usage panel */ +.usage-block { background:var(--surface-hi); border:1px solid var(--border); border-radius:6px; padding:10px 12px; margin-bottom:14px; } +.usage-row { display:flex; gap:8px; align-items:flex-start; padding:4px 0; } +.usage-key { font-size:10px; font-weight:600; letter-spacing:0.06em; text-transform:uppercase; color:var(--text-dim); min-width:74px; padding-top:2px; } +.chips { display:flex; flex-wrap:wrap; gap:5px; } +.chip { font-size:10px; padding:2px 8px; border-radius:10px; border:1px solid var(--border); } +.chip.used { background:rgba(123,111,211,0.15); color:var(--accent); border-color:rgba(123,111,211,0.3); } +.chip.miss { background:var(--fail-bg); color:var(--fail); border-color:rgba(224,85,85,0.2); } +.chip.mcp { background:rgba(95,184,214,0.12); color:var(--mcp); border-color:rgba(95,184,214,0.25); } +.chip.none { color:var(--text-dim); } + +/* Route gallery */ +.route-gallery { display:flex; flex-direction:column; gap:14px; } +.route-row { border:1px solid var(--border); border-radius:8px; padding:10px; background:#0B0F1A; } +.route-label { font-size:12px; font-weight:600; color:var(--text); margin-bottom:8px; } +.route-path { font-size:10px; font-weight:500; color:var(--text-dim); font-family:ui-monospace,monospace; margin-left:6px; } +.route-shots { display:flex; flex-wrap:wrap; gap:10px; } +.shot { display:flex; flex-direction:column; gap:4px; } +.plat-label { font-size:9px; font-weight:600; letter-spacing:0.09em; text-transform:uppercase; color:var(--text-dim); } +.screenshot { max-width:100%; max-height:320px; width:auto; border-radius:12px; border:1px solid #1A2540; cursor:zoom-in; display:block; box-shadow:0 4px 24px rgba(0,0,0,0.5); } +.screenshot.zoomed { max-height:none; cursor:zoom-out; } +.no-screenshot { color:var(--text-dim); font-size:11px; padding:16px; background:var(--surface-hi); border-radius:6px; text-align:center; border:1px dashed var(--border); } + +.exp-list { list-style:none; margin-top:12px; } +.exp-list li { padding:6px 0; border-bottom:1px solid var(--border); font-size:12px; color:var(--text-muted); display:flex; flex-wrap:wrap; gap:6px; align-items:flex-start; } +.exp-list li:last-child { border-bottom:none; } +.evidence { font-size:10px; color:var(--text-dim); width:100%; line-height:1.5; margin-top:2px; } + +.ref-match { margin-top:12px; font-size:12px; color:var(--text-muted); background:var(--surface-hi); padding:10px 12px; border-radius:6px; border-left:2px solid var(--accent); } +.ref-match b { color:var(--text); } +.quality-block { margin-top:12px; background:var(--surface-hi); border:1px solid var(--border); border-radius:6px; padding:12px; } +.quality-header { font-size:11px; font-weight:600; letter-spacing:0.05em; text-transform:uppercase; color:var(--text-muted); margin-bottom:10px; } +.quality-dim { margin-bottom:10px; } +.dim-label { font-size:10px; color:var(--text-muted); margin-bottom:4px; display:flex; justify-content:space-between; } +.dim-bar-wrap { height:3px; background:var(--border); border-radius:2px; overflow:hidden; } +.dim-bar { height:100%; border-radius:2px; } +.dim-evidence { font-size:10px; color:var(--text-dim); margin-top:3px; line-height:1.45; } +.quality-summary { font-size:11px; color:var(--text-dim); margin-top:8px; padding-top:8px; border-top:1px solid var(--border); } +.reviewer-notes { margin-top:10px; font-size:11px; color:var(--text-dim); background:var(--ground); padding:8px 10px; border-radius:4px; } +""" + +JS = """\ +function showTab(i) { + var panels = document.querySelectorAll('.panel'); + var tabs = document.querySelectorAll('.tab'); + panels.forEach(function(p, idx) { p.style.display = idx === i ? 'block' : 'none'; }); + tabs.forEach(function(t, idx) { t.classList.toggle('active', idx === i); }); + try { localStorage.setItem('expo-plugin-eval-tab', i); } catch(e) {} +} +(function() { + var panels = document.querySelectorAll('.panel'); + try { + var saved = parseInt(localStorage.getItem('expo-plugin-eval-tab') || '0', 10); + if (saved > 0 && saved < panels.length) showTab(saved); + } catch(e) {} +})(); +""" + + +def build_page(ws, title, artifact=False): + iterations = sorted(p for p in ws.glob("iteration-*") if p.is_dir()) + if not iterations: + return f"

No iteration directories found under {ws}.

" + + tabs_html = "" + panels_html = "" + for i, it in enumerate(iterations): + active = "active" if i == 0 else "" + tabs_html += f'' + panels_html += f'
{render_iteration(it)}
' + + safe_title = esc(title) + body = f'

{safe_title}

\n
{tabs_html}
\n{panels_html}' + if artifact: + return f'{safe_title}\n\n{body}\n' + return f''' + + + + +{safe_title} + + + +{body} + + +''' + + +def main(): + args = [a for a in sys.argv[1:] if a != "--artifact"] + artifact = "--artifact" in sys.argv + if not args: + print("usage: generate_viewer.py [--artifact]", file=sys.stderr) + sys.exit(1) + + ws = Path(args[0]).resolve() + name = ws.name.replace("expo-plugin-eval-", "") + title = f"{name} — Expo Plugin Eval" + html = build_page(ws, title, artifact=artifact) + + if artifact: + out = ws / "viewer_artifact.html" + out.write_text(html) + print(f"Artifact viewer written: {out}") + else: + out = ws / "viewer.html" + out.write_text(html) + print(f"Viewer written: {out}") + webbrowser.open("file://" + str(out)) + + +if __name__ == "__main__": + main() diff --git a/.claude/skills/expo-plugin-eval/scripts/latest-sdk.sh b/.claude/skills/expo-plugin-eval/scripts/latest-sdk.sh new file mode 120000 index 0000000..931dd7f --- /dev/null +++ b/.claude/skills/expo-plugin-eval/scripts/latest-sdk.sh @@ -0,0 +1 @@ +../../expo-skill-eval/scripts/latest-sdk.sh \ No newline at end of file diff --git a/.claude/skills/expo-plugin-eval/scripts/make-fixture.sh b/.claude/skills/expo-plugin-eval/scripts/make-fixture.sh new file mode 120000 index 0000000..68cf95c --- /dev/null +++ b/.claude/skills/expo-plugin-eval/scripts/make-fixture.sh @@ -0,0 +1 @@ +../../../eval-shared/scripts/make-fixture.sh \ No newline at end of file diff --git a/.claude/skills/expo-plugin-eval/scripts/make-workspace.sh b/.claude/skills/expo-plugin-eval/scripts/make-workspace.sh new file mode 120000 index 0000000..bc03594 --- /dev/null +++ b/.claude/skills/expo-plugin-eval/scripts/make-workspace.sh @@ -0,0 +1 @@ +../../expo-skill-eval/scripts/make-workspace.sh \ No newline at end of file diff --git a/.claude/skills/expo-plugin-eval/scripts/snapshot-routes-android.sh b/.claude/skills/expo-plugin-eval/scripts/snapshot-routes-android.sh new file mode 100755 index 0000000..eaea042 --- /dev/null +++ b/.claude/skills/expo-plugin-eval/scripts/snapshot-routes-android.sh @@ -0,0 +1,207 @@ +#!/usr/bin/env bash +# Run an Expo app on an Android emulator and screenshot EVERY route by +# deep-linking to each one in turn. Metro is booted ONCE and reused. +# +# Usage: snapshot-routes-android.sh [port] [routes-csv] [settle] +# out-dir Directory for per-route screenshots: /.png. +# The Metro log is written to /metro.log. +# routes-csv Comma-separated route paths, e.g. "/,/recipe/1,/settings" +# (default "/"; list "/" first). +# settle Seconds after the FIRST (cold) route paints (default 8). +# +# Env: EXPO_SKILL_EVAL_RUNNER = expo-go (default) | dev-build +# EXPO_SKILL_EVAL_NAV_SETTLE = settle after each later route (default 5) +set -uo pipefail + +PROJECT_PATH="${1:?usage: snapshot-routes-android.sh [port] [routes-csv] [settle]}" +OUT_DIR="${2:?usage: snapshot-routes-android.sh [port] [routes-csv] [settle]}" +PORT="${3:-8082}" +ROUTES_CSV="${4:-/}" +SETTLE="${5:-8}" +NAV_SETTLE="${EXPO_SKILL_EVAL_NAV_SETTLE:-5}" +RUNNER="${EXPO_SKILL_EVAL_RUNNER:-expo-go}" + +log_step() { echo "[$(date '+%H:%M:%S')] [snapshot-routes-android] STEP: $*" >&2; } +EMULATOR_PID="" +emulator_alive() { [[ -n "$EMULATOR_PID" ]] && kill -0 "$EMULATOR_PID" 2>/dev/null; } + +# See snapshot-android.sh: host GPU (Metal) by default; switch to "guest" if the +# emulator self-aborts under load. Avoid swiftshader_indirect (hangs on arm64). +GPU_MODE="host" + +if [[ "$RUNNER" == "dev-build" ]]; then + BUNDLE_TIMEOUT="${EXPO_SKILL_EVAL_BUNDLE_TIMEOUT:-900}" +else + BUNDLE_TIMEOUT="${EXPO_SKILL_EVAL_BUNDLE_TIMEOUT:-240}" +fi +LOG="$OUT_DIR/metro.log" + +mkdir -p "$OUT_DIR" + +slug() { + local r="${1#/}" + r="${r//\//-}" + r="${r%-}" + [[ -z "$r" ]] && r="index" + echo "$r" +} + +SCHEME="$(node -e "try{var a=require('$PROJECT_PATH/app.json');console.log((a.expo&&a.expo.scheme)||'exposkilleval')}catch(e){console.log('exposkilleval')}" 2>/dev/null || echo 'exposkilleval')" + +deeplink() { + local route="$1" + if [[ "$RUNNER" == "dev-build" ]]; then + echo "${SCHEME}://${route#/}" + else + echo "exp://127.0.0.1:$PORT/--/${route#/}" + fi +} + +lsof -ti "tcp:$PORT" -sTCP:LISTEN 2>/dev/null | xargs kill -9 2>/dev/null || true + +find_sdk_tool() { + local tool="$1" subdir="$2" + if command -v "$tool" >/dev/null; then command -v "$tool"; return; fi + local sdk + for sdk in "${ANDROID_HOME:-}" "${ANDROID_SDK_ROOT:-}" "$HOME/Library/Android/sdk" "$HOME/Android/Sdk"; do + if [[ -n "$sdk" && -x "$sdk/$subdir/$tool" ]]; then echo "$sdk/$subdir/$tool"; return; fi + done + return 1 +} + +ADB="$(find_sdk_tool adb platform-tools)" || { echo "error: adb not found (set ANDROID_HOME)" >&2; exit 1; } + +# Recycle a wedged/offline emulator before the boot check (see snapshot-android.sh). +STALE="$("$ADB" devices | awk 'NR>1 && $1 ~ /^emulator-/ && $2!="device" {print $1}')" +if [[ -n "$STALE" ]]; then + echo "recycling wedged emulator(s): $STALE" >&2 + for serial in $STALE; do "$ADB" -s "$serial" emu kill 2>/dev/null || true; done + sleep 2 + if "$ADB" devices | awk 'NR>1 && $1 ~ /^emulator-/ && $2!="device"' | grep -q .; then + pkill -9 -f qemu-system-aarch64 2>/dev/null || true + "$ADB" kill-server 2>/dev/null || true; "$ADB" start-server 2>/dev/null || true; sleep 2 + fi +fi + +CURRENT_GPU="$(ps aux | grep qemu-system-aarch64 | grep -v grep | grep -o -- '-gpu [a-z_]*' | awk '{print $2}' | head -1)" +if [[ -n "$CURRENT_GPU" && "$CURRENT_GPU" != "$GPU_MODE" ]]; then + echo "GPU mismatch: running=$CURRENT_GPU required=$GPU_MODE — recycling emulator" >&2 + "$ADB" emu kill 2>/dev/null || true; sleep 2 + pkill -9 -f qemu-system-aarch64 2>/dev/null || true + "$ADB" kill-server 2>/dev/null || true; "$ADB" start-server 2>/dev/null || true; sleep 2 +fi + +if ! "$ADB" devices | awk 'NR>1 && $2=="device"' | grep -q .; then + EMULATOR="$(find_sdk_tool emulator emulator)" || { echo "error: no device attached and emulator binary not found" >&2; exit 1; } + AVD="$("$EMULATOR" -list-avds | head -1)" + if [[ -z "$AVD" ]]; then echo "error: no AVDs configured" >&2; exit 1; fi + log_step "resetting adb server before cold boot" + "$ADB" kill-server 2>/dev/null || true; "$ADB" start-server 2>/dev/null || true; sleep 1 + log_step "booting emulator avd=$AVD gpu=$GPU_MODE" + nohup "$EMULATOR" -avd "$AVD" -no-boot-anim -gpu "$GPU_MODE" -no-snapshot /dev/null 2>&1 & + disown + EMULATOR_PID="$(pgrep -f "emulator.*-avd $AVD" | head -1)" + "$ADB" wait-for-device + until [[ "$("$ADB" shell getprop sys.boot_completed 2>/dev/null | tr -d '\r')" == "1" ]]; do + if [[ -n "$EMULATOR_PID" ]] && ! kill -0 "$EMULATOR_PID" 2>/dev/null; then + echo "error: emulator pid=$EMULATOR_PID crashed during boot" >&2; exit 1 + fi + sleep 2 + done + log_step "emulator fully booted" +else + EMULATOR_PID="$(pgrep -f qemu-system-aarch64 | head -1)" + log_step "reusing running emulator pid=${EMULATOR_PID:-unknown}" +fi + +if [[ "$RUNNER" == "dev-build" ]]; then + (cd "$PROJECT_PATH" && exec env -u CI REACT_NATIVE_PACKAGER_HOSTNAME=127.0.0.1 bunx expo run:android --port "$PORT") "$LOG" 2>&1 & +else + (cd "$PROJECT_PATH" && exec env -u CI REACT_NATIVE_PACKAGER_HOSTNAME=127.0.0.1 bunx expo start --port "$PORT" --android) "$LOG" 2>&1 & +fi +METRO_PID=$! +cleanup() { + kill "$METRO_PID" 2>/dev/null + lsof -ti "tcp:$PORT" -sTCP:LISTEN 2>/dev/null | xargs kill -9 2>/dev/null || true + wait "$METRO_PID" 2>/dev/null +} +trap cleanup EXIT + +APP_PKG="$(node -e "try{var a=require('$PROJECT_PATH/app.json');console.log((a.expo&&a.expo.android&&a.expo.android.package)||'com.exposkilleval.fixture')}catch(e){console.log('com.exposkilleval.fixture')}" 2>/dev/null || echo 'com.exposkilleval.fixture')" + +DEADLINE=$((SECONDS + BUNDLE_TIMEOUT)) +STATUS=0 + +log_step "waiting for Metro on port=$PORT" +until curl -sf "http://localhost:$PORT/status" >/dev/null; do + if ! kill -0 "$METRO_PID" 2>/dev/null; then echo "error: Metro exited before coming up" >&2; tail -40 "$LOG" >&2; exit 1; fi + if emulator_alive; then : ; elif [[ -n "$EMULATOR_PID" ]]; then echo "error: emulator crashed while waiting for Metro" >&2; exit 1; fi + if ((SECONDS > DEADLINE)); then echo "error: timed out waiting for Metro" >&2; tail -40 "$LOG" >&2; exit 1; fi + sleep 2 +done + +if [[ "$RUNNER" != "dev-build" ]]; then + log_step "waiting for Expo Go install on device" + until "$ADB" shell pm list packages 2>/dev/null | grep -q host.exp.exponent; do + if ((SECONDS > DEADLINE)); then echo "error: Expo Go was not installed" >&2; tail -40 "$LOG" >&2; exit 1; fi + sleep 2 + done +fi + +log_step "setting up adb reverse tunnel port=$PORT" +"$ADB" reverse "tcp:$PORT" "tcp:$PORT" + +# Stop any stale experience/app so the first deep link starts clean. +if [[ "$RUNNER" == "dev-build" ]]; then + "$ADB" shell am force-stop "$APP_PKG" 2>/dev/null || true +else + "$ADB" shell am force-stop host.exp.exponent 2>/dev/null || true +fi +sleep 2 + +# Routes come from a model-emitted routes.json. `adb shell am start -d "$URL"` +# forwards its args to the device's shell, which re-parses them, so a route +# containing shell metacharacters would be interpreted guest-side. Real Expo +# Router paths only use these characters — reject anything else. +SAFE_ROUTE_RE='^/?[A-Za-z0-9/_.-]*$' +IFS=',' read -ra ROUTES <<<"$ROUTES_CSV" +FIRST=1 +for route in "${ROUTES[@]}"; do + [[ -z "$route" ]] && continue + if [[ ! "$route" =~ $SAFE_ROUTE_RE ]]; then + echo "skipping route with unsafe characters: $route" >&2 + continue + fi + SLUG="$(slug "$route")" + OUT="$OUT_DIR/$SLUG.png" + URL="$(deeplink "$route")" + log_step "route $route -> $SLUG.png ($URL)" + + if [[ "$RUNNER" == "dev-build" && "$FIRST" == "1" && "$route" == "/" ]]; then + "$ADB" shell am start -W -n "${APP_PKG}/.MainActivity" >/dev/null 2>&1 || true + else + "$ADB" shell am start -W -a android.intent.action.VIEW -d "$URL" >/dev/null 2>&1 || true + fi + + if [[ "$FIRST" == "1" ]]; then + until grep -q 'Bundled' "$LOG"; do + if ! kill -0 "$METRO_PID" 2>/dev/null; then echo "error: Metro exited before bundling" >&2; tail -40 "$LOG" >&2; STATUS=1; break; fi + if emulator_alive; then : ; elif [[ -n "$EMULATOR_PID" ]]; then echo "error: emulator crashed while bundling" >&2; STATUS=1; break; fi + if ((SECONDS > DEADLINE)); then echo "error: timed out waiting for bundle" >&2; tail -40 "$LOG" >&2; STATUS=1; break; fi + sleep 2 + done + sleep "$SETTLE" + FIRST=0 + else + sleep "$NAV_SETTLE" + fi + + "$ADB" exec-out screencap -p >"$OUT" || STATUS=1 + if [[ -s "$OUT" ]]; then + sips -Z "${EXPO_SKILL_EVAL_MAX_DIM:-600}" "$OUT" >/dev/null 2>&1 || true + fi + log_step "screenshot: $OUT" +done + +echo "metro log: $LOG" +exit "$STATUS" diff --git a/.claude/skills/expo-plugin-eval/scripts/snapshot-routes-ios.sh b/.claude/skills/expo-plugin-eval/scripts/snapshot-routes-ios.sh new file mode 100755 index 0000000..6c762fd --- /dev/null +++ b/.claude/skills/expo-plugin-eval/scripts/snapshot-routes-ios.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# Run an Expo app on an iOS simulator and screenshot EVERY route by deep-linking +# to each one in turn. Metro is booted ONCE and reused across all routes. +# +# Usage: snapshot-routes-ios.sh [port] [routes-csv] [settle] +# out-dir Directory for per-route screenshots: /.png. +# The Metro log is written to /metro.log. +# routes-csv Comma-separated route paths, e.g. "/,/recipe/1,/settings". +# Defaults to "/" (root only). "/" should be listed first. +# settle Seconds to wait after the FIRST (cold) route paints (default 8). +# +# Env: +# EXPO_SKILL_EVAL_RUNNER = expo-go (default) | dev-build +# EXPO_SKILL_EVAL_NAV_SETTLE = seconds to settle after each subsequent route +# navigation (default 5 — the bundle is warm, only a transition is needed). +# +# Deep links: expo-go uses exp://127.0.0.1:/--/; dev-build uses the +# app's ://. On the iOS Simulator a custom-scheme openurl can +# raise an "Open in ?" dialog on first use, so multi-route capture under +# dev-build is best-effort; Expo Go is the fully-supported path. +set -uo pipefail + +PROJECT_PATH="${1:?usage: snapshot-routes-ios.sh [port] [routes-csv] [settle]}" +OUT_DIR="${2:?usage: snapshot-routes-ios.sh [port] [routes-csv] [settle]}" +PORT="${3:-8081}" +ROUTES_CSV="${4:-/}" +SETTLE="${5:-8}" +NAV_SETTLE="${EXPO_SKILL_EVAL_NAV_SETTLE:-5}" +RUNNER="${EXPO_SKILL_EVAL_RUNNER:-expo-go}" +if [[ "$RUNNER" == "dev-build" ]]; then + BUNDLE_TIMEOUT="${EXPO_SKILL_EVAL_BUNDLE_TIMEOUT:-900}" +else + BUNDLE_TIMEOUT="${EXPO_SKILL_EVAL_BUNDLE_TIMEOUT:-240}" +fi +LOG="$OUT_DIR/metro.log" + +mkdir -p "$OUT_DIR" + +# slug: "/" -> index, "/recipe/1" -> recipe-1, "/settings" -> settings +slug() { + local r="${1#/}" + r="${r//\//-}" + r="${r%-}" + [[ -z "$r" ]] && r="index" + echo "$r" +} + +SCHEME="$(node -e "try{var a=require('$PROJECT_PATH/app.json');console.log((a.expo&&a.expo.scheme)||'exposkilleval')}catch(e){console.log('exposkilleval')}" 2>/dev/null || echo 'exposkilleval')" +BUNDLE_ID="$(node -e "try{var a=require('$PROJECT_PATH/app.json');console.log((a.expo&&a.expo.ios&&a.expo.ios.bundleIdentifier)||'com.exposkilleval.fixture')}catch(e){console.log('com.exposkilleval.fixture')}" 2>/dev/null || echo 'com.exposkilleval.fixture')" + +deeplink() { + local route="$1" + if [[ "$RUNNER" == "dev-build" ]]; then + echo "${SCHEME}://${route#/}" + else + echo "exp://127.0.0.1:$PORT/--/${route#/}" + fi +} + +# Free the port up front so `expo start` binds the port we expect. +lsof -ti "tcp:$PORT" -sTCP:LISTEN 2>/dev/null | xargs kill -9 2>/dev/null || true + +if ! xcrun simctl list devices booted | grep -q '(Booted)'; then + UDID="$(xcrun simctl list devices available | grep 'iPhone' | tail -1 | grep -oE '[0-9A-F-]{36}')" + if [[ -z "$UDID" ]]; then + echo "error: no available iPhone simulator found" >&2 + exit 1 + fi + echo "booting simulator $UDID..." >&2 + xcrun simctl boot "$UDID" + open -a Simulator + xcrun simctl bootstatus "$UDID" +fi + +if [[ "$RUNNER" == "dev-build" ]]; then + (cd "$PROJECT_PATH" && exec env -u CI bunx expo run:ios --port "$PORT") "$LOG" 2>&1 & +else + (cd "$PROJECT_PATH" && exec env -u CI REACT_NATIVE_PACKAGER_HOSTNAME=127.0.0.1 bunx expo start --port "$PORT" --ios) "$LOG" 2>&1 & +fi +METRO_PID=$! +cleanup() { + kill "$METRO_PID" 2>/dev/null + lsof -ti "tcp:$PORT" -sTCP:LISTEN 2>/dev/null | xargs kill -9 2>/dev/null || true + wait "$METRO_PID" 2>/dev/null +} +trap cleanup EXIT + +DEADLINE=$((SECONDS + BUNDLE_TIMEOUT)) +STATUS=0 + +until curl -sf "http://localhost:$PORT/status" >/dev/null; do + if ! kill -0 "$METRO_PID" 2>/dev/null || ((SECONDS > DEADLINE)); then + echo "error: Metro did not come up. Log tail:" >&2 + tail -40 "$LOG" >&2 + exit 1 + fi + sleep 2 +done + +# expo-go: wait for Expo Go to install, then start from a clean slate. +if [[ "$RUNNER" != "dev-build" ]]; then + until xcrun simctl get_app_container booted host.exp.Exponent >/dev/null 2>&1; do + if ((SECONDS > DEADLINE)); then + echo "error: Expo Go was not installed on the simulator. Log tail:" >&2 + tail -40 "$LOG" >&2 + exit 1 + fi + sleep 2 + done + xcrun simctl terminate booted host.exp.Exponent 2>/dev/null || true + sleep 2 +fi + +# Routes come from a model-emitted routes.json. Keep the deep link well-formed +# and consistent with the Android guard — real Expo Router paths only use these +# characters; reject anything else before building the URL. +SAFE_ROUTE_RE='^/?[A-Za-z0-9/_.-]*$' +IFS=',' read -ra ROUTES <<<"$ROUTES_CSV" +FIRST=1 +for route in "${ROUTES[@]}"; do + [[ -z "$route" ]] && continue + if [[ ! "$route" =~ $SAFE_ROUTE_RE ]]; then + echo "skipping route with unsafe characters: $route" >&2 + continue + fi + SLUG="$(slug "$route")" + OUT="$OUT_DIR/$SLUG.png" + URL="$(deeplink "$route")" + echo "[route] $route -> $SLUG.png ($URL)" >&2 + + if [[ "$RUNNER" == "dev-build" && "$FIRST" == "1" && "$route" == "/" ]]; then + # Relaunch the dev client at root via simctl launch (no "Open in?" dialog). + xcrun simctl launch --terminate-running-process booted "$BUNDLE_ID" >/dev/null 2>&1 || true + else + xcrun simctl openurl booted "$URL" >/dev/null 2>&1 || true + fi + + if [[ "$FIRST" == "1" ]]; then + # First route pays the cold-bundle wait. + until grep -q 'Bundled' "$LOG"; do + if ! kill -0 "$METRO_PID" 2>/dev/null; then + echo "error: Metro exited before bundling. Log tail:" >&2 + tail -40 "$LOG" >&2 + STATUS=1 + break + fi + if ((SECONDS > DEADLINE)); then + echo "error: timed out after ${BUNDLE_TIMEOUT}s waiting for bundle. Log tail:" >&2 + tail -40 "$LOG" >&2 + STATUS=1 + break + fi + sleep 2 + done + sleep "$SETTLE" + FIRST=0 + else + sleep "$NAV_SETTLE" + fi + + xcrun simctl io booted screenshot "$OUT" || STATUS=1 + if [[ -f "$OUT" ]]; then + sips -Z "${EXPO_SKILL_EVAL_MAX_DIM:-600}" "$OUT" >/dev/null 2>&1 || true + fi + echo "screenshot: $OUT" >&2 +done + +echo "metro log: $LOG" +exit "$STATUS" diff --git a/.claude/skills/expo-plugin-eval/scripts/snapshot-routes-web.sh b/.claude/skills/expo-plugin-eval/scripts/snapshot-routes-web.sh new file mode 100755 index 0000000..b7c2b9b --- /dev/null +++ b/.claude/skills/expo-plugin-eval/scripts/snapshot-routes-web.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Run an Expo app's web target and screenshot EVERY route. The dev server is +# started ONCE; each route is just a URL (http://localhost:/), so +# Playwright navigates and screenshots each in turn. +# +# Usage: snapshot-routes-web.sh [port] [routes-csv] [settle] +# out-dir /.png per route; Metro log at /metro.log. +# routes-csv Comma-separated routes, e.g. "/,/recipe/1,/settings" (default "/"). +# settle Seconds to wait for each route to paint (default 8). +set -uo pipefail + +PROJECT_PATH="${1:?usage: snapshot-routes-web.sh [port] [routes-csv] [settle]}" +OUT_DIR="${2:?usage: snapshot-routes-web.sh [port] [routes-csv] [settle]}" +PORT="${3:-8081}" +ROUTES_CSV="${4:-/}" +SETTLE="${5:-8}" +SERVER_TIMEOUT="${EXPO_SKILL_EVAL_BUNDLE_TIMEOUT:-240}" +LOG="$OUT_DIR/metro.log" + +mkdir -p "$OUT_DIR" + +slug() { + local r="${1#/}" + r="${r//\//-}" + r="${r%-}" + [[ -z "$r" ]] && r="index" + echo "$r" +} + +lsof -ti "tcp:$PORT" -sTCP:LISTEN 2>/dev/null | xargs kill -9 2>/dev/null || true + +bunx playwright install chromium >/dev/null 2>&1 || true + +(cd "$PROJECT_PATH" && exec env -u CI bunx expo start --port "$PORT" --web) "$LOG" 2>&1 & +METRO_PID=$! +cleanup() { + kill "$METRO_PID" 2>/dev/null + lsof -ti "tcp:$PORT" -sTCP:LISTEN 2>/dev/null | xargs kill -9 2>/dev/null || true + wait "$METRO_PID" 2>/dev/null +} +trap cleanup EXIT + +DEADLINE=$((SECONDS + SERVER_TIMEOUT)) +until curl -sf "http://localhost:$PORT" >/dev/null; do + if ! kill -0 "$METRO_PID" 2>/dev/null || ((SECONDS > DEADLINE)); then + echo "error: web dev server did not come up. Log tail:" >&2 + tail -40 "$LOG" >&2 + exit 1 + fi + sleep 2 +done + +# Routes come from a model-emitted routes.json. Keep the URL well-formed and +# consistent with the Android guard — real Expo Router paths only use these +# characters; reject anything else before building the URL. +SAFE_ROUTE_RE='^/?[A-Za-z0-9/_.-]*$' +IFS=',' read -ra ROUTES <<<"$ROUTES_CSV" +STATUS=0 +for route in "${ROUTES[@]}"; do + [[ -z "$route" ]] && continue + if [[ ! "$route" =~ $SAFE_ROUTE_RE ]]; then + echo "skipping route with unsafe characters: $route" >&2 + continue + fi + SLUG="$(slug "$route")" + OUT="$OUT_DIR/$SLUG.png" + URL="http://localhost:$PORT/${route#/}" + echo "[route] $route -> $SLUG.png ($URL)" >&2 + bunx playwright screenshot \ + --browser chromium \ + --viewport-size "390,844" \ + --wait-for-timeout "$((SETTLE * 1000))" \ + "$URL" "$OUT" || STATUS=1 + if [[ -f "$OUT" ]]; then + sips -Z "${EXPO_SKILL_EVAL_MAX_DIM:-600}" "$OUT" >/dev/null 2>&1 || true + fi +done + +echo "metro log: $LOG" +exit "$STATUS" diff --git a/.claude/skills/expo-skill-eval/scripts/check-static.sh b/.claude/skills/expo-skill-eval/scripts/check-static.sh index bd087ab..31c8a6b 100755 --- a/.claude/skills/expo-skill-eval/scripts/check-static.sh +++ b/.claude/skills/expo-skill-eval/scripts/check-static.sh @@ -59,6 +59,35 @@ IFS=',' read -ra PLATFORM_LIST <<<"$PLATFORMS" for p in "${PLATFORM_LIST[@]}"; do EXPORT_ARGS+=(--platform "$p") done -run_gate export env CI=1 bunx expo "${EXPORT_ARGS[@]}" + +# Run export; on failure, auto-install any unresolved native packages and retry once. +run_export() { + rm -rf "$LOG_DIR/export" + env CI=1 bunx expo "${EXPORT_ARGS[@]}" >"$LOG_DIR/export.log" 2>&1 +} + +EXPORT_EXIT=0 +run_export || EXPORT_EXIT=$? + +if [[ $EXPORT_EXIT -ne 0 ]]; then + # Extract npm package names from "Unable to resolve module " lines. + MISSING=$(grep -oE "Unable to resolve module [^[:space:]]+" "$LOG_DIR/export.log" \ + | sed 's/Unable to resolve module //' | sed 's|/.*||' | sort -u || true) + if [[ -n "$MISSING" ]]; then + echo " [auto-install] missing packages: $MISSING" + # shellcheck disable=SC2086 + bunx expo install $MISSING >/dev/null 2>&1 || true + EXPORT_EXIT=0 + run_export || EXPORT_EXIT=$? # one retry + fi +fi + +if [[ $EXPORT_EXIT -eq 0 ]]; then + echo "[PASS] export" +else + echo "[FAIL] export (log: $PROJECT_PATH/$LOG_DIR/export.log)" + tail -5 "$LOG_DIR/export.log" | sed 's/^/ /' + FAILED=1 +fi exit "$FAILED" diff --git a/.claude/skills/expo-skill-eval/scripts/clean-fixture.sh b/.claude/skills/expo-skill-eval/scripts/clean-fixture.sh deleted file mode 100755 index 0bfb962..0000000 --- a/.claude/skills/expo-skill-eval/scripts/clean-fixture.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash -# Reclaim disk after a fixture's screenshots are captured. Essential for -# dev-build runs, where each `expo run:` leaves multi-GB native build -# output (iOS Pods + DerivedData, Android Gradle build). Removes the heavy, -# regenerable artifacts and keeps the app source + git history, so the grader's -# `git diff` still works. -# -# Usage: clean-fixture.sh -# -# Env: EXPO_SKILL_EVAL_KEEP_DERIVEDDATA=1 skips the iOS DerivedData sweep (set it -# only if you keep a real Xcode project literally named "fixture"). -set -uo pipefail - -APP="${1:?usage: clean-fixture.sh }" - -# Safety guard: only ever clean inside an expo-skill-eval workspace. -case "$APP" in - *expo-skill-eval-*) : ;; - *) echo "clean-fixture: refusing to clean '$APP' (not an expo-skill-eval fixture)" >&2; exit 1 ;; -esac -[[ -d "$APP" ]] || { echo "clean-fixture: $APP not found, skipping"; exit 0; } - -# Stop any Metro / Expo dev server that may still be running for this fixture. -# The snapshot scripts' EXIT trap normally handles this, but cleaning up here -# catches leftover processes from crashes or interrupted runs. -# Ports 8081 (iOS) and 8082 (Android) are the two ports the eval harness -# reserves — freeing them here is safe because clean-fixture.sh only runs after -# all screenshots for this fixture have been captured. -# -# `-sTCP:LISTEN` is REQUIRED, not a refinement: without it, `lsof -ti tcp:8082` -# also matches the *established* connection the adb daemon holds from the -# `adb reverse tcp:8082` tunnel, and SIGKILLing the adb daemon crashes the -# emulator with a gfxstream gRPC SIGABRT (std::bad_function_call). Keep the flag -# so only the Metro listener is killed. -lsof -ti tcp:8081 -sTCP:LISTEN 2>/dev/null | xargs kill -9 2>/dev/null || true -lsof -ti tcp:8082 -sTCP:LISTEN 2>/dev/null | xargs kill -9 2>/dev/null || true - -# Per-fixture heavy dirs — all gitignored / regenerable (node_modules, the -# prebuilt native projects incl. iOS Pods and Android Gradle output, caches). -rm -rf \ - "$APP/node_modules" \ - "$APP/ios" \ - "$APP/android" \ - "$APP/.expo" \ - "$APP/dist" \ - "$APP/web-build" 2>/dev/null || true - -# iOS DerivedData for fixture builds. create-expo-app names the project -# "fixture", so its build output lives under DerivedData/fixture-. This is -# a cache (worst case a rebuild), so it's safe to drop between fixtures. -if [[ "${EXPO_SKILL_EVAL_KEEP_DERIVEDDATA:-0}" != "1" ]]; then - DD="$HOME/Library/Developer/Xcode/DerivedData" - [[ -d "$DD" ]] && rm -rf "$DD"/fixture-* 2>/dev/null || true -fi - -echo "cleaned fixture: $APP" diff --git a/.claude/skills/expo-skill-eval/scripts/clean-fixture.sh b/.claude/skills/expo-skill-eval/scripts/clean-fixture.sh new file mode 120000 index 0000000..a6585a0 --- /dev/null +++ b/.claude/skills/expo-skill-eval/scripts/clean-fixture.sh @@ -0,0 +1 @@ +../../../eval-shared/scripts/clean-fixture.sh \ No newline at end of file diff --git a/.claude/skills/expo-skill-eval/scripts/generate_viewer.py b/.claude/skills/expo-skill-eval/scripts/generate_viewer.py index a7b6370..b42d033 100644 --- a/.claude/skills/expo-skill-eval/scripts/generate_viewer.py +++ b/.claude/skills/expo-skill-eval/scripts/generate_viewer.py @@ -18,12 +18,24 @@ """ import base64 +import html import json import sys import webbrowser from pathlib import Path +def esc(value): + """HTML-escape an untrusted string before interpolating it into element text. + + Eval prompts, grading evidence/summaries, and trigger-table prompts come + from model-generated JSON (evals.json, grading.json, trigger_results.json), + so they must be escaped to keep injected markup inert in the local viewer + and any published Artifact. + """ + return html.escape("" if value is None else str(value)) + + def b64_img(path): if not path: return None @@ -75,10 +87,10 @@ def render_expectations(expectations): badge = 'FAIL' else: badge = '?' - ev_html = f'
{evidence}
' if evidence else "" - html += f'
  • {badge} {text}{ev_html}
    • ' + ev_html = f'
    {esc(evidence)}
    ' if evidence else "" + html += f'
  • {badge} {esc(text)}{ev_html}
    • ' else: - html += f'
  • ? {exp}
    • ' + html += f'
  • ? {esc(exp)}
    • ' html += "" return html @@ -100,12 +112,12 @@ def render_quality(quality): pct = (score / mx * 100) if mx else 0 color = score_color(score, mx) html += f'''
    -
    {name} {score}/{mx}
    +
    {esc(name)} {score}/{mx}
    - {f'
    {evidence}
    ' if evidence else ""} + {f'
    {esc(evidence)}
    ' if evidence else ""}
    ''' if summary: - html += f'
    {summary}
    ' + html += f'
    {esc(summary)}
    ' html += "" return html @@ -148,14 +160,14 @@ def render_config_card(iter_dir, eval_id, config, eval_case, grading, static_res rm_color = score_color(rm_score, rm_max) html += f'
    Reference Match: {rm_score}/{rm_max}' if ref_match.get("evidence"): - html += f'
    {ref_match["evidence"]}
    ' + html += f'
    {esc(ref_match["evidence"])}
    ' html += '
    ' html += render_quality(grading.get("quality")) notes = grading.get("user_notes_summary", {}) if notes and notes.get("notes"): - html += f'
    Notes: {notes["notes"]}
    ' + html += f'
    Notes: {esc(notes["notes"])}
    ' html += '' return html @@ -196,7 +208,7 @@ def render_iteration(iter_dir): html += '
    ' html += f'
    Eval #{eid}
    ' - html += f'
    {ev.get("prompt", "")[:200]}
    ' + html += f'
    {esc(ev.get("prompt", "")[:200])}
    ' if ref_img_b64: html += '
    Target Reference
    ' @@ -252,7 +264,7 @@ def render_trigger_table(results_path): should = "Yes" if should_trigger else "No" triggered_str = "Yes" if triggered else "No" elapsed = r.get("elapsed_s", r.get("duration", "?")) - html += f'{rid}{prompt[:80]}{should}{triggered_str}{result_badge}{elapsed}s' + html += f'{esc(rid)}{esc(prompt[:80])}{should}{triggered_str}{result_badge}{esc(elapsed)}s' html += '
    ' return html @@ -395,21 +407,22 @@ def build_page(ws, title, artifact=False): panels_html = "" for i, it in enumerate(iterations): active = "active" if i == 0 else "" - tabs_html += f'' + tabs_html += f'' content = render_iteration(it) panels_html += f'
    {content}
    ' trigger_html = render_trigger_table(ws / "trigger-evals" / "trigger_results.json") - body_content = f'

    {title}

    \n
    {tabs_html}
    \n{panels_html}\n{trigger_html}' + safe_title = esc(title) + body_content = f'

    {safe_title}

    \n
    {tabs_html}
    \n{panels_html}\n{trigger_html}' if artifact: - return f'{title}\n\n{body_content}\n' + return f'{safe_title}\n\n{body_content}\n' return f''' -{title} +{safe_title} diff --git a/.claude/skills/expo-skill-eval/scripts/make-fixture.sh b/.claude/skills/expo-skill-eval/scripts/make-fixture.sh deleted file mode 100755 index 9e9eae3..0000000 --- a/.claude/skills/expo-skill-eval/scripts/make-fixture.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash -# Create an Expo app fixture for an eval run. -# -# Usage: make-fixture.sh [sdk-version] [variant] -# project-path Where the app should end up (must not exist yet) -# sdk-version Expo SDK major version, e.g. "56". Omit for the latest template. -# variant "clean" (default) runs the template's reset-project script so -# executors start from a blank app instead of the example -# screens. "full" keeps the example tabs (Home/Explore) - use -# it for evals whose prompt assumes an existing app. -# -# One pristine app is created per SDK version + variant with -# `bunx create-expo-app -t default@sdk-` and cached under -# $EXPO_SKILL_EVAL_CACHE (default ~/.cache/expo-skill-eval/fixtures). -# Each call clones the cache with APFS copy-on-write, so only the first -# call per SDK version pays the install cost. -set -euo pipefail - -PROJECT_PATH="${1:?usage: make-fixture.sh [sdk-version] [variant]}" -SDK_VERSION="${2:-}" -VARIANT="${3:-clean}" -if [[ "$VARIANT" != "clean" && "$VARIANT" != "full" ]]; then - echo "error: variant must be 'clean' or 'full', got '$VARIANT'" >&2 - exit 1 -fi -CACHE_ROOT="${EXPO_SKILL_EVAL_CACHE:-$HOME/.cache/expo-skill-eval/fixtures}" - -if [[ -e "$PROJECT_PATH" ]]; then - echo "error: $PROJECT_PATH already exists" >&2 - exit 1 -fi - -TEMPLATE="default" -CACHE_KEY="latest" -if [[ -n "$SDK_VERSION" ]]; then - TEMPLATE="default@sdk-${SDK_VERSION}" - CACHE_KEY="sdk-${SDK_VERSION}" -fi -if [[ "$VARIANT" == "clean" ]]; then - CACHE_KEY="${CACHE_KEY}-clean" -fi -CACHE_DIR="$CACHE_ROOT/$CACHE_KEY" - -if [[ ! -d "$CACHE_DIR" ]]; then - echo "building fixture cache for template '$TEMPLATE'..." >&2 - mkdir -p "$CACHE_ROOT" - TMP_DIR="$(mktemp -d)" - trap 'rm -rf "$TMP_DIR"' EXIT - (cd "$TMP_DIR" && bunx create-expo-app --yes --template "$TEMPLATE" fixture) >&2 - if [[ "$VARIANT" == "clean" ]]; then - # Strip the example screens so executors start from a blank app. - # Answer "n" to the "move to /example?" prompt - delete them instead. - (cd "$TMP_DIR/fixture" && echo n | bun run reset-project) >&2 - fi - # Pin a stable iOS bundle identifier and Android package. The default template - # leaves both unset, so a dev-build run (`expo run:ios`/`expo run:android`) - # would prompt for them - which fails in the snapshot scripts' non-interactive - # mode - and the snapshot scripts need a known id to relaunch the app by. - (cd "$TMP_DIR/fixture" && bun -e ' - const f = "app.json"; - const j = await Bun.file(f).json(); - j.expo ??= {}; - j.expo.ios ??= {}; - j.expo.android ??= {}; - j.expo.ios.bundleIdentifier ??= "com.exposkilleval.fixture"; - j.expo.android.package ??= "com.exposkilleval.fixture"; - await Bun.write(f, JSON.stringify(j, null, 2) + "\n"); - ') >&2 - # The CLI normally generates expo-env.d.ts on first `expo start`; the static - # gate runs tsc before any start, so create it up front (it provides the - # CSS-module and Metro type declarations from expo/types). - printf '/// \n' >"$TMP_DIR/fixture/expo-env.d.ts" - # expo-dev-client is required for dev-build snapshots: it registers the - # expo-development-client URL scheme and enables custom Metro port connections - # (expo run:ios/android --port N opens the app via a deep link that the dev - # client handles). Without it, a custom port deep link may not open the app. - (cd "$TMP_DIR/fixture" && bunx expo install expo-dev-client) >&2 - # First `expo lint` self-installs eslint + eslint-config-expo and writes - # eslint.config.js if missing; it may exit non-zero on that bootstrap run. - (cd "$TMP_DIR/fixture" && CI=1 bunx expo lint) >&2 || true - printf '.eval-static/\n' >>"$TMP_DIR/fixture/.gitignore" - mv "$TMP_DIR/fixture" "$CACHE_DIR" -fi - -mkdir -p "$(dirname "$PROJECT_PATH")" -# -c uses APFS clonefile (instant, no extra disk); fall back to a plain copy. -cp -Rc "$CACHE_DIR" "$PROJECT_PATH" 2>/dev/null || cp -R "$CACHE_DIR" "$PROJECT_PATH" - -# Fresh git history so `git diff` in the app shows exactly what the executor changed. -rm -rf "$PROJECT_PATH/.git" -git -C "$PROJECT_PATH" init -q -git -C "$PROJECT_PATH" add -A -git -C "$PROJECT_PATH" commit -qm "fixture: pristine $TEMPLATE template" --no-verify - -echo "$PROJECT_PATH" diff --git a/.claude/skills/expo-skill-eval/scripts/make-fixture.sh b/.claude/skills/expo-skill-eval/scripts/make-fixture.sh new file mode 120000 index 0000000..68cf95c --- /dev/null +++ b/.claude/skills/expo-skill-eval/scripts/make-fixture.sh @@ -0,0 +1 @@ +../../../eval-shared/scripts/make-fixture.sh \ No newline at end of file