diff --git a/.github/workflows/test-plugins.yml b/.github/workflows/test-plugins.yml new file mode 100644 index 00000000..75a8c4a4 --- /dev/null +++ b/.github/workflows/test-plugins.yml @@ -0,0 +1,152 @@ +name: Plugin Safety + +# Renders every changed plugin across a representative spread of matrix sizes and +# screens using the LEDMatrix core harness, and validates manifests. Gates PRs +# that touch plugin code so a change can't silently break a size or screen. + +on: + pull_request: + paths: + - 'plugins/**' + workflow_dispatch: + inputs: + all: + description: 'Check every plugin (not just changed ones)' + type: boolean + default: false + +env: + # The core repo provides the harness (scripts/check_plugin.py). Point these at + # the fork/branch where the harness lives until it is merged upstream. + CORE_REPO: ChuckBuilds/LEDMatrix + CORE_REF: main + +jobs: + safety: + runs-on: ubuntu-latest + steps: + - name: Checkout plugins + uses: actions/checkout@v4 + with: + fetch-depth: 0 + path: plugins-repo + + - name: Checkout core (harness) + uses: actions/checkout@v4 + with: + repository: ${{ env.CORE_REPO }} + ref: ${{ env.CORE_REF }} + path: core + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + + - name: Install core + harness deps + working-directory: core + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt -r requirements-test.txt + pip install RGBMatrixEmulator + + - name: Determine changed plugins (non-test code only) + id: changed + working-directory: plugins-repo + env: + ALL_INPUT: ${{ github.event.inputs.all }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + if [ "$ALL_INPUT" = "true" ]; then + ids=$(ls plugins) + else + # Plugins with a changed file outside their test/ dir. + ids=$(git diff --name-only "$BASE_SHA"...HEAD -- 'plugins/*' \ + | grep -vE '^plugins/[^/]+/test/' \ + | sed -E 's#^plugins/([^/]+)/.*#\1#' | sort -u) + fi + echo "ids<> "$GITHUB_OUTPUT" + echo "$ids" >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + echo "Changed plugins:"; echo "$ids" + + - name: Enforce version bump on changed plugins + if: steps.changed.outputs.ids != '' && github.event.inputs.all != 'true' + working-directory: plugins-repo + env: + # Read IDs from env (never interpolate ${{ }} into the script body) and + # validate each one — plugin ids flow into python -c strings below, so a + # crafted directory name on a PR branch could otherwise inject code. + IDS: ${{ steps.changed.outputs.ids }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + fail=0 + for pid in $IDS; do + case "$pid" in + '' | *[!a-z0-9._-]*) echo "::error::invalid plugin id '$pid'"; fail=1; continue ;; + esac + cur=$(python -c "import json;print(json.load(open('plugins/$pid/manifest.json'))['version'])") + old=$(git show "$BASE_SHA:plugins/$pid/manifest.json" 2>/dev/null \ + | python -c "import json,sys;print(json.load(sys.stdin)['version'])" 2>/dev/null || echo "") + if [ -n "$old" ] && [ "$cur" = "$old" ]; then + echo "::error::$pid code changed but version not bumped (still $cur). Users won't get the update." + fail=1 + else + echo "[ok] $pid version $old -> $cur" + fi + done + exit $fail + + - name: Validate manifests against schema + if: steps.changed.outputs.ids != '' + env: + IDS: ${{ steps.changed.outputs.ids }} + run: | + python - <<'PY' + import json, os, re, sys + from pathlib import Path + import jsonschema + schema = json.load(open("core/schema/manifest_schema.json")) + valid = re.compile(r"^[a-z0-9][a-z0-9._-]*$") + ids = os.environ.get("IDS", "").split() + failed = False + for pid in ids: + if not valid.match(pid): + print(f"[FAIL] invalid plugin id: {pid!r}"); failed = True; continue + mpath = Path("plugins-repo/plugins") / pid / "manifest.json" + if not mpath.exists(): + print(f"[skip] {pid}: no manifest (deleted?)"); continue + try: + jsonschema.validate(json.load(open(mpath)), schema) + print(f"[ok] {pid}: manifest valid") + except jsonschema.ValidationError as e: + print(f"[FAIL] {pid}: {e.message}"); failed = True + sys.exit(1 if failed else 0) + PY + + - name: Run safety harness on changed plugins + if: steps.changed.outputs.ids != '' + env: + IDS: ${{ steps.changed.outputs.ids }} + run: | + set -e + fail=0 + for pid in $IDS; do + case "$pid" in + '' | *[!a-z0-9._-]*) echo "::error::invalid plugin id '$pid'"; fail=1; continue ;; + esac + pdir="plugins-repo/plugins/$pid" + [ -d "$pdir" ] || { echo "::notice::$pid removed, skipping"; continue; } + echo "::group::$pid" + # Install the plugin's own runtime deps so it loads like a real install. + # A failure here is a real problem (the plugin can't load), so let it fail. + if [ -f "$pdir/requirements.txt" ]; then pip install -r "$pdir/requirements.txt"; fi + python core/scripts/check_plugin.py --plugin "$pid" \ + --plugin-dir "$PWD/plugins-repo/plugins" || fail=1 + echo "::endgroup::" + done + exit $fail + + - name: Nothing to check + if: steps.changed.outputs.ids == '' + run: echo "No plugin code changed (test-only or docs change) — safety harness skipped." diff --git a/CLAUDE.md b/CLAUDE.md index aa44ea26..62e12856 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -49,3 +49,34 @@ Third-party plugins keep their own `repo` URL and empty `plugin_path`. ## Git Hooks - `scripts/pre-commit` — Auto-syncs `plugins.json` when manifest versions change - Install: `cp scripts/pre-commit .git/hooks/pre-commit` + +## Plugin Safety Harness (cross-size / cross-screen) + +Each plugin can expose multiple screens and must render on every supported matrix +size (64×32, 128×32, 128×64, 256×32). The harness lives in the **core** repo +(`LEDMatrix/scripts/check_plugin.py`) and renders every screen at every size, +failing on crashes, content drawn past the panel edge, or visual drift vs +committed golden images. + +**Before opening a PR that changes a plugin:** +```bash +# from a LEDMatrix (core) checkout, with the monorepo plugins on the path: +python scripts/check_plugin.py --plugin \ + --plugin-dir /path/to/ledmatrix-plugins/plugins --out-dir /tmp/preview +``` +Eyeball the PNGs in `/tmp/preview`, then fix any FAIL (overflow/crash) before pushing. + +**Golden images (optional, per plugin):** commit reference PNGs so visual drift is +caught automatically: +```text +plugins//test/harness.json # deterministic config / mock data / frozen time +plugins//test/golden//.png +``` +Regenerate with `check_plugin.py --update-golden` and review the diff. See +`clock-simple/test/` for a worked example and `LEDMatrix/docs/plugin-safety-harness.md` +for the full reference. + +**CI:** `.github/workflows/test-plugins.yml` runs the harness against every +*changed* plugin on each PR (installs that plugin's `requirements.txt` first), +validates its manifest against `schema/manifest_schema.json`, and enforces the +version bump. Test-only changes (`plugins//test/**`) don't trigger the gate. diff --git a/plugins/clock-simple/test/golden/128x32/clock-simple.png b/plugins/clock-simple/test/golden/128x32/clock-simple.png new file mode 100644 index 00000000..ca9e396c Binary files /dev/null and b/plugins/clock-simple/test/golden/128x32/clock-simple.png differ diff --git a/plugins/clock-simple/test/golden/128x64/clock-simple.png b/plugins/clock-simple/test/golden/128x64/clock-simple.png new file mode 100644 index 00000000..b6e01a7c Binary files /dev/null and b/plugins/clock-simple/test/golden/128x64/clock-simple.png differ diff --git a/plugins/clock-simple/test/golden/256x32/clock-simple.png b/plugins/clock-simple/test/golden/256x32/clock-simple.png new file mode 100644 index 00000000..8f17a245 Binary files /dev/null and b/plugins/clock-simple/test/golden/256x32/clock-simple.png differ diff --git a/plugins/clock-simple/test/harness.json b/plugins/clock-simple/test/harness.json new file mode 100644 index 00000000..f3eea3b5 --- /dev/null +++ b/plugins/clock-simple/test/harness.json @@ -0,0 +1,11 @@ +{ + "_comment": "Settings the plugin safety harness uses to render clock-simple deterministically for golden-image comparison. timezone=UTC + a frozen UTC instant make the output identical on any host.", + "config": { + "timezone": "UTC", + "time_format": "12h", + "show_seconds": false, + "show_date": true + }, + "mock_data": {}, + "freeze_time": "2025-08-01 15:25:00" +}