From 8a452992897d2cb9c7ce384ca95720ff070f3463 Mon Sep 17 00:00:00 2001 From: Gabriel Taveira Date: Thu, 21 May 2026 15:25:18 -0300 Subject: [PATCH] ci(e2e): make Maestro CI a real gate (no more continue-on-error) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Splits the single soft-gated e2e job into a required hard gate plus a soft-gated extended suite, both reporting per-flow status. Adds a quarantine file with age-based lint and a daily flake-stats job. The previous single-job design hid real failures behind `continue-on-error: true` — the launch step had `conclusion: failure` in 3 of the last 5 runs while the workflow reported `success`. Required suite (hard gate, retried 3x): only 01-launch today since every other flow needs the backend stack which can't run on GitHub-hosted macOS runners. Extended suite (soft gate, documented as temporary) runs everything else when EXPO_PUBLIC_API_URL is set. --- .github/scripts/maestro-flake-stats.py | 171 ++++++ .github/scripts/maestro-report.py | 242 +++++++++ .github/scripts/quarantine-lint.py | 77 +++ .github/workflows/e2e-mobile.yml | 723 +++++++++++++++++++------ apps/mobile/.maestro/quarantined.txt | 28 + 5 files changed, 1084 insertions(+), 157 deletions(-) create mode 100755 .github/scripts/maestro-flake-stats.py create mode 100755 .github/scripts/maestro-report.py create mode 100755 .github/scripts/quarantine-lint.py create mode 100644 apps/mobile/.maestro/quarantined.txt diff --git a/.github/scripts/maestro-flake-stats.py b/.github/scripts/maestro-flake-stats.py new file mode 100755 index 0000000..4644800 --- /dev/null +++ b/.github/scripts/maestro-flake-stats.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Compute per-flow failure rate from the last N e2e-mobile.yml runs. + +Pulls the JUnit XML artifacts uploaded by each run, parses them, and +writes a markdown table to $GITHUB_STEP_SUMMARY. The output is meant +to inform decisions like: + + - Flow X has 0% failure rate over 20 runs -> graduate to required. + - Flow Y has 80% failure rate -> add to quarantine. + - Flow Z has 5% failure rate -> investigate flake before promoting. + +This is intentionally read-only. It does NOT block CI; the per-PR +required/extended split is the source of truth for blocking. This +job's purpose is observability. + +Requires: gh CLI authenticated as a token with `actions:read` scope. +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import tempfile +import xml.etree.ElementTree as ET +import zipfile +from collections import defaultdict +from pathlib import Path + + +def gh_json(*args: str) -> list | dict: + out = subprocess.check_output(["gh", *args], text=True) + return json.loads(out) + + +def list_recent_runs(workflow: str, repo: str, limit: int) -> list[dict]: + return gh_json( + "run", + "list", + "--workflow", + workflow, + "--repo", + repo, + "--limit", + str(limit), + "--json", + "databaseId,conclusion,headBranch,createdAt,event", + ) + + +def download_artifact(repo: str, run_id: int, name: str, dest: Path) -> bool: + """Download a single artifact by name. Returns False if not present.""" + try: + subprocess.check_output( + [ + "gh", + "run", + "download", + str(run_id), + "--repo", + repo, + "--name", + name, + "--dir", + str(dest), + ], + stderr=subprocess.STDOUT, + text=True, + ) + return True + except subprocess.CalledProcessError: + return False + + +def parse_run_junits(dir_: Path) -> dict[str, bool]: + """Map flow-stem -> passed (True/False) across all xml in dir_.""" + out: dict[str, bool] = {} + for xml in dir_.rglob("*.xml"): + try: + root = ET.parse(xml).getroot() + except ET.ParseError: + continue + for suite in root.iter("testsuite"): + name = suite.get("name") or "unknown" + failed = any(b.tag in ("failure", "error") for b in suite.iter()) + # Last write wins; ok because we only run each flow once per job. + out[name] = not failed + return out + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--workflow", default="e2e-mobile.yml") + ap.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY", "")) + ap.add_argument("--limit", type=int, default=20) + ap.add_argument( + "--summary", + type=Path, + default=Path(os.environ.get("GITHUB_STEP_SUMMARY", "/dev/stdout")), + ) + args = ap.parse_args() + + if not args.repo: + print("::error::--repo or $GITHUB_REPOSITORY required", file=sys.stderr) + return 1 + + runs = list_recent_runs(args.workflow, args.repo, args.limit) + if not runs: + with args.summary.open("a") as fh: + fh.write("## Maestro flake stats\n\n_No runs found._\n") + return 0 + + # flow-stem -> [pass_count, total_count] + stats: dict[str, list[int]] = defaultdict(lambda: [0, 0]) + inspected = 0 + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + for run in runs: + run_id = run["databaseId"] + run_dir = tmp_path / str(run_id) + run_dir.mkdir() + # Artifact names produced by this workflow. + for artifact_name in ("maestro-results-required", "maestro-results-extended"): + ok = download_artifact(args.repo, run_id, artifact_name, run_dir) + if not ok: + continue + results = parse_run_junits(run_dir) + for stem, passed in results.items(): + stats[stem][1] += 1 + if passed: + stats[stem][0] += 1 + inspected += 1 + + lines = [ + f"## Maestro flake stats (last {inspected} runs of `{args.workflow}`)", + "", + "| Flow | Pass rate | Pass | Total | Verdict |", + "|---|---|---|---|---|", + ] + if not stats: + lines.append( + "_No JUnit artifacts found in recent runs. " + "The workflow either didn't reach the maestro step or didn't upload artifacts._" + ) + else: + for stem in sorted(stats): + passes, total = stats[stem] + rate = (passes / total * 100) if total else 0.0 + if total < 5: + verdict = "low-signal" + elif rate >= 95: + verdict = "stable — candidate for required" + elif rate <= 30: + verdict = "broken — quarantine or fix" + elif rate < 80: + verdict = "flaky — investigate" + else: + verdict = "ok" + lines.append(f"| `{stem}` | {rate:.0f}% | {passes} | {total} | {verdict} |") + + with args.summary.open("a") as fh: + fh.write("\n".join(lines) + "\n") + print("\n".join(lines)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/maestro-report.py b/.github/scripts/maestro-report.py new file mode 100755 index 0000000..53b0587 --- /dev/null +++ b/.github/scripts/maestro-report.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""Parse Maestro JUnit XML output, classify each flow, and emit a +GitHub Actions Job Summary plus a structured exit code. + +Usage: + maestro-report.py --junit \ + --quarantine apps/mobile/.maestro/quarantined.txt \ + --required 01-launch \ + --summary $GITHUB_STEP_SUMMARY \ + --mode required|extended + +Exit codes: + 0 — all REQUIRED flows passed (extended flows are reported but never + cause a non-zero exit when --mode=extended) + 1 — at least one REQUIRED, non-quarantined flow failed + +Why a custom reporter: + `maestro test --format junit` writes one per `name:` field + inside the flow. We want a flow-level rollup (one row per .yaml file) + plus a quarantine-aware verdict. GitHub Actions' built-in JUnit + reporters don't support quarantine semantics. +""" + +from __future__ import annotations + +import argparse +import re +import sys +import xml.etree.ElementTree as ET +from dataclasses import dataclass, field +from pathlib import Path + + +@dataclass +class FlowResult: + name: str + passed: bool + failure_message: str = "" + duration_s: float = 0.0 + + +@dataclass +class QuarantineEntry: + stem: str + reason: str + owner: str + added: str + + +def load_quarantine(path: Path) -> dict[str, QuarantineEntry]: + """Parse the quarantine file. Returns map of flow-stem -> entry. + + Lines starting with # are comments. Each entry line is + ` # reason: ...; owner: @handle; added: YYYY-MM-DD`. + Malformed lines are ignored with a warning so a broken comment can't + accidentally re-enable a quarantined flow. + """ + out: dict[str, QuarantineEntry] = {} + if not path.is_file(): + return out + for line_no, raw in enumerate(path.read_text().splitlines(), start=1): + line = raw.strip() + if not line or line.startswith("#"): + continue + stem, _, rest = line.partition("#") + stem = stem.strip() + if not stem: + continue + reason = _extract(rest, "reason") + owner = _extract(rest, "owner") + added = _extract(rest, "added") + if not (reason and owner and added): + print( + f"::warning::quarantined.txt line {line_no}: missing " + "reason/owner/added — entry ignored", + file=sys.stderr, + ) + continue + out[stem] = QuarantineEntry(stem=stem, reason=reason, owner=owner, added=added) + return out + + +def _extract(blob: str, key: str) -> str: + m = re.search(rf"{key}\s*:\s*([^;]+)", blob, flags=re.IGNORECASE) + return m.group(1).strip() if m else "" + + +def parse_junit(path: Path) -> list[FlowResult]: + """Maestro emits one per flow file with children. + + We collapse to one FlowResult per testsuite. A flow is "passed" iff + none of its testcases contain or . + """ + if not path.is_file(): + return [] + try: + root = ET.parse(path).getroot() + except ET.ParseError as exc: + print(f"::error::Failed to parse {path}: {exc}", file=sys.stderr) + return [] + + suites = root.iter("testsuite") + results: list[FlowResult] = [] + for suite in suites: + # Maestro's suite name is the flow's `name:` field. We prefer the + # file stem so quarantine entries (which use stems) line up. + # The `file` attribute is missing, so derive from `hostname` or + # the first testcase's `classname` if present. + suite_name = ( + suite.get("name") + or (next(iter(suite), ET.Element("x")).get("classname") or "") + or "unknown" + ) + stem = _to_stem(suite_name) + + failure_msgs: list[str] = [] + duration = float(suite.get("time", "0") or 0) + for case in suite.iter("testcase"): + for bad in case.iter(): + if bad.tag in ("failure", "error"): + msg = (bad.get("message") or "").strip() or (bad.text or "").strip() + failure_msgs.append(msg[:200]) + + results.append( + FlowResult( + name=stem, + passed=not failure_msgs, + failure_message="; ".join(failure_msgs)[:500], + duration_s=duration, + ) + ) + return results + + +def _to_stem(name: str) -> str: + """Convert a free-form flow name to a quarantine-comparable stem. + + Examples: + "01-launch" -> "01-launch" + "Cold launch shows SignIn" -> "cold-launch-shows-signin" + "apps/mobile/.maestro/01-launch.yaml" -> "01-launch" + """ + s = name.strip() + if "/" in s or s.endswith(".yaml"): + s = Path(s).stem + # If maestro gave us the `name:` field (which is a human string), we + # can't recover the file stem from JUnit alone — caller must align + # quarantine entries with whatever `name:` the flow declares OR use + # the stem (preferred). We fall through and let the comparison miss. + return s + + +def render_summary( + results: list[FlowResult], + quarantine: dict[str, QuarantineEntry], + required: set[str], + mode: str, +) -> tuple[str, int]: + """Render markdown for $GITHUB_STEP_SUMMARY and compute exit code.""" + lines: list[str] = [] + lines.append(f"## Maestro E2E — {mode.title()} suite") + lines.append("") + if not results: + lines.append("_No JUnit results found. Did the maestro step crash before writing output?_") + return "\n".join(lines), (1 if mode == "required" else 0) + + lines.append("| Flow | Status | Gate | Duration | Notes |") + lines.append("|---|---|---|---|---|") + + hard_fail = False + for r in sorted(results, key=lambda x: x.name): + is_required = r.name in required + is_quarantined = r.name in quarantine + if r.passed: + status = "pass" + else: + status = "fail" + if is_required and not is_quarantined: + gate = "required" + if not r.passed: + hard_fail = True + elif is_quarantined: + gate = f"quarantined ({quarantine[r.name].owner})" + else: + gate = "extended" + + notes = "" + if is_quarantined: + notes = f"reason: {quarantine[r.name].reason}; since {quarantine[r.name].added}" + elif not r.passed: + notes = r.failure_message or "(no message)" + + lines.append( + f"| `{r.name}` | {status} | {gate} | {r.duration_s:.1f}s | {notes} |" + ) + + lines.append("") + if hard_fail and mode == "required": + lines.append("**Verdict: FAIL** — at least one required, non-quarantined flow failed.") + exit_code = 1 + elif mode == "required": + lines.append("**Verdict: PASS** — all required flows green.") + exit_code = 0 + else: + any_fail = any(not r.passed for r in results) + lines.append( + "**Extended verdict (soft gate):** " + + ("regressions detected — see table above" if any_fail else "all green") + ) + exit_code = 0 # extended never blocks + + return "\n".join(lines), exit_code + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--junit", required=True, type=Path) + ap.add_argument("--quarantine", required=True, type=Path) + ap.add_argument( + "--required", + default="", + help="Comma-separated list of required flow stems (e.g. 01-launch,02-sign-in)", + ) + ap.add_argument("--summary", required=False, type=Path) + ap.add_argument("--mode", required=True, choices=["required", "extended"]) + args = ap.parse_args() + + required = {s.strip() for s in args.required.split(",") if s.strip()} + quarantine = load_quarantine(args.quarantine) + results = parse_junit(args.junit) + md, exit_code = render_summary(results, quarantine, required, args.mode) + + # Always emit to stdout for the workflow log. + print(md) + if args.summary: + with args.summary.open("a") as fh: + fh.write(md + "\n") + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/quarantine-lint.py b/.github/scripts/quarantine-lint.py new file mode 100755 index 0000000..16f90f0 --- /dev/null +++ b/.github/scripts/quarantine-lint.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Lint the Maestro quarantine file. + +Fails the CI step (exit 1) if: + - any entry is malformed (missing reason/owner/added) + - any entry is older than MAX_AGE_DAYS (default: 30) + +The age check is the anti-drift mechanism: quarantine is meant to be a +temporary escape hatch, not a permanent "skip" list. Stale entries +demand a follow-up — either fix the flow or downgrade it to extended. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import re +import sys +from pathlib import Path + +MAX_AGE_DAYS = 30 + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("path", type=Path) + ap.add_argument("--max-age-days", type=int, default=MAX_AGE_DAYS) + args = ap.parse_args() + + if not args.path.is_file(): + print(f"::error::quarantine file not found: {args.path}") + return 1 + + today = dt.date.today() + failures: list[str] = [] + + for line_no, raw in enumerate(args.path.read_text().splitlines(), start=1): + line = raw.strip() + if not line or line.startswith("#"): + continue + stem, _, rest = line.partition("#") + stem = stem.strip() + if not stem: + failures.append(f"line {line_no}: blank flow stem") + continue + + for key in ("reason", "owner", "added"): + if not re.search(rf"{key}\s*:", rest, flags=re.IGNORECASE): + failures.append(f"line {line_no} ({stem}): missing `{key}:`") + + m = re.search(r"added\s*:\s*(\d{4}-\d{2}-\d{2})", rest, flags=re.IGNORECASE) + if m: + try: + added = dt.date.fromisoformat(m.group(1)) + except ValueError: + failures.append(f"line {line_no} ({stem}): added date not ISO 8601") + continue + age = (today - added).days + if age > args.max_age_days: + failures.append( + f"line {line_no} ({stem}): quarantined for {age}d " + f"(>{args.max_age_days}d limit) — fix the flow or " + "downgrade to extended" + ) + + if failures: + print("::error::Quarantine lint failed:") + for msg in failures: + print(f" - {msg}") + return 1 + + print("Quarantine lint: ok") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/e2e-mobile.yml b/.github/workflows/e2e-mobile.yml index db34998..84b7eb7 100644 --- a/.github/workflows/e2e-mobile.yml +++ b/.github/workflows/e2e-mobile.yml @@ -1,36 +1,70 @@ name: E2E Mobile (Maestro) # --------------------------------------------------------------------------- -# REQUIRED GITHUB SECRETS +# DESIGN: required + extended split (Option D), with retry on required # --------------------------------------------------------------------------- -# Set these in Settings → Secrets and variables → Actions: +# Why two jobs instead of one with continue-on-error: +# +# - The previous single-job design used `continue-on-error: true` at the +# job level, which made every red run report green on the dashboard. +# A spot-check of the last 5 runs (2026-05-20) showed the `launch` +# step had `conclusion: failure` in 3/5 runs while the workflow +# reported `success` to PR checks. That is the failure mode we are +# eliminating here. +# +# - We can't simply flip the flag off: most flows depend on a live API +# (Postgres + PostGIS + Redis + serverless-redis-http + the tRPC +# server), and GitHub-hosted macOS runners can't run Docker services. +# Until we either (a) stand up a staging API and wire its URL into +# EXPO_PUBLIC_API_URL, or (b) split the API onto a Linux runner with +# services and proxy it to the macOS runner, the API-dependent flows +# can't run reliably in CI. +# +# - The split lets us hard-gate what we CAN run reliably today (cold +# launch) while keeping the rest of the suite visible and reportable +# via a soft-gated extended job. As flows become reliable they get +# promoted to required via REQUIRED_FLOWS below. +# +# Flow classifications (see also apps/mobile/.maestro/quarantined.txt): +# +# required: 01-launch +# Cold launch, no API needed, no RN-tree assertions. Currently the +# only flow that meets the bar. +# +# extended: every other flow under apps/mobile/.maestro/ +# Soft-gated. Runs only when EXPO_PUBLIC_API_URL secret is set so a +# staging backend is reachable. Per-flow status is reported in the +# job summary even though the job itself never fails. +# +# quarantined: listed in apps/mobile/.maestro/quarantined.txt +# Known-flaky, do-not-block. The quarantine-lint step fails the +# workflow if any entry is older than 30 days, so the list cannot +# grow unbounded. # -# APPLE_MAGIC_EMAIL – email accepted by the API bypass (default: test@pegada.app) -# APPLE_MAGIC_CODE – 6-digit OTP accepted by the API bypass (default: 424242) +# Retry semantics: # -# EXPO_PUBLIC_API_URL must also be configured (see "API backend" note below). +# The required job retries the maestro invocation up to 3 times. Real +# regressions fail consistently across retries; single-run noise (e.g. +# the iOS 26 keyboard accessory pop occasionally eating a tap) does +# not. This is Option C layered on top of Option D. # +# Simulator pinning: +# +# We pin to iPhone 17 Pro Max on the newest available iOS 26 runtime. +# The previous "best available" heuristic picked different devices on +# different runner images, which is itself a flake source. # --------------------------------------------------------------------------- -# API BACKEND NOTE (KNOWN PARTIAL BLOCKER) +# REQUIRED GITHUB SECRETS # --------------------------------------------------------------------------- -# The sign-in, create-profile, and swipe Maestro flows all call the tRPC API -# (via EXPO_PUBLIC_API_URL). The API server (apps/nextjs) requires: -# - PostgreSQL with PostGIS extension -# - Redis (BullMQ queues) -# - Serverless-Redis-HTTP (Upstash-compatible, used by the rate-limiter) +# Set these in Settings -> Secrets and variables -> Actions: # -# GitHub-hosted macOS runners do not support Docker services, so spinning up -# these containers in CI is not straightforward. Options: -# 1. Use a persistent staging API — set EXPO_PUBLIC_API_URL to its URL. -# The APPLE_MAGIC_EMAIL/CODE bypass means no real email is ever sent. -# 2. Swap to a Linux runner for the API step and use Docker services there, -# then build/run Maestro on macOS. Requires a matrix approach. -# 3. Replace the tRPC calls with an MSW service-worker mock baked into -# a dedicated "e2e" build variant. -# -# For now this workflow ONLY runs launch.yaml (cold-launch assertion, no API -# needed) by default. The remaining flows are present but gated behind the -# EXPO_PUBLIC_API_URL secret. Set it to enable them. +# APPLE_MAGIC_EMAIL - email accepted by the API bypass +# (default: test@pegada.app) +# APPLE_MAGIC_CODE - 6-digit OTP accepted by the API bypass +# (default: 424242) +# EXPO_PUBLIC_API_URL - URL of a reachable backend with the same +# APPLE_MAGIC_* + MAESTRO_E2E config. Without +# this the extended job no-ops. # --------------------------------------------------------------------------- on: @@ -40,33 +74,65 @@ on: - "apps/mobile/**" - "packages/**" - ".github/workflows/e2e-mobile.yml" + - ".github/scripts/maestro-*.py" + - ".github/scripts/quarantine-lint.py" pull_request: paths: - "apps/mobile/**" - "packages/**" - ".github/workflows/e2e-mobile.yml" + - ".github/scripts/maestro-*.py" + - ".github/scripts/quarantine-lint.py" + workflow_dispatch: + schedule: + # Daily 09:00 UTC flake-stats sweep over the last 20 runs. + # Cheap and lets us spot drift even on weeks with low PR volume. + - cron: "0 9 * * *" concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +# Workflow-level env. Each job inherits and may extend. +env: + # The single source of truth for which flow stems are HARD-GATED. + # Adding a stem here moves it from extended to required immediately. + # Use the bare file stem (no .yaml extension, no path). + REQUIRED_FLOWS: "01-launch" + # Pinned simulator. Pinning eliminates a class of "passes on Tuesday, + # fails on Wednesday" flakes caused by GitHub rolling new runner images. + SIM_DEVICE: "iPhone 17 Pro Max" + SIM_RUNTIME_PREFIX: "iOS 26" + jobs: - e2e-ios: - name: iOS E2E (Maestro) + # ------------------------------------------------------------------------- + # 0. Quarantine lint — runs on Ubuntu, gates the whole workflow. + # Cheap, fast, prevents the quarantine list from rotting. + # ------------------------------------------------------------------------- + quarantine-lint: + name: Quarantine lint + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - uses: actions/checkout@v6 + - name: Lint quarantine file + run: | + python3 .github/scripts/quarantine-lint.py \ + apps/mobile/.maestro/quarantined.txt + + # ------------------------------------------------------------------------- + # 1. REQUIRED iOS suite — HARD GATE. + # No continue-on-error anywhere. Retries the maestro invocation up + # to 3 times to absorb single-run noise. + # ------------------------------------------------------------------------- + e2e-ios-required: + name: iOS E2E (required, hard gate) + needs: quarantine-lint runs-on: macos-latest timeout-minutes: 60 - # continue-on-error: Maestro can't read RN Fabric tree on iOS 26 in CI; flows verified locally via screenshot-hash diff (see /tmp/pegada-flows-v2 docs). - continue-on-error: true - env: - # Magic credentials — fall back to safe defaults when secrets are not set. - # The API only bypasses auth when BOTH vars match what is configured - # on the server side, so these defaults only work against a server that - # has the same defaults configured (e.g. a local staging instance). APPLE_MAGIC_EMAIL: ${{ secrets.APPLE_MAGIC_EMAIL || 'test@pegada.app' }} APPLE_MAGIC_CODE: ${{ secrets.APPLE_MAGIC_CODE || '424242' }} - # Stub values for non-critical EXPO_PUBLIC_ vars so the Zod schema passes. - # Replace with real secrets for production-grade CI. EXPO_PUBLIC_ENV: development EXPO_PUBLIC_BUGSNAG_API_KEY: ci-stub EXPO_PUBLIC_AMPLITUDE_API_KEY: ci-stub @@ -74,24 +140,13 @@ jobs: EXPO_PUBLIC_REVENUE_CAT_ANDROID_API_KEY: ci-stub EXPO_PUBLIC_IOS_GOOGLE_MAPS_API_KEY: ci-stub EXPO_PUBLIC_ANDROID_GOOGLE_MAPS_API_KEY: ci-stub - # Activates the Maestro-only mock-purchase path in apps/mobile/src/services/payments. - # Mirrors MAESTRO_E2E on the API side (see packages/api/src/shared/config.ts); - # both halves must be set for the dev-only payment.maestroGrantPremium endpoint - # to function. NEVER set this in production builds. EXPO_PUBLIC_MAESTRO_E2E: "1" - # EXPO_PUBLIC_API_URL: set via secret or leave empty to skip API-dependent flows. - EXPO_PUBLIC_API_URL: ${{ secrets.EXPO_PUBLIC_API_URL || 'http://localhost:3009' }} + # No EXPO_PUBLIC_API_URL — required flows must not depend on the API. steps: - # ------------------------------------------------------------------ - # 0. Select newest installed Xcode (Expo SDK 55 / expo-modules-core - # needs Swift 6 features like MainActor isolation — Xcode 16+). - # ------------------------------------------------------------------ - name: Select Xcode run: | set -euo pipefail - # Pick the highest-version Xcode_*.app available on the runner. - # macos-latest typically has Xcode 16 and Xcode 26 side by side. XCODE=$(ls -d /Applications/Xcode_*.app 2>/dev/null \ | sed 's|.*/Xcode_||;s|\.app$||' \ | sort -V \ @@ -105,15 +160,9 @@ jobs: sudo xcode-select -s "/Applications/Xcode_${XCODE}.app/Contents/Developer" xcodebuild -version - # ------------------------------------------------------------------ - # 1. Checkout - # ------------------------------------------------------------------ - name: Checkout uses: actions/checkout@v6 - # ------------------------------------------------------------------ - # 2. Node + pnpm - # ------------------------------------------------------------------ - name: Setup pnpm uses: pnpm/action-setup@v4.2.0 @@ -122,9 +171,6 @@ jobs: with: node-version: 20.x - # ------------------------------------------------------------------ - # 3. Caches - # ------------------------------------------------------------------ - name: Get pnpm store path id: pnpm-store run: echo "dir=$(pnpm store path)" >> $GITHUB_OUTPUT @@ -143,54 +189,41 @@ jobs: key: ${{ runner.os }}-pods-${{ hashFiles('apps/mobile/ios/Podfile.lock') }} restore-keys: ${{ runner.os }}-pods- + # Maestro release line moves; key on the install script's + # current pointer (we re-download if the binary is missing). + # Pinning to a tagged version would be safer once we standardize + # one — for now we cache the latest install across the workflow's + # macOS runs and rely on the install step to be idempotent. - name: Cache Maestro uses: actions/cache@v5 with: path: ~/.maestro - key: maestro-${{ runner.os }}-latest + key: maestro-${{ runner.os }}-v1 - # ------------------------------------------------------------------ - # 4. Install JS dependencies - # ------------------------------------------------------------------ - name: Install dependencies run: pnpm install --frozen-lockfile - # ------------------------------------------------------------------ - # 5. Setup Java 17 (for future Android support) - # ------------------------------------------------------------------ - name: Setup Java 17 uses: actions/setup-java@v4 with: distribution: temurin java-version: "17" - # ------------------------------------------------------------------ - # 6. Install Maestro - # ------------------------------------------------------------------ - name: Install Maestro run: | - if [ ! -f "$HOME/.maestro/bin/maestro" ]; then + if [ ! -x "$HOME/.maestro/bin/maestro" ]; then curl -fsSL "https://get.maestro.mobile.dev" | bash fi echo "$HOME/.maestro/bin" >> $GITHUB_PATH - # ------------------------------------------------------------------ - # 7. Stub Google Services files (required by expo prebuild) - # ------------------------------------------------------------------ - name: Stub Google Services files working-directory: apps/mobile run: | - # Create minimal stub files so prebuild doesn't error. - # These are not used for the auth flow under test. - # Use -s (non-empty) instead of -f: the apps/mobile preinstall - # script writes a single newline when GOOGLE_SERVICES_JSON env is - # unset, leaving a 1-byte file that passes -f but breaks parsers. if [ ! -s google-services.json ]; then echo '{"project_info":{"project_number":"000","project_id":"ci-stub","storage_bucket":""},"client":[]}' \ > google-services.json fi if [ ! -s GoogleService-Info.plist ] || ! head -1 GoogleService-Info.plist | grep -q '' \ '' \ @@ -212,25 +245,14 @@ jobs: '' \ '' \ > GoogleService-Info.plist - echo "::group::Stub plist contents" - cat GoogleService-Info.plist - echo "::endgroup::" fi - # ------------------------------------------------------------------ - # 8. Expo prebuild (generates ios/ directory, does not commit it) - # ------------------------------------------------------------------ - name: Expo prebuild (iOS) working-directory: apps/mobile run: npx expo prebuild --platform ios --no-install env: EXPO_NO_TELEMETRY: 1 - # ------------------------------------------------------------------ - # 8b. Patch Podfile: @expo/config-plugins emits `react-native-google-maps` - # but react-native-maps@1.27.2 ships as `react-native-maps`. Rewrite - # the pod name in the generated Podfile so CocoaPods can resolve it. - # ------------------------------------------------------------------ - name: Patch Podfile (react-native-maps pod name) working-directory: apps/mobile/ios run: | @@ -238,41 +260,24 @@ jobs: sed -i.bak "s|pod 'react-native-google-maps'|pod 'react-native-maps'|g" Podfile sed -i.bak "s|:path => '../../../node_modules/react-native-maps/google'|:path => '../../../node_modules/react-native-maps'|g" Podfile rm -f Podfile.bak - echo "::group::Patched Podfile" - grep -n "react-native-maps" Podfile || true - echo "::endgroup::" fi - # ------------------------------------------------------------------ - # 9. CocoaPods install - # ------------------------------------------------------------------ - name: CocoaPods install working-directory: apps/mobile/ios run: pod install - # ------------------------------------------------------------------ - # 9b. Neutralize Bugsnag source-map upload build phase. - # The phase fails when BUGSNAG_API_KEY is unset (CI never has it). - # Prepend `exit 0` so it short-circuits without blocking the build. - # ------------------------------------------------------------------ - name: Disable Bugsnag source-map upload phase working-directory: apps/mobile/ios run: | set -euo pipefail PBXPROJ=$(ls *.xcodeproj/project.pbxproj | head -1) - # Brace-aware .pbxproj patcher — see .github/scripts/patch-bugsnag-phase.py PBXPROJ="$PBXPROJ" python3 "$GITHUB_WORKSPACE/.github/scripts/patch-bugsnag-phase.py" - # ------------------------------------------------------------------ - # 10. Build for iOS simulator - # ------------------------------------------------------------------ - name: Build iOS (Debug simulator) working-directory: apps/mobile/ios run: | set -euo pipefail WORKSPACE=$(ls -d *.xcworkspace | head -1) - # The app scheme matches the .xcodeproj name (Pegada). Workspace - # also lists every Pod scheme — pick the one tied to the .xcodeproj. export APP_NAME=$(ls -d *.xcodeproj | head -1 | sed 's/.xcodeproj$//') SCHEME=$(xcodebuild -workspace "$WORKSPACE" -list -json \ | python3 -c " @@ -296,7 +301,6 @@ jobs: CODE_SIGNING_ALLOWED=NO \ build \ | xcpretty - # Verify the .app actually exists before moving on. PROD_DIR=build/Build/Products/Debug-iphonesimulator if ! ls "$PROD_DIR"/*.app >/dev/null 2>&1; then echo "::error::xcodebuild reported success but produced no .app in $PROD_DIR" @@ -305,34 +309,47 @@ jobs: fi echo "Produced: $(ls -d $PROD_DIR/*.app)" - # ------------------------------------------------------------------ - # 11. Boot simulator and install app - # ------------------------------------------------------------------ - - name: Boot iOS simulator + # Pin the device + runtime. We pick the highest patch version that + # starts with $SIM_RUNTIME_PREFIX so we still benefit from runner + # updates within the iOS 26 major. If no match is found we hard + # fail rather than silently falling back to whatever's available. + - name: Boot iOS simulator (pinned) run: | set -euo pipefail - # Parse the JSON device list so we never grab a section header - # like "-- iOS 18.0 --" or the word "Shutdown" by accident. DEVICE_ID=$(xcrun simctl list devices available --json \ - | python3 -c " - import json,sys - d=json.load(sys.stdin)['devices'] - # Prefer iPhone 17/16/15 Pro then any iPhone in the newest iOS runtime - best=None - for runtime in sorted(d.keys(), reverse=True): - for dev in d[runtime]: - name=dev.get('name','') - if not name.startswith('iPhone'): continue - if best is None: best=dev['udid'] - if 'Pro' in name: best=dev['udid']; break - if best and any('Pro' in x.get('name','') for x in d[runtime]): break - print(best or '')") + | SIM_DEVICE="$SIM_DEVICE" SIM_RUNTIME_PREFIX="$SIM_RUNTIME_PREFIX" python3 -c " + import json,os,sys,re + want_name=os.environ['SIM_DEVICE'] + want_runtime=os.environ['SIM_RUNTIME_PREFIX'] + data=json.load(sys.stdin)['devices'] + candidates=[] + for runtime,devices in data.items(): + # runtime keys look like 'com.apple.CoreSimulator.SimRuntime.iOS-26-0' + friendly=runtime.split('SimRuntime.')[-1].replace('-', ' ').replace('iOS ', 'iOS ') + if 'iOS' not in runtime: continue + # Normalize for prefix match: 'iOS-26-0' -> 'iOS 26' + m=re.search(r'iOS-(\\d+)', runtime) + if not m: continue + major=m.group(1) + normalized=f'iOS {major}' + if not normalized.startswith(want_runtime): continue + for dev in devices: + if dev.get('name')==want_name: + candidates.append((runtime, dev['udid'])) + if not candidates: + print('', end='') + else: + # Sort by runtime descending (newest patch first) + candidates.sort(reverse=True) + print(candidates[0][1]) + ") if [ -z "$DEVICE_ID" ]; then - echo "::error::No iPhone simulator available" + echo "::error::No simulator matching device='$SIM_DEVICE' runtime prefix='$SIM_RUNTIME_PREFIX'" + echo "Available devices:" xcrun simctl list devices available exit 1 fi - echo "Booting simulator: $DEVICE_ID" + echo "Booting pinned simulator: $DEVICE_ID ($SIM_DEVICE, $SIM_RUNTIME_PREFIX)" xcrun simctl boot "$DEVICE_ID" || true xcrun simctl bootstatus "$DEVICE_ID" -b echo "SIMULATOR_UDID=$DEVICE_ID" >> $GITHUB_ENV @@ -340,16 +357,11 @@ jobs: - name: Install app on simulator run: | set -euo pipefail - echo "::group::derivedData tree" - find apps/mobile/ios/build -name "*.app" -type d 2>/dev/null || true - ls apps/mobile/ios/build/Build/Products/ 2>/dev/null || true - echo "::endgroup::" APP_PATH=$(find apps/mobile/ios/build -name "*.app" -type d -path "*Debug-iphonesimulator*" | head -1) if [ -z "$APP_PATH" ]; then - echo "::error::No .app produced under apps/mobile/ios/build — xcodebuild likely failed silently. Check the build step log." + echo "::error::No .app produced under apps/mobile/ios/build" exit 1 fi - echo "Installing: $APP_PATH" xcrun simctl install "$SIMULATOR_UDID" "$APP_PATH" - name: Launch app (warm up) @@ -357,55 +369,452 @@ jobs: xcrun simctl launch "$SIMULATOR_UDID" app.pegada || true sleep 3 - # ------------------------------------------------------------------ - # 12. Run Maestro flows - # ------------------------------------------------------------------ - - name: Run Maestro launch flow (no API required) + # The required suite is small enough to enumerate. We loop through + # the comma-separated REQUIRED_FLOWS and execute each as its own + # maestro invocation so a single failing flow has a clean failure + # boundary in the log. Retry up to 3x per flow. + - name: Run required Maestro flows (hard gate, retry 3x) + id: maestro-required run: | - maestro test apps/mobile/.maestro/launch.yaml \ - --format junit \ - --output maestro-results-launch.xml + set -euo pipefail + IFS=',' read -ra FLOWS <<< "$REQUIRED_FLOWS" + FAILED=() + for stem in "${FLOWS[@]}"; do + stem="$(echo "$stem" | xargs)" # trim + flow="apps/mobile/.maestro/${stem}.yaml" + if [ ! -f "$flow" ]; then + echo "::error::Required flow not found: $flow" + FAILED+=("$stem (missing file)") + continue + fi + ok=0 + for attempt in 1 2 3; do + echo "::group::$stem attempt $attempt" + if maestro test "$flow" \ + --format junit \ + --output "maestro-results-required-${stem}-attempt-${attempt}.xml"; then + # Canonical result file the reporter expects. + cp "maestro-results-required-${stem}-attempt-${attempt}.xml" \ + "maestro-results-required.xml" 2>/dev/null || true + ok=1 + echo "::endgroup::" + break + fi + echo "::endgroup::" + echo "::warning::$stem attempt $attempt failed" + done + if [ "$ok" -ne 1 ]; then + FAILED+=("$stem") + # Preserve the last attempt's JUnit so the reporter shows it. + cp "maestro-results-required-${stem}-attempt-3.xml" \ + "maestro-results-required.xml" 2>/dev/null || true + fi + done + if [ "${#FAILED[@]}" -gt 0 ]; then + echo "::error::Required flows failed after 3 retries: ${FAILED[*]}" + exit 1 + fi env: APPLE_MAGIC_EMAIL: ${{ env.APPLE_MAGIC_EMAIL }} APPLE_MAGIC_CODE: ${{ env.APPLE_MAGIC_CODE }} - - name: Run Maestro full flows (requires EXPO_PUBLIC_API_URL secret) - if: env.EXPO_PUBLIC_API_URL != '' + # Report runs even when the maestro step failed so the summary + # is always populated. The reporter's own exit code is what + # actually gates the job. + - name: Report required flow results + if: always() + run: | + # Aggregate per-attempt JUnit files into a single rollup so the + # reporter sees the last-attempt outcome per flow. + ls maestro-results-required-*-attempt-*.xml 2>/dev/null || true + python3 .github/scripts/maestro-report.py \ + --junit maestro-results-required.xml \ + --quarantine apps/mobile/.maestro/quarantined.txt \ + --required "$REQUIRED_FLOWS" \ + --summary "$GITHUB_STEP_SUMMARY" \ + --mode required + + - name: Upload required Maestro artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: maestro-results-required + path: | + maestro-results-required*.xml + ~/.maestro/tests/**/*.png + if-no-files-found: ignore + + - name: Capture simulator log on failure + if: failure() + run: | + xcrun simctl spawn "$SIMULATOR_UDID" log collect \ + --output simulator-required.logarchive || true + continue-on-error: true + + - name: Upload simulator log on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: simulator-log-required + path: simulator-required.logarchive + if-no-files-found: ignore + + # ------------------------------------------------------------------------- + # 2. EXTENDED iOS suite — SOFT GATE (documented temporary). + # Runs the full suite when a backend URL is configured. Reports + # per-flow status. Honors the quarantine list. Never fails the + # workflow. + # + # The soft gate is intentional and explicitly time-bound: it will + # be flipped to a hard gate once either (a) staging API is wired + # via EXPO_PUBLIC_API_URL and proven stable for 20 consecutive + # runs (see flake-stats job), or (b) the backend services move to + # a Linux runner. Until then, treat the extended summary as the + # source of truth for what's actually green. + # ------------------------------------------------------------------------- + e2e-ios-extended: + name: iOS E2E (extended, soft gate) + needs: quarantine-lint + runs-on: macos-latest + timeout-minutes: 90 + # Soft gate. The reporter step inside this job never exits non-zero + # in --mode=extended, so this flag is belt-and-suspenders against + # an infrastructure failure (e.g. xcodebuild OOM) bubbling up. + # When we promote the extended suite to required, REMOVE this line. + continue-on-error: true + + env: + APPLE_MAGIC_EMAIL: ${{ secrets.APPLE_MAGIC_EMAIL || 'test@pegada.app' }} + APPLE_MAGIC_CODE: ${{ secrets.APPLE_MAGIC_CODE || '424242' }} + EXPO_PUBLIC_ENV: development + EXPO_PUBLIC_BUGSNAG_API_KEY: ci-stub + EXPO_PUBLIC_AMPLITUDE_API_KEY: ci-stub + EXPO_PUBLIC_REVENUE_CAT_IOS_API_KEY: ci-stub + EXPO_PUBLIC_REVENUE_CAT_ANDROID_API_KEY: ci-stub + EXPO_PUBLIC_IOS_GOOGLE_MAPS_API_KEY: ci-stub + EXPO_PUBLIC_ANDROID_GOOGLE_MAPS_API_KEY: ci-stub + EXPO_PUBLIC_MAESTRO_E2E: "1" + EXPO_PUBLIC_API_URL: ${{ secrets.EXPO_PUBLIC_API_URL }} + + steps: + - name: Preflight — check for backend URL + id: preflight + run: | + if [ -z "${EXPO_PUBLIC_API_URL:-}" ]; then + echo "::warning::EXPO_PUBLIC_API_URL secret is not set. The extended suite cannot run without a reachable backend (Postgres+PostGIS, Redis, serverless-redis-http, tRPC server). Skipping. To enable, set the secret to a staging URL with the same APPLE_MAGIC_* + MAESTRO_E2E config as the local stack." + echo "skip=true" >> "$GITHUB_OUTPUT" + else + echo "skip=false" >> "$GITHUB_OUTPUT" + fi + + - name: Write skip summary + if: steps.preflight.outputs.skip == 'true' + run: | + { + echo "## Maestro E2E — Extended suite" + echo "" + echo "Skipped: EXPO_PUBLIC_API_URL secret is not configured." + echo "" + echo "The extended suite (flows 02-27) needs a reachable backend." + echo "GitHub-hosted macOS runners can't run Docker services, so" + echo "options to unblock are:" + echo "" + echo " 1. Point EXPO_PUBLIC_API_URL at a persistent staging API." + echo " 2. Split the API onto a Linux runner with Docker services" + echo " and proxy/tunnel it to the macOS runner." + echo " 3. Bake an MSW mock into an \\\`e2e\\\` build variant." + } >> "$GITHUB_STEP_SUMMARY" + + - name: Select Xcode + if: steps.preflight.outputs.skip != 'true' + run: | + set -euo pipefail + XCODE=$(ls -d /Applications/Xcode_*.app 2>/dev/null \ + | sed 's|.*/Xcode_||;s|\.app$||' \ + | sort -V \ + | tail -1) + sudo xcode-select -s "/Applications/Xcode_${XCODE}.app/Contents/Developer" + xcodebuild -version + + - name: Checkout + if: steps.preflight.outputs.skip != 'true' + uses: actions/checkout@v6 + + - name: Setup pnpm + if: steps.preflight.outputs.skip != 'true' + uses: pnpm/action-setup@v4.2.0 + + - name: Setup Node + if: steps.preflight.outputs.skip != 'true' + uses: actions/setup-node@v6 + with: + node-version: 20.x + + - name: Get pnpm store path + if: steps.preflight.outputs.skip != 'true' + id: pnpm-store + run: echo "dir=$(pnpm store path)" >> $GITHUB_OUTPUT + + - name: Cache pnpm store + if: steps.preflight.outputs.skip != 'true' + uses: actions/cache@v5 + with: + path: ${{ steps.pnpm-store.outputs.dir }} + key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: ${{ runner.os }}-pnpm- + + - name: Cache CocoaPods + if: steps.preflight.outputs.skip != 'true' + uses: actions/cache@v5 + with: + path: apps/mobile/ios/Pods + key: ${{ runner.os }}-pods-${{ hashFiles('apps/mobile/ios/Podfile.lock') }} + restore-keys: ${{ runner.os }}-pods- + + - name: Cache Maestro + if: steps.preflight.outputs.skip != 'true' + uses: actions/cache@v5 + with: + path: ~/.maestro + key: maestro-${{ runner.os }}-v1 + + - name: Install dependencies + if: steps.preflight.outputs.skip != 'true' + run: pnpm install --frozen-lockfile + + - name: Setup Java 17 + if: steps.preflight.outputs.skip != 'true' + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "17" + + - name: Install Maestro + if: steps.preflight.outputs.skip != 'true' run: | - maestro test apps/mobile/.maestro/ \ + if [ ! -x "$HOME/.maestro/bin/maestro" ]; then + curl -fsSL "https://get.maestro.mobile.dev" | bash + fi + echo "$HOME/.maestro/bin" >> $GITHUB_PATH + + - name: Stub Google Services files + if: steps.preflight.outputs.skip != 'true' + working-directory: apps/mobile + run: | + if [ ! -s google-services.json ]; then + echo '{"project_info":{"project_number":"000","project_id":"ci-stub","storage_bucket":""},"client":[]}' \ + > google-services.json + fi + if [ ! -s GoogleService-Info.plist ] || ! head -1 GoogleService-Info.plist | grep -q '' \ + '' \ + '' \ + '' \ + ' API_KEYci-stub' \ + ' BUNDLE_IDapp.pegada' \ + ' CLIENT_IDci-stub.apps.googleusercontent.com' \ + ' REVERSED_CLIENT_IDcom.googleusercontent.apps.ci-stub' \ + ' GOOGLE_APP_ID1:000:ios:000' \ + ' GCM_SENDER_ID000' \ + ' PROJECT_IDci-stub' \ + ' STORAGE_BUCKET' \ + '' \ + '' \ + > GoogleService-Info.plist + fi + + - name: Expo prebuild (iOS) + if: steps.preflight.outputs.skip != 'true' + working-directory: apps/mobile + run: npx expo prebuild --platform ios --no-install + env: + EXPO_NO_TELEMETRY: 1 + + - name: Patch Podfile (react-native-maps pod name) + if: steps.preflight.outputs.skip != 'true' + working-directory: apps/mobile/ios + run: | + if grep -q "react-native-google-maps" Podfile; then + sed -i.bak "s|pod 'react-native-google-maps'|pod 'react-native-maps'|g" Podfile + sed -i.bak "s|:path => '../../../node_modules/react-native-maps/google'|:path => '../../../node_modules/react-native-maps'|g" Podfile + rm -f Podfile.bak + fi + + - name: CocoaPods install + if: steps.preflight.outputs.skip != 'true' + working-directory: apps/mobile/ios + run: pod install + + - name: Disable Bugsnag source-map upload phase + if: steps.preflight.outputs.skip != 'true' + working-directory: apps/mobile/ios + run: | + set -euo pipefail + PBXPROJ=$(ls *.xcodeproj/project.pbxproj | head -1) + PBXPROJ="$PBXPROJ" python3 "$GITHUB_WORKSPACE/.github/scripts/patch-bugsnag-phase.py" + + - name: Build iOS (Debug simulator) + if: steps.preflight.outputs.skip != 'true' + working-directory: apps/mobile/ios + run: | + set -euo pipefail + WORKSPACE=$(ls -d *.xcworkspace | head -1) + export APP_NAME=$(ls -d *.xcodeproj | head -1 | sed 's/.xcodeproj$//') + SCHEME=$(xcodebuild -workspace "$WORKSPACE" -list -json \ + | python3 -c " + import json,sys,os + app=os.environ['APP_NAME'] + schemes=json.load(sys.stdin)['workspace']['schemes'] + if app in schemes: print(app) + else: + hit=[s for s in schemes if s.lower()==app.lower()] + print(hit[0] if hit else app) + ") + set -o pipefail + xcodebuild \ + -workspace "$WORKSPACE" \ + -scheme "$SCHEME" \ + -configuration Debug \ + -sdk iphonesimulator \ + -derivedDataPath build \ + -destination 'generic/platform=iOS Simulator' \ + CODE_SIGNING_ALLOWED=NO \ + build \ + | xcpretty + + - name: Boot iOS simulator (pinned) + if: steps.preflight.outputs.skip != 'true' + run: | + set -euo pipefail + DEVICE_ID=$(xcrun simctl list devices available --json \ + | SIM_DEVICE="$SIM_DEVICE" SIM_RUNTIME_PREFIX="$SIM_RUNTIME_PREFIX" python3 -c " + import json,os,sys,re + want_name=os.environ['SIM_DEVICE'] + want_runtime=os.environ['SIM_RUNTIME_PREFIX'] + data=json.load(sys.stdin)['devices'] + candidates=[] + for runtime,devices in data.items(): + if 'iOS' not in runtime: continue + m=re.search(r'iOS-(\\d+)', runtime) + if not m: continue + normalized=f'iOS {m.group(1)}' + if not normalized.startswith(want_runtime): continue + for dev in devices: + if dev.get('name')==want_name: + candidates.append((runtime, dev['udid'])) + candidates.sort(reverse=True) + print(candidates[0][1] if candidates else '') + ") + if [ -z "$DEVICE_ID" ]; then + echo "::error::No simulator matching device='$SIM_DEVICE' runtime prefix='$SIM_RUNTIME_PREFIX'" + exit 1 + fi + xcrun simctl boot "$DEVICE_ID" || true + xcrun simctl bootstatus "$DEVICE_ID" -b + echo "SIMULATOR_UDID=$DEVICE_ID" >> $GITHUB_ENV + + - name: Install app on simulator + if: steps.preflight.outputs.skip != 'true' + run: | + set -euo pipefail + APP_PATH=$(find apps/mobile/ios/build -name "*.app" -type d -path "*Debug-iphonesimulator*" | head -1) + xcrun simctl install "$SIMULATOR_UDID" "$APP_PATH" + + - name: Run full Maestro suite (excludes quarantined) + if: steps.preflight.outputs.skip != 'true' + id: maestro-extended + run: | + set -uo pipefail + # Build a Maestro `--exclude-tags` list isn't enough because + # quarantine entries are by stem, not tag. Instead we collect + # every non-quarantined .yaml under .maestro/ and feed them. + QUARANTINED=$(grep -v '^[[:space:]]*#' apps/mobile/.maestro/quarantined.txt \ + | awk '{print $1}' | grep -v '^$' | sort -u || true) + FLOWS=() + for f in apps/mobile/.maestro/*.yaml; do + stem="$(basename "$f" .yaml)" + if echo "$QUARANTINED" | grep -qx "$stem"; then + echo "::notice::skipping quarantined flow: $stem" + continue + fi + FLOWS+=("$f") + done + if [ "${#FLOWS[@]}" -eq 0 ]; then + echo "::warning::No extended flows to run" + exit 0 + fi + # Run as a single maestro invocation so suite-level ordering + # (config.yaml) is respected. Allow failures — the reporter + # decides what's blocking. + maestro test "${FLOWS[@]}" \ --format junit \ - --output maestro-results-full.xml \ - --exclude-tags no-api + --output maestro-results-extended.xml \ + --exclude-tags no-api,util || true env: APPLE_MAGIC_EMAIL: ${{ env.APPLE_MAGIC_EMAIL }} APPLE_MAGIC_CODE: ${{ env.APPLE_MAGIC_CODE }} - EXPO_PUBLIC_API_URL: ${{ secrets.EXPO_PUBLIC_API_URL }} - continue-on-error: true + EXPO_PUBLIC_API_URL: ${{ env.EXPO_PUBLIC_API_URL }} - # ------------------------------------------------------------------ - # 13. Upload artifacts on failure - # ------------------------------------------------------------------ - - name: Upload Maestro results - if: always() + - name: Report extended flow results + if: steps.preflight.outputs.skip != 'true' && always() + run: | + python3 .github/scripts/maestro-report.py \ + --junit maestro-results-extended.xml \ + --quarantine apps/mobile/.maestro/quarantined.txt \ + --required "$REQUIRED_FLOWS" \ + --summary "$GITHUB_STEP_SUMMARY" \ + --mode extended + + - name: Upload extended Maestro artifacts + if: steps.preflight.outputs.skip != 'true' && always() uses: actions/upload-artifact@v4 with: - name: maestro-results + name: maestro-results-extended path: | - maestro-results-*.xml + maestro-results-extended*.xml ~/.maestro/tests/**/*.png if-no-files-found: ignore - name: Capture simulator log on failure - if: failure() + if: steps.preflight.outputs.skip != 'true' && failure() run: | xcrun simctl spawn "$SIMULATOR_UDID" log collect \ - --output simulator.logarchive || true + --output simulator-extended.logarchive || true continue-on-error: true - name: Upload simulator log on failure - if: failure() + if: steps.preflight.outputs.skip != 'true' && failure() uses: actions/upload-artifact@v4 with: - name: simulator-log - path: simulator.logarchive + name: simulator-log-extended + path: simulator-extended.logarchive if-no-files-found: ignore + + # ------------------------------------------------------------------------- + # 3. Flake stats — read-only observability. + # Pulls JUnit artifacts from the last N runs and emits a per-flow + # pass-rate table. Never blocks. Runs on PRs (so authors can see the + # state of the world before deciding to promote/quarantine) and + # daily via cron. + # ------------------------------------------------------------------------- + flake-stats: + name: Flake stats (last 20 runs) + runs-on: ubuntu-latest + timeout-minutes: 5 + # Don't block, don't depend on the suites — this is observability. + if: github.event_name != 'schedule' || github.repository == github.event.repository.full_name + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@v6 + - name: Compute pass rate per flow + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python3 .github/scripts/maestro-flake-stats.py \ + --workflow e2e-mobile.yml \ + --repo "$GITHUB_REPOSITORY" \ + --limit 20 \ + --summary "$GITHUB_STEP_SUMMARY" diff --git a/apps/mobile/.maestro/quarantined.txt b/apps/mobile/.maestro/quarantined.txt new file mode 100644 index 0000000..6295608 --- /dev/null +++ b/apps/mobile/.maestro/quarantined.txt @@ -0,0 +1,28 @@ +# Quarantined Maestro flows +# --------------------------------------------------------------------------- +# Flows listed here are KNOWN to fail or be flaky in CI and are temporarily +# soft-gated. Each entry MUST include: +# - the flow stem (no path, no .yaml extension) +# - a # reason: ... comment on the same line +# - a # owner: comment so we know who is unblocking it +# - a # added: YYYY-MM-DD comment so we can spot stale entries +# +# Quarantine is a TEMPORARY measure. Entries older than 30 days are flagged +# by the `quarantine-lint` step in .github/workflows/e2e-mobile.yml. The PR +# author owns the entry until removed. +# +# To add a flow: +# 1. Open the failure logs and confirm it's flake (not a real regression). +# 2. Add a line below with reason/owner/added. +# 3. Open a Linear ticket and link it in the reason. +# +# To remove a flow: +# 1. Run the flow 10x in CI (push the branch repeatedly or use workflow_dispatch). +# 2. If 10/10 pass, delete the line. +# 3. The flow is now a hard gate again. +# +# Format (one flow per line, # comments allowed): +# # reason: ; owner: @handle; added: YYYY-MM-DD +# +# Example (commented out — uncomment and edit when needed): +# 22-new-match-journey # reason: race on socket reconnect; owner: @gstj; added: 2026-05-20