ci-host.yaml

name: CI Host

on:
  push:
    branches: [main]
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review]
    paths:
      - "packages/host/**"
      - "packages/base/**"
      - "packages/boxel-icons/**"
      - "packages/boxel-ui/**"
      - "packages/eslint-plugin-boxel/**"
      - "packages/realm-server/**"
      - "packages/runtime-common/**"
      - ".github/workflows/ci-host.yaml"
      - "package.json"
      - "pnpm-lock.yaml"
      # CI boot path: see comment on the equivalent block in ci.yaml.
      - ".mise.toml"
      - ".github/actions/**"
      - "mise-tasks/**"
      - "packages/observability/scripts/**"
  workflow_dispatch:

concurrency:
  group: ci-host-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

permissions:
  contents: read
  id-token: write
  # Lets the warm-test-images step pull the private GHCR mirror packages.
  packages: read
  pull-requests: read

jobs:
  check-percy:
    name: Check if Percy is needed
    runs-on: ubuntu-latest
    outputs:
      percy_needed: ${{ steps.check.outputs.percy_needed }}
    steps:
      - name: Check for UI-relevant changes
        id: check
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          if [[ "${{ github.event_name }}" != "pull_request" ]]; then
            echo "percy_needed=true" >> "$GITHUB_OUTPUT"
            echo "Not a pull request — Percy will always run"
            exit 0
          fi

          if [[ "${{ github.event.pull_request.draft }}" == "true" ]]; then
            echo "percy_needed=false" >> "$GITHUB_OUTPUT"
            echo "Skipping Percy — PR is a draft. Percy snapshots are limited (10k/month), so they only run once a PR is marked as ready for review to avoid burning through the quota on work-in-progress PRs."
            exit 0
          fi

          CHANGED_FILES=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files --paginate --jq '.[].filename')

          percy_needed=false
          while IFS= read -r file; do
            case "$file" in
              packages/host/*|packages/boxel-ui/*|packages/boxel-icons/*|packages/base/*|packages/runtime-common/*)
                percy_needed=true
                echo "UI-relevant change detected: $file"
                break
                ;;
            esac
          done <<< "$CHANGED_FILES"

          echo "percy_needed=$percy_needed" >> "$GITHUB_OUTPUT"
          if [ "$percy_needed" = "true" ]; then
            echo "Percy will run — UI-relevant changes detected"
          else
            echo "Percy will be skipped — no UI-relevant changes"
          fi

  test-web-assets:
    name: Build test web assets
    uses: ./.github/workflows/test-web-assets.yaml
    with:
      caller: ci-host
      skip_catalog: true
      # Both jobs in this workflow (live-test and host-test) run their
      # chrome against the env-mode service stack at
      # `https://*.ci.localhost`, so environment.js needs to bake those
      # URLs into the host bundle. The standard-mode build still exists
      # in ci.yaml for `matrix-client-test` / `vscode-boxel-tools-package`.
      boxel_environment: ci
    concurrency:
      group: ci-host-test-web-assets-${{ github.head_ref || github.run_id }}
      cancel-in-progress: true

  live-test:
    name: Live Tests (realm)
    if: github.event.action != 'ready_for_review'
    runs-on: ubuntu-latest
    needs: test-web-assets
    concurrency:
      group: boxel-live-test-${{ github.head_ref || github.run_id }}
      cancel-in-progress: true
    env:
      BOXEL_ENVIRONMENT: ci
      SKIP_CATALOG: "true"
      # env-vars.sh sets these only when the cert files exist at sourcing
      # time, but mise-action sources it before `actions/init` runs
      # `mise run infra:ensure-dev-cert`, so the conditional silently
      # skips. Set explicitly to the path infra:ensure-dev-cert writes
      # to on ubuntu-latest. Required by `createListener` in env mode.
      REALM_SERVER_TLS_CERT_FILE: /home/runner/.local/share/boxel/dev-certs/localhost.pem
      REALM_SERVER_TLS_KEY_FILE: /home/runner/.local/share/boxel/dev-certs/localhost-key.pem
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - uses: ./.github/actions/init

      - name: Download test web assets
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: ${{ needs.test-web-assets.outputs.artifact_name }}
          path: .test-web-assets-artifact
      - name: Restore test web assets into workspace
        shell: bash
        run: |
          shopt -s dotglob
          cp -a .test-web-assets-artifact/. ./

      - name: Disable TCP/UDP network offloading
        run: sudo ethtool -K eth0 tx off rx off
      # Install + restart dbus/upower BEFORE the test services come up.
      # `sudo service dbus restart` triggers chrome's NetworkChangeNotifier
      # in any already-running chromium (the prerender's standby probe and
      # the realm-server's prerender workers), which aborts every in-flight
      # h2 stream with ERR_NETWORK_CHANGED and leaves wait-for-host-standby
      # stuck waiting for #standby-ready that never lands.
      - name: Install D-Bus helpers
        run: |
          sudo apt-get update
          sudo apt-get install -y dbus-x11 upower
          sudo service dbus restart
          sudo service upower restart

      # Trust mkcert's root in the system store too, so plain `curl` (which
      # ignores NODE_EXTRA_CA_CERTS) can validate Traefik's leaf.
      - name: Install mkcert root into system trust store
        run: mkcert -install

      - name: Start Traefik
        run: mise run infra:ensure-traefik

      # The env-mode service stack uses `*.localhost` hostnames behind
      # Traefik (host.ci.localhost, realm-server.ci.localhost, etc.).
      # Fail the job here with a focused error if loopback resolution
      # for those names breaks, rather than deep in a service boot.
      - name: Verify *.localhost resolves to loopback
        run: |
          set -x
          getent hosts realm-server.ci.localhost || true
          python3 -c "import socket; print(socket.getaddrinfo('realm-server.ci.localhost', 443))" || true
          curl -sS -o /dev/null -w '%{http_code}\n' --max-time 10 https://realm-server.ci.localhost/ || true

      # Explicit boxel-skills clone before any pnpm workspace op can
      # materialize an empty packages/skills-realm/contents directory and
      # trip services/realm-server's `[ -d contents ]` skip in
      # `pnpm skills:setup`. The repo is public and CI has no SSH key, so
      # go straight to HTTPS; skip if a real working copy is already
      # present (idempotent on reruns).
      - name: Populate skills-realm content
        run: |
          set -eux
          cd packages/skills-realm
          if [ -d contents/Skill ]; then
            echo "skills content already present, skipping clone"
          else
            rm -rf contents
            git clone --depth=1 https://github.com/cardstack/boxel-skills.git contents
          fi
          ls contents/Skill/ | head -5

      # Warm the test images from the GHCR mirror so the services
      # below start from a local image instead of an unauthenticated Docker Hub
      # pull (the cause of the recurring "Failed to reach Synapse" failures).
      # Best-effort: a miss falls back to a hardened-retry Docker Hub pull.
      - name: Warm test Docker images from the GHCR mirror
        continue-on-error: true
        uses: ./.github/actions/warm-test-images
      - name: Start test services (icons + host dist + realm servers)
        run: mise run test-services:host | tee -a /tmp/server.log &

      # Register the matrix users (realm_server, base_realm, …) the
      # realm-server logs in as. Without this the worker's `_mtimes`
      # request to the realm-server is unauthenticated and gets 404, so
      # the from-scratch index finishes with zero files and every later
      # card fetch 404s. Script waits for Synapse to be reachable before
      # registering, so the ordering with `Start test services` is safe.
      - name: create realm users
        run: pnpm register-realm-users
        working-directory: packages/matrix

      # Block until base + skills realms are fully indexed and publicly
      # readable. `_readiness-check` returns 200 only after the realm's
      # from-scratch index finishes (which itself waits for the prerender
      # standby pool the index depends on). The subsequent settle pause
      # lets the prerender's standby pool finish populating before the
      # test runner launches its own chrome instance — concurrent chrome
      # lifecycle events otherwise trip NetworkChangeNotifier and abort
      # the still-loading standby pages.
      - name: Wait for realms and assert public-read parity
        run: |
          set -u
          accept='application/vnd.api+json'
          ready='_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson'
          info='_info'
          base_ready="https://realm-server.ci.localhost/base/${ready}"
          skills_ready="https://realm-server.ci.localhost/skills/${ready}"
          skills_info="https://realm-server.ci.localhost/skills/${info}"
          base_info="https://realm-server.ci.localhost/base/${info}"
          ok=0
          for i in $(seq 1 120); do
            code=$(curl -sS -o /tmp/base_ready.json -w '%{http_code}' \
              -H "Accept: ${accept}" --max-time 15 "$base_ready" || echo 000)
            if [ "$code" = "200" ]; then ok=1; break; fi
            echo "attempt $i: base/_readiness-check -> $code (waiting)"
            sleep 10
          done
          if [ "$ok" != "1" ]; then
            echo "::error::base/_readiness-check did not return 200 within timeout (last $code)"
            exit 1
          fi
          ok=0
          for i in $(seq 1 120); do
            code=$(curl -sS -o /tmp/skills_ready.json -w '%{http_code}' \
              -H "Accept: ${accept}" --max-time 15 "$skills_ready" || echo 000)
            if [ "$code" = "200" ]; then ok=1; break; fi
            echo "attempt $i: skills/_readiness-check -> $code (waiting)"
            sleep 10
          done
          if [ "$ok" != "1" ]; then
            echo "::error::skills/_readiness-check did not return 200 within timeout (last $code)"
            exit 1
          fi
          skills_code=$(curl -sS -o /tmp/skills_info.json -w '%{http_code}' \
            -H "Accept: ${accept}" --max-time 15 "$skills_info" || echo 000)
          base_code=$(curl -sS -o /tmp/base_info.json -w '%{http_code}' \
            -H "Accept: ${accept}" --max-time 15 "$base_info" || echo 000)
          if [ "$skills_code" != "200" ] || [ "$base_code" != "200" ]; then
            echo "::error::Public-read parity broken (skills=$skills_code base=$base_code)"
            exit 1
          fi

          # icons.<slug>.localhost serves a static dist via http-server
          # behind Traefik; there's no _readiness-check, so probe a stable
          # file. A failure here means either the Traefik route isn't
          # registered yet or http-server hasn't bound its port.
          icons_probe="https://icons.ci.localhost/@cardstack/boxel-icons/v1/icons/folder-pen.js"
          ok=0
          for i in $(seq 1 30); do
            code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 "$icons_probe" || echo 000)
            if [ "$code" = "200" ]; then ok=1; break; fi
            echo "attempt $i: icons probe -> $code (waiting)"
            sleep 2
          done
          if [ "$ok" != "1" ]; then
            echo "::error::icons probe did not return 200 within timeout (last $code)"
            exit 1
          fi

          echo "::notice::Environment-mode public-read parity confirmed"

      - name: Live test suite
        run: dbus-run-session -- pnpm test:live
        working-directory: packages/host
        env:
          DBUS_SYSTEM_BUS_ADDRESS: unix:path=/run/dbus/system_bus_socket
          # testem-live.js defaults to https://localhost:4201/skills/ as the
          # realm under test; in env mode the skills realm sits at
          # https://realm-server.<slug>.localhost/skills/. live-test-wait-for-
          # servers.sh also reads this to know which realm readiness probe to
          # wait on alongside base.
          REALM_URL: https://realm-server.ci.localhost/skills/

      - name: Upload junit report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ !cancelled() }}
        with:
          name: host-live-test-report
          path: junit/host-live.xml
          retention-days: 30
      - name: Print realm server logs
        if: ${{ !cancelled() }}
        run: cat /tmp/server.log

  host-test:
    name: Host Tests
    runs-on: ubuntu-latest
    needs: [test-web-assets, check-percy]
    strategy:
      fail-fast: false
      matrix:
        shardIndex:
          [
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            10,
            11,
            12,
            13,
            14,
            15,
            16,
            17,
            18,
            19,
            20,
          ]
        shardTotal: [20]
    concurrency:
      group: boxel-host-test${{ github.head_ref || github.run_id }}-shard${{ matrix.shardIndex }}
      cancel-in-progress: true
    env:
      BOXEL_ENVIRONMENT: ci
      SKIP_CATALOG: "true"
      # env-vars.sh sets these only if the cert files exist at sourcing
      # time, but mise-action sources it before `actions/init` runs
      # `mise run infra:ensure-dev-cert` (which provisions the cert), so
      # the conditional silently skips and the env vars never propagate
      # to the test step. Setting them explicitly at the job level matches
      # the hardcoded path `infra:ensure-dev-cert` writes to on
      # ubuntu-latest. Required by `createListener` in env mode — HTTP/2
      # is a system invariant and there's no plain-HTTP fallback.
      REALM_SERVER_TLS_CERT_FILE: /home/runner/.local/share/boxel/dev-certs/localhost.pem
      REALM_SERVER_TLS_KEY_FILE: /home/runner/.local/share/boxel/dev-certs/localhost-key.pem
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      # init provisions mise, deps, mkcert + the dev TLS cert (and points
      # NODE_EXTRA_CA_CERTS at mkcert's root), which Traefik terminates with.
      - uses: ./.github/actions/init

      # this is to hopefully address the CI network flakiness that we
      # occasionally see in host tests.
      # https://github.com/actions/runner-images/issues/1187#issuecomment-686735760
      - name: Disable TCP/UDP network offloading
        run: sudo ethtool -K eth0 tx off rx off
      # Install + restart dbus/upower BEFORE the test services come up.
      # `sudo service dbus restart` triggers chrome's NetworkChangeNotifier
      # in any already-running chromium (the prerender's standby probe and
      # the realm-server's prerender workers), which aborts every in-flight
      # h2 stream with ERR_NETWORK_CHANGED and leaves wait-for-host-standby
      # stuck waiting for #standby-ready that never lands.
      - name: Install D-Bus helpers
        run: |
          sudo apt-get update
          sudo apt-get install -y dbus-x11 upower
          sudo service dbus restart
          sudo service upower restart

      # Trust mkcert's root in the system store too, so plain `curl` (which
      # ignores NODE_EXTRA_CA_CERTS) can validate Traefik's leaf.
      - name: Install mkcert root into system trust store
        run: mkcert -install

      - name: Start Traefik
        run: mise run infra:ensure-traefik

      # The env-mode service stack uses `*.localhost` hostnames behind
      # Traefik (host.ci.localhost, realm-server.ci.localhost, etc.).
      # Fail the job here with a focused error if loopback resolution
      # for those names breaks, rather than deep in a service boot.
      - name: Verify *.localhost resolves to loopback
        run: |
          set -x
          getent hosts realm-server.ci.localhost || true
          python3 -c "import socket; print(socket.getaddrinfo('realm-server.ci.localhost', 443))" || true
          curl -sS -o /dev/null -w '%{http_code}\n' --max-time 10 https://realm-server.ci.localhost/ || true

      # The env-mode host dist (with realm-server.ci.localhost URLs
      # baked in by environment.js) is built once in test-web-assets and
      # downloaded here, avoiding a per-shard rebuild. The artifact
      # bundles boxel-icons/dist, boxel-ui/addon/dist, packages/host/dist,
      # and the build manifest.
      - name: Download test web assets
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: ${{ needs.test-web-assets.outputs.artifact_name }}
          path: .test-web-assets-artifact
      - name: Restore test web assets into workspace
        shell: bash
        run: |
          shopt -s dotglob
          cp -a .test-web-assets-artifact/. ./

      # Explicit boxel-skills clone before any pnpm workspace op can
      # materialize an empty packages/skills-realm/contents directory and trip
      # services/realm-server's `[ -d contents ]` skip in `pnpm skills:setup`.
      # The repo is public and CI has no SSH key, so go straight to HTTPS;
      # skip if a real working copy is already present (idempotent on reruns).
      - name: Populate skills-realm content
        run: |
          set -eux
          cd packages/skills-realm
          if [ -d contents/Skill ]; then
            echo "skills content already present, skipping clone"
          else
            rm -rf contents
            git clone --depth=1 https://github.com/cardstack/boxel-skills.git contents
          fi
          ls contents/Skill/ | head -5

      # Warm the test images from the GHCR mirror so the services
      # below start from a local image instead of an unauthenticated Docker Hub
      # pull (the cause of the recurring "Failed to reach Synapse" failures).
      # Best-effort: a miss falls back to a hardened-retry Docker Hub pull.
      - name: Warm test Docker images from the GHCR mirror
        continue-on-error: true
        uses: ./.github/actions/warm-test-images
      - name: Start test services (icons + host dist + realm servers)
        run: mise run test-services:host | tee -a /tmp/server.log &
        env:
          # Dump HTTP/2 session/stream state for any realm-server stream that
          # stalls, to localize the Chrome ↔ Node http2 hang behind the flaky
          # 60s host-test timeouts. Diagnostics-only; see server.ts.
          REALM_SERVER_HTTP2_DIAGNOSTICS: "1"

      # Register the matrix users (realm_server, base_realm, …) the
      # realm-server logs in as. Without this the worker's `_mtimes`
      # request to the realm-server is unauthenticated and gets 404, so
      # the from-scratch index finishes with zero files and every later
      # card fetch 404s. Script waits for Synapse to be reachable before
      # registering, so the ordering with `Start test services` is safe.
      - name: create realm users
        run: pnpm register-realm-users
        working-directory: packages/matrix

      # Block until base + skills realms are fully indexed and publicly
      # readable. `_readiness-check` returns 200 only after the realm's
      # from-scratch index finishes (which itself waits for the
      # prerender standby pool the index depends on). The subsequent
      # settle pause lets the prerender's standby pool finish
      # populating before the test runner launches its own chrome
      # instance — concurrent chrome lifecycle events otherwise trip
      # NetworkChangeNotifier and abort the still-loading standby pages.
      - name: Wait for realms and assert public-read parity
        run: |
          set -u
          accept='application/vnd.api+json'
          ready='_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson'
          info='_info'
          base_ready="https://realm-server.ci.localhost/base/${ready}"
          skills_ready="https://realm-server.ci.localhost/skills/${ready}"
          skills_info="https://realm-server.ci.localhost/skills/${info}"
          base_info="https://realm-server.ci.localhost/base/${info}"

          # base/_readiness-check returns 200 only after the from-scratch
          # index of the base realm finishes. The per-slug Postgres is fresh
          # every run so this always runs, ~60s. Tests that load base cards
          # block on this being indexed.
          ok=0
          for i in $(seq 1 120); do
            code=$(curl -sS -o /tmp/base_ready.json -w '%{http_code}' \
              -H "Accept: ${accept}" --max-time 15 "$base_ready" || echo 000)
            if [ "$code" = "200" ]; then ok=1; break; fi
            echo "attempt $i: base/_readiness-check -> $code (waiting)"
            sleep 10
          done
          if [ "$ok" != "1" ]; then
            echo "::error::base/_readiness-check did not return 200 within timeout (last $code)"
            exit 1
          fi

          # skills/_readiness-check is the parallel gate for the skills
          # realm. The realm-server boots base and skills in sequence, so
          # skills indexing only starts after base finishes; the AI-assistant
          # skills tests fetch Skill/boxel-environment, which requires skills
          # to be indexed before testem launches its chrome. Without this
          # wait the test would 404 on skills cards even though base is ready.
          ok=0
          for i in $(seq 1 120); do
            code=$(curl -sS -o /tmp/skills_ready.json -w '%{http_code}' \
              -H "Accept: ${accept}" --max-time 15 "$skills_ready" || echo 000)
            if [ "$code" = "200" ]; then ok=1; break; fi
            echo "attempt $i: skills/_readiness-check -> $code (waiting)"
            sleep 10
          done
          if [ "$ok" != "1" ]; then
            echo "::error::skills/_readiness-check did not return 200 within timeout (last $code)"
            exit 1
          fi

          # Public-read parity: must be 200 unauthenticated.
          skills_code=$(curl -sS -o /tmp/skills_info.json -w '%{http_code}' \
            -H "Accept: ${accept}" --max-time 15 "$skills_info" || echo 000)
          base_code=$(curl -sS -o /tmp/base_info.json -w '%{http_code}' \
            -H "Accept: ${accept}" --max-time 15 "$base_info" || echo 000)
          echo "skills/_info -> $skills_code"
          echo "base/_info   -> $base_code"
          if [ "$skills_code" != "200" ] || [ "$base_code" != "200" ]; then
            echo "::error::Public-read parity broken (skills=$skills_code base=$base_code)"
            exit 1
          fi

          # icons.<slug>.localhost serves a static dist via http-server
          # behind Traefik; there's no _readiness-check, so probe a stable
          # file. A failure here means either the Traefik route isn't
          # registered yet or http-server hasn't bound its port.
          icons_probe="https://icons.ci.localhost/@cardstack/boxel-icons/v1/icons/folder-pen.js"
          ok=0
          for i in $(seq 1 30); do
            code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 "$icons_probe" || echo 000)
            if [ "$code" = "200" ]; then ok=1; break; fi
            echo "attempt $i: icons probe -> $code (waiting)"
            sleep 2
          done
          if [ "$ok" != "1" ]; then
            echo "::error::icons probe did not return 200 within timeout (last $code)"
            exit 1
          fi

          # The live test realm-server's `/test/` realm contains the
          # source for `testModuleRealm` URLs the host tests load
          # (hassan, mango, etc.). `start-server-and-test` releases when
          # `/node-test/_readiness-check` 200s, but node-test and test
          # index sequentially in the same realm-server process — so
          # `/node-test/` can be ready while `/test/` is still
          # indexing. Without this wait, fast tests race the indexer
          # and see the realm-server's notFound response (rendered as
          # "Card Error: Not Found" in Percy snapshots).
          test_realm_ready="https://realm-test.ci.localhost/test/${ready}"
          ok=0
          for i in $(seq 1 60); do
            code=$(curl -sS -o /dev/null -w '%{http_code}' \
              -H "Accept: ${accept}" --max-time 15 "$test_realm_ready" || echo 000)
            if [ "$code" = "200" ]; then ok=1; break; fi
            echo "attempt $i: realm-test/test/_readiness-check -> $code (waiting)"
            sleep 5
          done
          if [ "$ok" != "1" ]; then
            echo "::error::realm-test/test/_readiness-check did not return 200 within timeout (last $code)"
            exit 1
          fi

          echo "::notice::Environment-mode public-read parity confirmed (skills/_info and base/_info -> 200 unauthenticated)"

      - name: host test suite (shard ${{ matrix.shardIndex }})
        run: |
          if [ "$PERCY_ENABLED" = "true" ]; then
            TEST_CMD="pnpm test-with-percy"
          else
            echo "::notice::Skipping Percy snapshots — no UI-relevant changes detected"
            TEST_CMD="pnpm ember-test-pre-built"
          fi

          set +e
          dbus-run-session -- $TEST_CMD 2>&1 | tee /tmp/test-output.log
          exit_code=${PIPESTATUS[0]}

          # Transient chunk-fetch failures present as `ChunkLoadError` under
          # webpack and as `Failed to fetch dynamically imported module` /
          # `NetworkError when attempting to fetch resource` under Vite.
          # Env-mode CI also sees momentary `Failed to fetch` against the
          # per-slug Traefik hostnames mid-run (icons.<slug>.localhost
          # most often, occasionally realm-server.<slug>.localhost) — same
          # category: a service was briefly unreachable. The realm-server
          # path has its own `withRetries` and recovers silently; icons
          # doesn't, so the failure surfaces in any test that imports an
          # icon. Retry the whole shard once when we detect any of these.
          # The realm-test 404 lines are a transient env-mode boot race:
          # when the dev realm-server's from-scratch index takes the long
          # path on a given runner, the test realm-server's from-scratch
          # index occasionally finishes with zero files indexed, so any
          # test that reads a card from `${testModuleRealm}` sees a 404
          # from the live test realm. Both `cross-realm fetch failed` (the
          # loader's wrapped error during cross-realm linksTo resolution)
          # and `Could not find https://realm-test.<host>` (the realm-server's
          # raw notFound when the host store does a direct GET) point at
          # the same race; match both. A second shard pass typically lands
          # after the race resolves.
          RETRY_PATTERN='ChunkLoadError|Failed to fetch dynamically imported module|NetworkError when attempting to fetch resource|unable to fetch https://icons\.[^:]+: fetch failed|cross-realm fetch failed for https://realm-test\.[^/]+|Could not find https://realm-test\.[^/"]+'
          if [ $exit_code -ne 0 ] && grep -Eq "$RETRY_PATTERN" /tmp/test-output.log; then
            echo ""
            echo "::warning::Transient chunk-fetch failure detected — retrying shard ${{ matrix.shardIndex }}"
            echo "::group::Matched lines (first 5)"
            grep -E "$RETRY_PATTERN" /tmp/test-output.log | head -5 || true
            echo "::endgroup::"
            echo ""

            dbus-run-session -- $TEST_CMD 2>&1 | tee /tmp/test-output-retry.log
            retry_exit_code=${PIPESTATUS[0]}
            if [ $retry_exit_code -eq 0 ]; then
              echo "::notice::Shard ${{ matrix.shardIndex }} recovered on retry — original failure was a transient chunk-fetch flake"
              # Promote the retry's log so downstream steps (memory-report
              # extraction, baseline check / update) read from the successful
              # run, not the failed first attempt.
              mv /tmp/test-output-retry.log /tmp/test-output.log
              exit_code=0
            else
              echo "::error::Shard ${{ matrix.shardIndex }} failed twice after chunk-fetch retry — investigate as a deterministic failure, not a flake"
              if grep -Eq "$RETRY_PATTERN" /tmp/test-output-retry.log; then
                echo "::warning::Retry hit the same transient pattern again — possible persistent network or asset-serving issue"
                echo "::group::Retry-run matched lines (first 5)"
                grep -E "$RETRY_PATTERN" /tmp/test-output-retry.log | head -5 || true
                echo "::endgroup::"
              fi
              exit_code=$retry_exit_code
            fi
          fi
          exit $exit_code
        env:
          PERCY_ENABLED: ${{ needs.check-percy.outputs.percy_needed }}
          PERCY_GZIP: true
          PERCY_TOKEN: ${{ needs.check-percy.outputs.percy_needed == 'true' && secrets.PERCY_TOKEN_HOST || '' }}
          PERCY_PARALLEL_NONCE: ${{ github.run_id }}-${{ github.run_attempt }}
          HOST_TEST_PARTITION: ${{ matrix.shardIndex }}
          HOST_TEST_PARTITION_COUNT: ${{ matrix.shardTotal }}
          DBUS_SYSTEM_BUS_ADDRESS: unix:path=/run/dbus/system_bus_socket
        working-directory: packages/host

      # The shard's pass/fail must reflect the test suite actually running,
      # not the exit code of whatever wraps it. testem's xunit reporter
      # writes junit/host-N.xml only once the suite runs to completion, so
      # a missing or empty report means the suite never ran (e.g. its
      # wrapper wedged before launch) — fail loudly here rather than
      # report a green shard that executed zero tests.
      - name: Assert junit report was produced
        if: ${{ !cancelled() }}
        run: test -s junit/host-${{ matrix.shardIndex }}.xml

      - name: Upload junit report to GitHub Actions Artifacts
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ !cancelled() }}
        with:
          # Scope the artifact name to the run attempt. Artifacts are retained
          # per attempt under one run_id, and download-artifact (in the merge
          # job) has no attempt filter — so a plain `host-test-report-N` name
          # lets a re-run's merge pull this attempt's report alongside the
          # failed prior attempt's, re-injecting stale failures into the
          # published "Host Test Results" check. The attempt prefix keeps the
          # merge scoped to the current attempt.
          name: host-test-report-${{ github.run_attempt }}-${{ matrix.shardIndex }}
          path: junit/host-${{ matrix.shardIndex }}.xml
          retention-days: 30
      - name: Extract memory report
        if: ${{ !cancelled() }}
        run: node scripts/extract-memory-report.mjs /tmp/test-output.log /tmp/memory-report-${{ matrix.shardIndex }}.json
        working-directory: packages/host
      - name: Upload memory report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ !cancelled() }}
        with:
          name: memory-report-${{ matrix.shardIndex }}
          path: /tmp/memory-report-${{ matrix.shardIndex }}.json
          retention-days: 30
      - name: Print realm server logs
        if: ${{ !cancelled() }}
        run: cat /tmp/server.log
      - name: Extract worker and prerender logs
        if: ${{ !cancelled() }}
        run: |
          grep -E '^\[start:worker-development' /tmp/server.log > /tmp/worker-manager.log || true
          grep -E '^\[start:prerender-dev' /tmp/server.log > /tmp/prerender-server.log || true
          grep -E '^\[start:prerender-manager-dev' /tmp/server.log > /tmp/prerender-manager.log || true
          grep -E '^\[start:development' /tmp/server.log > /tmp/start-development.log || true
          grep -E '^\[start:test-realms|^\[start:worker-test' /tmp/server.log > /tmp/test-realms.log || true
          grep -E '^\[start:icons' /tmp/server.log > /tmp/icon-server.log || true
          grep -E '^\[start:host-dist' /tmp/server.log > /tmp/host-dist.log || true
      - name: Upload realm server log
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ !cancelled() }}
        with:
          name: realm-server-log-${{ matrix.shardIndex }}
          path: /tmp/server.log
          retention-days: 30
      - name: Upload worker manager log
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ !cancelled() }}
        with:
          name: worker-manager-log-${{ matrix.shardIndex }}
          path: /tmp/worker-manager.log
          retention-days: 30
      - name: Upload prerender server log
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ !cancelled() }}
        with:
          name: prerender-server-log-${{ matrix.shardIndex }}
          path: /tmp/prerender-server.log
          retention-days: 30
      - name: Upload prerender manager log
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ !cancelled() }}
        with:
          name: prerender-manager-log-${{ matrix.shardIndex }}
          path: /tmp/prerender-manager.log
          retention-days: 30
      - name: Upload start:development log
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ !cancelled() }}
        with:
          name: start-development-log-${{ matrix.shardIndex }}
          path: /tmp/start-development.log
          retention-days: 30
      - name: Upload test-realms log
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ !cancelled() }}
        with:
          name: test-realms-log-${{ matrix.shardIndex }}
          path: /tmp/test-realms.log
          retention-days: 30
      - name: Upload icon server log
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ !cancelled() }}
        with:
          name: icon-server-log-${{ matrix.shardIndex }}
          path: /tmp/icon-server.log
          retention-days: 30
      - name: Upload host-dist log
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ !cancelled() }}
        with:
          name: host-dist-log-${{ matrix.shardIndex }}
          path: /tmp/host-dist.log
          retention-days: 30
      - name: Upload testem log
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ !cancelled() }}
        with:
          name: testem-log-${{ matrix.shardIndex }}
          path: junit/host-testem.log
          retention-days: 30
      # testem's internal server log — records the browser↔testem socket
      # handshake (attach / disconnect / disconnectCount / reconnection-limit
      # refusal / process exit) that the TAP output omits. It's the only
      # server-side view of the intermittent post-suite `Browser timeout
      # exceeded` synthetic failure.
      - name: Upload testem debug log
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ !cancelled() }}
        with:
          name: testem-debug-log-${{ matrix.shardIndex }}
          path: junit/host-testem-debug-${{ matrix.shardIndex }}.log
          retention-days: 30
          if-no-files-found: ignore

  host-percy-finalize:
    name: Finalise Percy
    if: ${{ !cancelled() && needs.check-percy.outputs.percy_needed == 'true' }}
    concurrency:
      group: host-percy-finalize-${{ github.head_ref || github.run_id }}
      cancel-in-progress: true
    needs: [host-test, check-percy]
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - uses: ./.github/actions/init

      - name: Finalise Percy
        run: npx percy build:finalize
        working-directory: packages/host
        env:
          PERCY_TOKEN: ${{ secrets.PERCY_TOKEN_HOST }}
          PERCY_PARALLEL_NONCE: ${{ github.run_id }}-${{ github.run_attempt }}

  host-merge-reports-and-publish:
    name: Merge Host reports and publish
    if: ${{ !cancelled() && (needs.host-test.result == 'success' || needs.host-test.result == 'failure') }}
    concurrency:
      group: host-merge-reports-and-publish-${{ github.head_ref || github.run_id }}
      cancel-in-progress: true
    needs: host-test
    runs-on: ubuntu-latest
    permissions:
      # checks/pull-requests: needed by publish-unit-test-result-action to post
      # the junit summary back to the PR / commit.
      checks: write
      pull-requests: write

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - uses: ./.github/actions/init

      - name: Download JUnit reports from GitHub Actions Artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          path: all-host-reports
          # Only the current attempt's shard reports. A bare `host-test-report-*`
          # also matched the `host-test-report-merged` output below and every
          # prior attempt's shard reports, so re-running a flaky shard re-merged
          # the failed attempt's results and the check could never go green.
          pattern: host-test-report-${{ github.run_attempt }}-*
          merge-multiple: true

      - run: ls
      - run: ls all-host-reports

      - name: Merge reports
        run: npx junit-report-merger host.xml "./all-host-reports/*.xml"

      # host.xml has classname="Chrome 134.0", change to classname="Chrome" to prevent false test removal/addition warnings
      - name: Remove Chrome version number
        run: sed -i -E 's/classname="Chrome [^"]*"/classname="Chrome"/' host.xml

      - name: Upload merged report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ !cancelled() }}
        with:
          name: host-test-report-merged
          path: host.xml
          retention-days: 30

      - name: Publish test results
        id: publish-tests
        uses: EnricoMi/publish-unit-test-result-action@170bf24d20d201b842d7a52403b73ed297e6645b # 2.18.0
        if: ${{ !cancelled() }}
        with:
          junit_files: host.xml
          check_name: Host Test Results
          comment_mode: "off"
          json_file: test-results.json

      - name: Build host-tests slot
        if: ${{ !cancelled() && hashFiles('test-results.json') != '' }}
        run: |
          set -euo pipefail
          mkdir -p slot
          {
            echo "### Host Test Results"
            echo
            jq -r '.summary' test-results.json

            failures=$(jq -r '((.stats.tests_fail // 0) + (.stats.tests_error // 0))' test-results.json)
            check_url=$(jq -r '.check_url // empty' test-results.json)
            if [ -n "$check_url" ] && [ "$failures" -gt 0 ]; then
              echo
              echo "For more details on these errors, see [this check]($check_url)."
            fi
          } > slot/slot.md

      - name: Upload host-tests slot
        if: ${{ !cancelled() && hashFiles('slot/slot.md') != '' }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: pr-comment-slot-host-tests
          path: slot/
          retention-days: 7

  post-pr-status:
    name: Update PR status comment
    needs:
      - host-merge-reports-and-publish
    if: ${{ !cancelled() && needs.host-merge-reports-and-publish.result == 'success' }}
    uses: ./.github/workflows/pr-status-comment.yml
    permissions:
      contents: read
      issues: write
      pull-requests: write
      actions: read
    with:
      head-sha: ${{ github.event.pull_request.head.sha || github.sha }}
      pr-number: ${{ github.event.pull_request.number }}
      slots: host-tests
    secrets: inherit

  host-memory-baseline:
    name: Host Memory Baseline
    if: ${{ !cancelled() && (needs.host-test.result == 'success' || needs.host-test.result == 'failure') }}
    concurrency:
      group: host-memory-baseline-${{ github.head_ref || github.run_id }}
      cancel-in-progress: true
    needs: host-test
    runs-on: ubuntu-latest
    permissions:
      contents: read

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - uses: ./.github/actions/init

      - name: Download memory reports
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          path: all-memory-reports
          pattern: memory-report-*
          merge-multiple: true

      - name: Check memory baseline
        run: node packages/host/scripts/check-memory-baseline.mjs all-memory-reports packages/host/memory-baseline.json

  host-memory-baseline-update:
    name: Update Host Memory Baseline
    # Intentionally scoped to push-to-main only so the job only runs in a
    # context where the PR workflow file cannot be modified by an untrusted
    # branch — this is the one job that holds `contents: write`.
    #
    # We refresh the baseline whenever host-test completes (success OR failure),
    # not only on full green. An unrelated shard flake otherwise freezes the
    # baseline indefinitely, which then blocks downstream PRs on drift that has
    # nothing to do with their diffs. Modules missing from the current reports
    # (because a shard failed to upload) keep their prior baseline values —
    # the merge is handled in update-memory-baseline.mjs.
    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && !cancelled() && (needs.host-test.result == 'success' || needs.host-test.result == 'failure') }}
    concurrency:
      group: host-memory-baseline-update-${{ github.head_ref || github.run_id }}
      cancel-in-progress: true
    needs: host-test
    runs-on: ubuntu-latest
    permissions:
      contents: write

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          # Need history past the triggering SHA so the staleness check below
          # can inspect commits between the workflow's SHA and origin/main.
          fetch-depth: 0
      - uses: ./.github/actions/init

      - name: Download memory reports
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          path: all-memory-reports
          pattern: memory-report-*
          merge-multiple: true

      - name: Update memory baseline
        # Retry from the latest main: another PR may merge to main while host
        # CI is running (~15+ min), causing `git push` to be rejected. On each
        # retry we fetch and hard-reset to the freshest tip-of-main, then
        # re-run the merge-aware script so prior-baseline preservation stays
        # correct.
        #
        # Staleness guard: if a non-baseline commit landed on main since this
        # workflow was triggered, skip — that newer push triggered its own CI
        # run which will publish a baseline derived from its own code. We
        # ignore intervening `[skip ci]` baseline-update commits from this
        # same job because those don't represent newer code, just concurrent
        # baseline updates that the merge-aware script handles correctly.
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"

          WORKFLOW_SHA="${{ github.sha }}"

          for attempt in 1 2 3 4 5; do
            git fetch origin main

            if [ "$(git rev-parse origin/main)" != "$WORKFLOW_SHA" ] && \
               git log --format="%s" "$WORKFLOW_SHA..origin/main" | grep -qv "Update host test memory baseline"; then
              echo "main has advanced past $WORKFLOW_SHA with non-baseline commits — skipping (a newer CI run will publish the baseline)."
              exit 0
            fi

            git reset --hard origin/main

            node packages/host/scripts/update-memory-baseline.mjs all-memory-reports packages/host/memory-baseline.json
            if git diff --quiet packages/host/memory-baseline.json; then
              echo "Baseline unchanged — nothing to commit."
              exit 0
            fi
            git add packages/host/memory-baseline.json
            git commit -m "Update host test memory baseline [skip ci]"

            if git push; then
              exit 0
            fi
            echo "Push rejected on attempt $attempt — retrying after $((attempt * 5))s"
            sleep $((attempt * 5))
          done
          echo "Failed to push memory baseline after 5 attempts"
          exit 1