Improved E2E CI runtime

The E2E CI shards were all paying the cost of analytics infrastructure even when they only ran the main Playwright project. Splitting analytics into its own shard set keeps coverage while avoiding repeated Tinybird setup for the main suite.

Author: JohnONolan

.github/workflows/ci.yml

@@ -1263,14 +1263,63 @@ jobs:
       use-artifact: ${{ steps.strategy.outputs.use-artifact }}
 
   job_e2e_tests:
-    name: E2E Tests (${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
+    name: E2E Tests (${{ matrix.projectName }} ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
     runs-on: ubuntu-latest
     needs: [job_build_e2e_image, job_setup]
     strategy:
       fail-fast: true
       matrix:
-        shardIndex: [1, 2, 3, 4, 5, 6, 7, 8]
-        shardTotal: [8]
+        include:
+          - projectName: Main
+            projects: main
+            analytics: 'false'
+            shardIndex: 1
+            shardTotal: 8
+          - projectName: Main
+            projects: main
+            analytics: 'false'
+            shardIndex: 2
+            shardTotal: 8
+          - projectName: Main
+            projects: main
+            analytics: 'false'
+            shardIndex: 3
+            shardTotal: 8
+          - projectName: Main
+            projects: main
+            analytics: 'false'
+            shardIndex: 4
+            shardTotal: 8
+          - projectName: Main
+            projects: main
+            analytics: 'false'
+            shardIndex: 5
+            shardTotal: 8
+          - projectName: Main
+            projects: main
+            analytics: 'false'
+            shardIndex: 6
+            shardTotal: 8
+          - projectName: Main
+            projects: main
+            analytics: 'false'
+            shardIndex: 7
+            shardTotal: 8
+          - projectName: Main
+            projects: main
+            analytics: 'false'
+            shardIndex: 8
+            shardTotal: 8
+          - projectName: Analytics
+            projects: analytics
+            analytics: 'true'
+            shardIndex: 1
+            shardTotal: 2
+          - projectName: Analytics
+            projects: analytics
+            analytics: 'true'
+            shardIndex: 2
+            shardTotal: 2
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -1279,6 +1328,7 @@ jobs:
         uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4
 
       - name: Pull or build Tinybird CLI Image
+        if: matrix.analytics == 'true'
         run: |
           COMPOSE_IMAGE="${COMPOSE_PROJECT_NAME:-ghost-dev}-tb-cli"
           # Try pulling pre-built image from GHCR first (fast path)
@@ -1314,13 +1364,15 @@ jobs:
         env:
           GHOST_E2E_IMAGE: ${{ steps.load.outputs.image-tag }}
           GHOST_E2E_SKIP_IMAGE_BUILD: 'true'
+          GHOST_E2E_ANALYTICS: ${{ matrix.analytics }}
         run: bash ./e2e/scripts/prepare-ci-e2e-job.sh
 
       - name: Run e2e tests in Playwright container
         env:
           TEST_WORKERS_COUNT: 1
           GHOST_E2E_MODE: build
           GHOST_E2E_IMAGE: ${{ steps.load.outputs.image-tag }}
+          E2E_PLAYWRIGHT_PROJECTS: ${{ matrix.projects }}
           E2E_SHARD_INDEX: ${{ matrix.shardIndex }}
           E2E_SHARD_TOTAL: ${{ matrix.shardTotal }}
           E2E_RETRIES: 2
@@ -1332,21 +1384,23 @@ jobs:
 
       - name: Stop E2E infra
         if: always()
+        env:
+          GHOST_E2E_ANALYTICS: ${{ matrix.analytics }}
         run: pnpm --filter @tryghost/e2e infra:down
 
       - name: Upload blob report to GitHub Actions Artifacts
         if: failure()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
-          name: blob-report-${{ matrix.shardIndex }}
+          name: blob-report-${{ matrix.projectName }}-${{ matrix.shardIndex }}
           path: e2e/blob-report
           retention-days: 1
 
       - name: Upload test results artifacts
         if: failure()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
-          name: test-results-${{ matrix.shardIndex }}
+          name: test-results-${{ matrix.projectName }}-${{ matrix.shardIndex }}
           path: e2e/test-results
           retention-days: 7
 

e2e/helpers/environment/environment-manager.ts

@@ -23,9 +23,9 @@ type GhostEnvOverrides = GhostConfig | Record<string, string>;
  * - dev: Uses dev infrastructure with hot-reloading
  * - build: Uses pre-built image (set GHOST_E2E_IMAGE for registry images)
  * 
- * All modes use the same infrastructure (MySQL, Redis, Mailpit, Tinybird)
- * started via docker compose. Ghost and gateway containers are created
- * dynamically per-worker for test isolation.
+ * All modes use the same core infrastructure (MySQL, Redis, Mailpit) started
+ * via docker compose. Analytics/Tinybird services are optional. Ghost and
+ * gateway containers are created dynamically per-worker for test isolation.
  */
 export class EnvironmentManager {
     private readonly mode: EnvironmentMode;

e2e/helpers/environment/service-availability.ts

@@ -21,6 +21,11 @@ async function isServiceAvailable(docker: Docker, serviceName: string) {
  * Checks for tinybird-local service in ghost-dev compose project.
  */
 export async function isTinybirdAvailable(): Promise<boolean> {
+    if (process.env.GHOST_E2E_ANALYTICS === 'false') {
+        debug('Tinybird disabled by GHOST_E2E_ANALYTICS=false');
+        return false;
+    }
+
     const docker = new Docker();
     const tinybirdAvailable = await isServiceAvailable(docker, TINYBIRD.LOCAL_HOST);
     debug(`Tinybird availability for compose project ${DEV_ENVIRONMENT.projectNamespace}:`, tinybirdAvailable);

e2e/scripts/infra-down.sh

@@ -6,5 +6,14 @@ REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
 
 cd "$REPO_ROOT"
 
-docker compose -f compose.dev.yaml -f compose.dev.analytics.yaml stop \
-  analytics tb-cli tinybird-local mailpit redis mysql
+ANALYTICS_ENABLED="${GHOST_E2E_ANALYTICS:-true}"
+
+compose_files=(-f compose.dev.yaml)
+services=(mailpit redis mysql)
+
+if [[ "$ANALYTICS_ENABLED" == "true" ]]; then
+  compose_files+=(-f compose.dev.analytics.yaml)
+  services=(analytics tb-cli tinybird-local "${services[@]}")
+fi
+
+docker compose "${compose_files[@]}" stop "${services[@]}"

e2e/scripts/infra-up.sh

@@ -9,6 +9,7 @@ cd "$REPO_ROOT"
 
 MODE="$(resolve_e2e_mode)"
 export GHOST_E2E_MODE="$MODE"
+ANALYTICS_ENABLED="${GHOST_E2E_ANALYTICS:-true}"
 
 if [[ "$MODE" != "build" ]]; then
   DEV_COMPOSE_PROJECT="${COMPOSE_PROJECT_NAME:-ghost-dev}"
@@ -21,5 +22,12 @@ if [[ "$MODE" != "build" ]]; then
   fi
 fi
 
-docker compose -f compose.dev.yaml -f compose.dev.analytics.yaml up -d --wait \
-  mysql redis mailpit tinybird-local analytics
+compose_files=(-f compose.dev.yaml)
+services=(mysql redis mailpit)
+
+if [[ "$ANALYTICS_ENABLED" == "true" ]]; then
+  compose_files+=(-f compose.dev.analytics.yaml)
+  services+=(tinybird-local analytics)
+fi
+
+docker compose "${compose_files[@]}" up -d --wait "${services[@]}"

e2e/scripts/prepare-ci-e2e-build-mode.sh

@@ -3,10 +3,12 @@ set -euo pipefail
 
 source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/load-playwright-container-env.sh"
 GATEWAY_IMAGE="${GHOST_E2E_GATEWAY_IMAGE:-caddy:2-alpine}"
+ANALYTICS_ENABLED="${GHOST_E2E_ANALYTICS:-true}"
 
 echo "Preparing E2E build-mode runtime"
 echo "Playwright image: ${PLAYWRIGHT_IMAGE}"
 echo "Gateway image: ${GATEWAY_IMAGE}"
+echo "Analytics enabled: ${ANALYTICS_ENABLED}"
 
 pids=()
 labels=()
@@ -25,7 +27,7 @@ run_bg() {
 
 run_bg "pull-gateway-image" docker pull "$GATEWAY_IMAGE"
 run_bg "pull-playwright-image" docker pull "$PLAYWRIGHT_IMAGE"
-run_bg "start-infra" env GHOST_E2E_MODE=build bash "$REPO_ROOT/e2e/scripts/infra-up.sh"
+run_bg "start-infra" env GHOST_E2E_MODE=build GHOST_E2E_ANALYTICS="$ANALYTICS_ENABLED" bash "$REPO_ROOT/e2e/scripts/infra-up.sh"
 
 for i in "${!pids[@]}"; do
     if ! wait "${pids[$i]}"; then
@@ -34,4 +36,6 @@ for i in "${!pids[@]}"; do
     fi
 done
 
-node "$REPO_ROOT/e2e/scripts/sync-tinybird-state.mjs"
+if [[ "$ANALYTICS_ENABLED" == "true" ]]; then
+    node "$REPO_ROOT/e2e/scripts/sync-tinybird-state.mjs"
+fi

e2e/scripts/run-playwright-container.sh

@@ -4,6 +4,7 @@ set -euo pipefail
 SHARD_INDEX="${E2E_SHARD_INDEX:-}"
 SHARD_TOTAL="${E2E_SHARD_TOTAL:-}"
 RETRIES="${E2E_RETRIES:-2}"
+PROJECTS="${E2E_PLAYWRIGHT_PROJECTS:-main,analytics}"
 
 if [[ -z "$SHARD_INDEX" || -z "$SHARD_TOTAL" ]]; then
     echo "Missing E2E_SHARD_INDEX or E2E_SHARD_TOTAL environment variables" >&2
@@ -12,6 +13,22 @@ fi
 
 source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/load-playwright-container-env.sh"
 
+project_args=()
+IFS=',' read -ra project_names <<< "$PROJECTS"
+for project in "${project_names[@]}"; do
+    project="${project//[[:space:]]/}"
+    if [[ -n "$project" ]]; then
+        project_args+=("--project=${project}")
+    fi
+done
+
+if [[ ${#project_args[@]} -eq 0 ]]; then
+    echo "No Playwright projects configured in E2E_PLAYWRIGHT_PROJECTS" >&2
+    exit 1
+fi
+
+printf -v project_args_string '%q ' "${project_args[@]}"
+
 docker run --rm --network host --ipc host \
   -v /var/run/docker.sock:/var/run/docker.sock \
   -v "${WORKSPACE_PATH}:${WORKSPACE_PATH}" \
@@ -22,5 +39,6 @@ docker run --rm --network host --ipc host \
   -e GHOST_E2E_MODE="${GHOST_E2E_MODE:-build}" \
   -e GHOST_E2E_IMAGE="${GHOST_E2E_IMAGE:-ghost-e2e:local}" \
   -e GHOST_E2E_GATEWAY_IMAGE="${GHOST_E2E_GATEWAY_IMAGE:-caddy:2-alpine}" \
+  -e GHOST_E2E_ANALYTICS="${GHOST_E2E_ANALYTICS:-true}" \
   "$PLAYWRIGHT_IMAGE" \
-  bash -c "corepack enable && pnpm test:all --shard=${SHARD_INDEX}/${SHARD_TOTAL} --retries=${RETRIES}"
+  bash -c "corepack enable && bash ./scripts/run-playwright-host.sh playwright test ${project_args_string}--shard=${SHARD_INDEX}/${SHARD_TOTAL} --retries=${RETRIES}"