feat(observability): opt-in capture of AI prompts and completions (#1… #2004
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Self-host stack CI (#980/#982). Provides integration coverage the main CI can't: | |
| # 1. Postgres integration test — needs a real PG service container | |
| # 2. Self-host bundle build validation (build-selfhost.ts) | |
| # 3. Docker image build + container smoke test (/health, /ready, /metrics) | |
| # Unit tests and typecheck are NOT duplicated here — the main CI validate job covers them. | |
| # | |
| # Push-to-main only since 2026-07-24 (was also per-PR): at this repo's PR volume the ~5-minute | |
| # build-boot fired ~100x/week (its path list includes migrations/**, which nearly every backend PR | |
| # touches) and was a top runner-queue consumer. A boot-breaking merge is caught by this run minutes | |
| # after landing, and self-host users only ever consume tagged releases (release-selfhost.yml), never | |
| # main. Revisit if a hosted deploy ever tracks main directly. | |
| # | |
| # #10138 CORRECTION. This header used to claim ci.yml validated "the selfhost pg integration suites" | |
| # pre-merge. It does not, and cannot: those suites require a real Postgres, ci.yml has no service | |
| # container, and without PG_TEST_URL they SKIP. The only place they run is this workflow -- post-merge. | |
| # | |
| # That gap shipped an outage. Migration 0209 used SQLite `AUTOINCREMENT`, which Postgres cannot parse, | |
| # so every Orb upgrading past 0208 crash-looped on boot. The "applies every migration" test here caught | |
| # it correctly on the very first push -- and then this workflow stayed red across five consecutive runs | |
| # while PRs kept merging, because a post-merge failure blocks nothing and pages no one. | |
| # | |
| # The pre-merge counterpart is test/unit/migration-dialect-portability.test.ts: it applies translateDdl | |
| # to every migration and fails if a SQLite-only construct survives. No Postgres, no service container, | |
| # ~11ms inside the unit run that already happens -- deliberately NOT a second PG job, because CI | |
| # runtime is the constraint that moved this workflow post-merge in the first place. It proves parseable, | |
| # not correct; this workflow remains the only real-Postgres coverage, and its red is worth acting on. | |
| name: self-host | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - "src/selfhost/**" | |
| - "src/server.ts" | |
| - "scripts/build-selfhost.ts" | |
| - "scripts/validate-selfhost-sourcemap.ts" | |
| - "Dockerfile" | |
| - "docker-compose.yml" | |
| - "migrations/**" | |
| - "test/unit/selfhost-*" | |
| - "test/integration/selfhost-pg*" | |
| - ".github/workflows/selfhost.yml" | |
| # Least privilege — the smoke test only reads the repo; no writes, no packages. | |
| permissions: | |
| contents: read | |
| concurrency: | |
| # Push-only workflow: sha-scoped so distinct main commits never cancel each other's validation. | |
| group: selfhost-${{ github.sha }} | |
| # NB: keep this a literal boolean, not an expression — see ci.yml for why (startup_failure). | |
| cancel-in-progress: true | |
| jobs: | |
| build-boot: | |
| name: build + boot smoke test | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| env: | |
| # Throwaway credential for the job-scoped ephemeral Postgres SERVICE CONTAINER below — deliberately | |
| # not a repository secret: the container exists only for this job's lifetime, is reachable only from | |
| # the runner's localhost, and holds nothing but synthetic test rows. Single-sourced here so the | |
| # service env and the test step can never drift. | |
| PG_TEST_PASSWORD: devpw | |
| services: | |
| postgres: | |
| image: postgres:18-alpine | |
| env: | |
| POSTGRES_PASSWORD: ${{ env.PG_TEST_PASSWORD }} | |
| POSTGRES_DB: loopover | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd "pg_isready -U postgres" --health-interval 5s --health-timeout 5s --health-retries 10 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 | |
| with: | |
| node-version-file: .nvmrc | |
| cache: "npm" | |
| - name: Install deps | |
| run: npm ci --ignore-scripts | |
| # Same #ci-engine-build-order fix as ci.yml (see its "Build engine package" step): the Postgres | |
| # integration test's import graph now reaches @loopover/engine transitively (#5117 moved | |
| # local-write action specs there; src/mcp/local-write-tools.ts re-exports them). That package's | |
| # dist/ is gitignored and only exists after this build step, so the test fails to resolve the | |
| # package's exports without it -- this workflow never needed the engine package built before, so it | |
| # never had this step; it does now. | |
| # Built before the engine for the same reason the Dockerfile does: src/'s import graph reaches | |
| # @loopover/contract, whose package exports resolve to dist/, so anything type-checking or | |
| # bundling src/ needs it emitted first. Zod-only leaf, no workspace dependencies of its own. | |
| - name: Build contract package | |
| run: npm run build --workspace @loopover/contract | |
| - name: Build engine package | |
| run: npm run build --workspace @loopover/engine | |
| - name: Postgres integration test (real PG) | |
| run: PG_TEST_URL=postgres://postgres:${{ env.PG_TEST_PASSWORD }}@localhost:5432/loopover npx vitest run test/integration/selfhost-pg.test.ts test/integration/selfhost-pg-calibration.test.ts | |
| - name: Build the self-host bundle | |
| run: node --experimental-strip-types scripts/build-selfhost.ts | |
| - name: Validate self-host source map | |
| run: node scripts/validate-selfhost-sourcemap.ts | |
| - name: Validate docker-compose.yml | |
| run: | | |
| docker compose config --quiet | |
| docker compose --profile workflows --profile storage config --quiet | |
| - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 | |
| - name: Build the Docker image | |
| run: | | |
| docker buildx build \ | |
| --cache-from type=gha \ | |
| --cache-to type=gha,mode=max \ | |
| --load \ | |
| -t loopover:selfhost-ci . | |
| - name: Smoke-test bundled AI CLIs | |
| run: | | |
| docker run --rm --entrypoint sh loopover:selfhost-ci -c \ | |
| 'command -v claude && claude --version && command -v codex && codex --version' | |
| - name: Free disk before visual-review image build | |
| run: | | |
| docker image prune -f | |
| docker builder prune -f --filter until=24h || true | |
| - name: Build release target with visual review deps | |
| run: | | |
| docker buildx build \ | |
| --target runtime-prebuilt \ | |
| --build-context selfhost_dist=./dist \ | |
| --build-arg INSTALL_VISUAL_REVIEW=true \ | |
| --load \ | |
| -t loopover:selfhost-prebuilt-visual-ci . | |
| docker run --rm --entrypoint node loopover:selfhost-prebuilt-visual-ci \ | |
| -e "import('puppeteer-core').then(() => console.log('puppeteer-core ok'))" | |
| docker rmi loopover:selfhost-prebuilt-visual-ci || true | |
| docker image prune -f | |
| - name: Boot the container + smoke-test /health, /ready, /metrics, migrations | |
| run: | | |
| docker network create gt-smoke | |
| docker run -d --name gt-redis --network gt-smoke redis:7-alpine | |
| trap 'docker rm -f gt gt-redis >/dev/null 2>&1 || true; docker network rm gt-smoke >/dev/null 2>&1 || true' EXIT | |
| ok=0 | |
| for _ in $(seq 1 30); do | |
| if docker exec gt-redis redis-cli ping | grep -q PONG; then ok=1; break; fi | |
| sleep 1 | |
| done | |
| if [ "$ok" != "1" ]; then echo "::error::redis did not become ready"; docker logs gt-redis; exit 1; fi | |
| docker run -d --name gt --network gt-smoke -p 8787:8787 \ | |
| -e REDIS_URL=redis://gt-redis:6379 \ | |
| -e SELFHOST_SETUP_TOKEN=selfhost-ci-setup-token \ | |
| -e PUBLIC_API_ORIGIN=https://selfhost-ci.example \ | |
| loopover:selfhost-ci | |
| ok=0 | |
| for _ in $(seq 1 30); do | |
| if curl -sf http://127.0.0.1:8787/health >/dev/null; then ok=1; break; fi | |
| sleep 2 | |
| done | |
| if [ "$ok" != "1" ]; then echo "::error::container did not become healthy"; docker logs gt; exit 1; fi | |
| curl -sf http://127.0.0.1:8787/health | grep -q '"status":"ok"' | |
| curl -sf http://127.0.0.1:8787/ready | grep -q '"ok":true' | |
| curl -sf http://127.0.0.1:8787/metrics | grep -q 'loopover_uptime_seconds' | |
| docker logs gt 2>&1 | grep -q 'selfhost_migrations_applied' | |
| echo "self-host smoke test passed" | |
| # #10146: a post-merge workflow going red blocks nothing and pages no one. This one caught migration 0209's | |
| # SQLite-only AUTOINCREMENT correctly on the very first push (#10138) and then stayed red across five | |
| # consecutive runs while PRs kept merging, because the only signal was a red check on a branch nobody was | |
| # watching. #9951 had already solved this exact class for the publish workflows; the escalation is now a | |
| # shared script rather than a second copy of it. | |
| # | |
| # Deliberately `failure()` and not `always()`: a success must never file anything. Deliberately consecutive | |
| # -- one red run is a flake and alerting on it is how an alert gets muted. | |
| escalate-persistent-failure: | |
| name: escalate persistent failure | |
| needs: build-boot | |
| if: ${{ failure() }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| permissions: | |
| contents: read | |
| # The one thing this job does that build-boot cannot: file the tracking issue. | |
| issues: write | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| persist-credentials: false | |
| # Needed only for `node --experimental-strip-types` below (the escalation is a .ts script), and pinned to | |
| # the same SHA as the other 32 setup-node uses in this repo -- including build-boot above -- so this job | |
| # adds no dependency that was not already here. | |
| # | |
| # A scanner flagged this as a filesystem-read risk, citing `__tests__/authutil.test.ts` in setup-node's | |
| # own repository. That is a test fixture, not runtime behaviour, and the feature it covers -- writing an | |
| # auth token into .npmrc -- only engages when `registry-url` is supplied. It is supplied in the five | |
| # publish-*.yml workflows and deliberately not here, so that path is inert in this job. Reviewed and | |
| # dismissed rather than silenced; re-check if this step ever gains `registry-url`. | |
| - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 | |
| with: | |
| node-version-file: .nvmrc | |
| - name: Escalate if this workflow has failed on consecutive runs | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: node --experimental-strip-types scripts/escalate-workflow-outage.ts --workflow selfhost.yml |