diff --git a/.changeset/cli-runner-fetch-resilience.md b/.changeset/cli-runner-fetch-resilience.md new file mode 100644 index 000000000..1f9da5ff7 --- /dev/null +++ b/.changeset/cli-runner-fetch-resilience.md @@ -0,0 +1,9 @@ +--- +'@walkeros/cli': patch +--- + +The managed flow runner now retries its bundle, config, and secret fetches on +transient failures (timeouts, network errors, 5xx) with bounded, jittered +backoff capped well inside the container health window, and the secret fetch is +now bounded by a timeout. A brief outage while a flow container starts no longer +hard-fails the run. diff --git a/.changeset/cli-runtime-log-rings.md b/.changeset/cli-runtime-log-rings.md new file mode 100644 index 000000000..a0b70b3ca --- /dev/null +++ b/.changeset/cli-runtime-log-rings.md @@ -0,0 +1,8 @@ +--- +'@walkeros/cli': patch +--- + +The managed flow runner now reports its recent errors and recent log output in +its heartbeat, so deployed flows can surface runtime errors and logs in the app +without any external log tooling. Secrets are redacted before leaving the +runner. diff --git a/.changeset/cli-runtime-observe-frozen-env.md b/.changeset/cli-runtime-observe-frozen-env.md new file mode 100644 index 000000000..1823ac19d --- /dev/null +++ b/.changeset/cli-runtime-observe-frozen-env.md @@ -0,0 +1,9 @@ +--- +'@walkeros/cli': patch +--- + +`walkeros run` reads two new environment variables. `WALKEROS_OBSERVE_LEVEL` +sets the runtime's baseline telemetry level (`off`, `standard`, or `trace`). +`WALKEROS_CONFIG_FROZEN` (`1` or `true`) serves the bundle as an immutable +snapshot: secrets are still injected at boot, but config hot-swap and heartbeat +are disabled. diff --git a/.changeset/cli-simulate-data-injection.md b/.changeset/cli-simulate-data-injection.md new file mode 100644 index 000000000..65779eb05 --- /dev/null +++ b/.changeset/cli-simulate-data-injection.md @@ -0,0 +1,10 @@ +--- +'@walkeros/cli': patch +--- + +All four simulate functions (`simulateSource`, `simulateTransformer`, +`simulateCollector`, `simulateDestination`) accept a new `data` option to run an +existing bundle with updated configuration values, without rebundling. The new +`buildDataPayload`, `classifyStepProperties`, and `containsCodeMarkers` exports +build and inspect that payload. Destination simulation results now include +`mappingKey`, the entity-action key of the matched mapping rule. diff --git a/.changeset/collector-consent-init-gate.md b/.changeset/collector-consent-init-gate.md new file mode 100644 index 000000000..1a96f2064 --- /dev/null +++ b/.changeset/collector-consent-init-gate.md @@ -0,0 +1,9 @@ +--- +'@walkeros/collector': patch +--- + +Enforce consent gating on destination initialization. A destination that +declares a consent requirement is never initialized while that consent is +denied, including the path that flushes queued `on` (consent) signals. +Initialization is now fail-closed: it requires an affirmative consent decision +from the caller, so a destination cannot load or send under denied consent. diff --git a/.changeset/core-simulation-mapping-key.md b/.changeset/core-simulation-mapping-key.md new file mode 100644 index 000000000..62a796947 --- /dev/null +++ b/.changeset/core-simulation-mapping-key.md @@ -0,0 +1,7 @@ +--- +'@walkeros/core': patch +--- + +Simulation step results gain an optional `mappingKey` field reporting the +entity-action key of the mapping rule a destination matched during simulation. +The field is additive and present only when a rule matched. diff --git a/.changeset/session-ungated-run-emit.md b/.changeset/session-ungated-run-emit.md new file mode 100644 index 000000000..b3f991ba6 --- /dev/null +++ b/.changeset/session-ungated-run-emit.md @@ -0,0 +1,9 @@ +--- +'@walkeros/web-source-session': patch +--- + +Fix `session start` being dropped when the collector starts with `run: false` +and no consent requirement. Without a consent rule the source emitted during +init, before the collector was allowed, so the event never reached destinations. +The emit now waits for the run lifecycle, matching the browser source's page +view timing, so it lands reliably once the collector runs. diff --git a/.changeset/usercentrics-official-consent.md b/.changeset/usercentrics-official-consent.md new file mode 100644 index 000000000..16eb88557 --- /dev/null +++ b/.changeset/usercentrics-official-consent.md @@ -0,0 +1,10 @@ +--- +'@walkeros/web-source-cmp-usercentrics': patch +--- + +Use the official Usercentrics events (UC_UI_INITIALIZED, UC_UI_CMP_EVENT) and +consent getters so a returning visitor's prior choice is applied on load and +first-visit defaults stay suppressed under explicitOnly. The configurable +eventName data-layer setting is removed; the source now uses the always-emitted +official events. Fixes consent-change events being dropped on the current +Usercentrics Web CMP. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 91f4c5d96..667c84460 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,7 +3,7 @@ name: Build and test on: push: branches: - - '*' + - main paths-ignore: - '**/*.md' - '**/*.mdx' @@ -39,7 +39,19 @@ jobs: with: node-version: '24.15.0' cache: 'npm' - - run: npm ci + - name: Cache node_modules + id: node_modules_cache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + node_modules + packages/*/node_modules + packages/**/node_modules + apps/*/node_modules + key: nm-${{ runner.os }}-node24.15.0-${{ hashFiles('package-lock.json') }} + - name: Install dependencies + if: steps.node_modules_cache.outputs.cache-hit != 'true' + run: npm ci # Drift gate: the committed CLI api.gen.d.ts (and spec.json) must equal a # fresh, prettier-formatted generate:types run. Prevents stale/unformatted # generated API types from being committed (the recurring eternal-diff). @@ -59,8 +71,7 @@ jobs: if: github.event_name != 'pull_request' run: npx turbo run typecheck lint test - name: Run integration tests - if: - github.event_name == 'pull_request' || github.ref == 'refs/heads/main' + if: github.event_name != 'pull_request' run: npm run test:integration - name: Upload test results on failure if: failure() diff --git a/apps/explorer/turbo.json b/apps/explorer/turbo.json index 76f2e42d0..ae243a7ec 100644 --- a/apps/explorer/turbo.json +++ b/apps/explorer/turbo.json @@ -2,7 +2,7 @@ "extends": ["//"], "tasks": { "test": { - "dependsOn": ["build"], + "dependsOn": ["build", "^build"], "inputs": ["$TURBO_DEFAULT$", "dist/**"] } } diff --git a/package.json b/package.json index 332f8b0aa..b6cdafb4c 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "verify:touched": "bash scripts/verify-touched.sh", "verify:affected": "turbo run typecheck lint test --filter='[origin/main]'", "verify:force": "turbo run test --filter='[origin/main]' --force", - "// scripts": "verify:touched for L1, verify:affected for L2, test:smoke for L3, test+lint+typecheck for L4, verify:force = uncached test run after a core-runtime change (turbo test cache hashes only own src). See /workspaces/developer/AGENT.md rule 11.", + "// scripts": "verify:touched for L1, verify:affected for L2, test:smoke for L3, test+lint+typecheck for L4, verify:force = rarely needed escape hatch for a fully uncached test run; the turbo test task depends on ^build, so an upstream change busts downstream test caches automatically. See /workspaces/developer/AGENT.md rule 11.", "prepare": "husky || true", "validate:links": "npx tsx apps/scripts/validate-links.ts", "validate:docs": "npx tsx apps/scripts/validate-docs.ts", diff --git a/packages/cli/openapi/spec.json b/packages/cli/openapi/spec.json index 6c2f3f021..9d5f7fdc5 100644 --- a/packages/cli/openapi/spec.json +++ b/packages/cli/openapi/spec.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "walkerOS Tag Manager API", - "version": "1.1.0", + "version": "2.1.0", "description": "API for managing walkerOS flows, projects, and real-time event observation.", "contact": { "name": "elbwalker", @@ -344,8 +344,11 @@ "enum": ["web", "server"], "example": "web" }, - "deploymentStatus": { - "type": ["string", "null"] + "serving": { + "$ref": "#/components/schemas/ServingStatus" + }, + "latestAttempt": { + "$ref": "#/components/schemas/LatestAttemptStatus" }, "deploymentUrl": { "type": ["string", "null"] @@ -358,11 +361,28 @@ "id", "name", "platform", - "deploymentStatus", + "serving", + "latestAttempt", "deploymentUrl", "deployedAt" ] }, + "ServingStatus": { + "type": "string", + "enum": ["live", "none"] + }, + "LatestAttemptStatus": { + "type": ["string", "null"], + "enum": [ + "idle", + "deploying", + "published", + "active", + "stopped", + "failed", + null + ] + }, "Version": { "type": "object", "properties": { @@ -466,322 +486,681 @@ }, "required": ["userId", "email", "role", "createdAt"] }, - "ApiTokenSummary": { + "DeleteAccountRequest": { "type": "object", "properties": { - "id": { - "type": "string", - "example": "tok_a1b2c3d4" - }, - "name": { - "type": "string", - "example": "CI Pipeline" - }, - "prefix": { - "type": "string", - "example": "sk-walkeros-abcd" - }, - "origin": { - "type": "string", - "example": "manual" - }, - "projectId": { - "type": ["string", "null"], - "example": null - }, - "scopes": { - "type": ["array", "null"], - "items": { - "type": "string" - }, - "example": null - }, - "createdAt": { + "confirm": { "type": "string", - "format": "date-time", - "example": "2026-01-26T14:30:00.000Z" - }, - "lastUsedAt": { - "type": ["string", "null"], - "format": "date-time", - "example": "2026-01-26T14:30:00.000Z" - }, - "expiresAt": { - "type": ["string", "null"], - "format": "date-time", - "example": "2026-01-26T14:30:00.000Z" - }, - "revokedAt": { - "type": ["string", "null"], - "format": "date-time", - "example": "2026-01-26T14:30:00.000Z" + "minLength": 1, + "example": "me@example.com" } }, - "required": [ - "id", - "name", - "prefix", - "origin", - "projectId", - "scopes", - "createdAt", - "lastUsedAt", - "expiresAt", - "revokedAt" - ] + "required": ["confirm"] }, - "FlowSettingsDetail": { + "DeleteAccountBlocked": { "type": "object", "properties": { - "id": { - "type": "string", - "pattern": "^cfg_[a-zA-Z0-9_-]+$", - "example": "cfg_a1b2c3d4" - }, - "name": { - "type": "string" - }, - "platform": { - "type": "string", - "enum": ["web", "server"], - "example": "web" - }, - "config": { + "error": { "type": "object", - "additionalProperties": {} - }, - "deployment": { - "type": ["object", "null"], "properties": { - "id": { - "type": "string" - }, - "status": { - "type": "string" + "code": { + "type": "string", + "enum": ["SOLE_OWNER"], + "example": "SOLE_OWNER" }, - "type": { + "message": { "type": "string" }, - "containerUrl": { - "type": ["string", "null"] - }, - "publicUrl": { - "type": ["string", "null"] - }, - "errorMessage": { - "type": ["string", "null"] - }, - "createdAt": { - "type": "string", - "format": "date-time", - "example": "2026-01-26T14:30:00.000Z" - }, - "updatedAt": { - "type": "string", - "format": "date-time", - "example": "2026-01-26T14:30:00.000Z" + "details": { + "type": "object", + "properties": { + "projects": { + "type": "array", + "items": { + "type": "string" + }, + "example": ["proj_abc123"] + } + }, + "required": ["projects"] } }, - "required": ["id", "status", "type", "createdAt", "updatedAt"] - }, - "createdAt": { - "type": "string", - "format": "date-time", - "example": "2026-01-26T14:30:00.000Z" - }, - "updatedAt": { - "type": "string", - "format": "date-time", - "example": "2026-01-26T14:30:00.000Z" - } - }, - "required": [ - "id", - "name", - "platform", - "config", - "createdAt", - "updatedAt" - ] - }, - "DeploySettingsResponse": { - "type": "object", - "properties": { - "deploymentId": { - "type": "string" - }, - "settingsId": { - "type": "string", - "pattern": "^cfg_[a-zA-Z0-9_-]+$", - "example": "cfg_a1b2c3d4" - }, - "status": { - "type": "string" - } - }, - "required": ["deploymentId", "settingsId", "status"] - }, - "FlowDetailResponse": { - "type": "object", - "properties": { - "id": { - "type": "string", - "pattern": "^flow_[a-zA-Z0-9_-]+$", - "example": "flow_a1b2c3d4" - }, - "name": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "example": "my-website-flow" - }, - "config": { - "$ref": "#/components/schemas/FlowConfig" - }, - "settings": { - "type": "array", - "items": { - "$ref": "#/components/schemas/FlowSettingsEnriched" - } - }, - "bundleId": { - "type": ["string", "null"] - }, - "createdAt": { - "type": "string", - "format": "date-time", - "example": "2026-01-26T14:30:00.000Z" - }, - "updatedAt": { - "type": "string", - "format": "date-time", - "example": "2026-01-26T14:30:00.000Z" - }, - "deletedAt": { - "type": ["string", "null"], - "format": "date-time" + "required": ["code", "message", "details"] } }, - "required": [ - "id", - "name", - "config", - "createdAt", - "updatedAt", - "deletedAt" - ] + "required": ["error"] }, - "FlowSettingsEnriched": { + "AccountExportResponse": { "type": "object", "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "platform": { + "exportedAt": { "type": "string", - "enum": ["web", "server"] + "example": "2026-06-10T12:00:00.000Z" }, - "deployment": { - "type": ["object", "null"], + "profile": { + "type": "object", "properties": { "id": { - "type": "string" + "type": "string", + "example": "user_a1b2c3d4" }, - "slug": { - "type": "string" + "email": { + "type": "string", + "example": "me@example.com" }, - "status": { - "type": "string" + "displayName": { + "type": ["string", "null"] }, - "type": { + "createdAt": { "type": "string" }, - "target": { - "type": ["string", "null"] - }, - "containerUrl": { + "lastLoginAt": { "type": ["string", "null"] }, - "createdAt": { + "globalRole": { "type": "string", - "format": "date-time", - "example": "2026-01-26T14:30:00.000Z" + "example": "user" }, - "updatedAt": { - "type": ["string", "null"] + "traits": { + "type": "array", + "items": { + "type": "string" + } } }, "required": [ "id", - "slug", - "status", - "type", - "target", - "containerUrl", + "email", + "displayName", "createdAt", - "updatedAt" + "lastLoginAt", + "globalRole", + "traits" ] }, - "createdAt": { - "type": "string", - "format": "date-time", - "example": "2026-01-26T14:30:00.000Z" - }, - "updatedAt": { - "type": "string", - "format": "date-time", - "example": "2026-01-26T14:30:00.000Z" + "memberships": { + "type": "array", + "items": { + "type": "object", + "properties": { + "projectId": { + "type": "string" + }, + "projectName": { + "type": "string" + }, + "role": { + "type": "string" + }, + "joinedAt": { + "type": "string" + } + }, + "required": ["projectId", "projectName", "role", "joinedAt"] + } + }, + "apiTokens": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "projectId": { + "type": ["string", "null"] + }, + "origin": { + "type": "string" + }, + "createdAt": { + "type": "string" + }, + "lastUsedAt": { + "type": ["string", "null"] + }, + "expiresAt": { + "type": ["string", "null"] + }, + "revokedAt": { + "type": ["string", "null"] + } + }, + "required": [ + "id", + "name", + "projectId", + "origin", + "createdAt", + "lastUsedAt", + "expiresAt", + "revokedAt" + ] + } + }, + "sessions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "createdAt": { + "type": "string" + }, + "expiresAt": { + "type": "string" + }, + "lastTouchedAt": { + "type": "string" + } + }, + "required": ["id", "createdAt", "expiresAt", "lastTouchedAt"] + } + }, + "mcpTokens": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "createdAt": { + "type": "string" + }, + "lastUsedAt": { + "type": ["string", "null"] + }, + "expiresAt": { + "type": "string" + }, + "revokedAt": { + "type": ["string", "null"] + } + }, + "required": [ + "id", + "name", + "createdAt", + "lastUsedAt", + "expiresAt", + "revokedAt" + ] + } + }, + "mcpSessions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "projectId": { + "type": ["string", "null"] + }, + "createdAt": { + "type": "string" + }, + "lastActiveAt": { + "type": "string" + }, + "expiresAt": { + "type": "string" + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "properties": { + "seq": { + "type": "number" + }, + "role": { + "type": "string" + }, + "content": {}, + "createdAt": { + "type": "string" + } + }, + "required": ["seq", "role", "createdAt"] + } + } + }, + "required": [ + "id", + "projectId", + "createdAt", + "lastActiveAt", + "expiresAt", + "messages" + ] + } + }, + "feedback": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "projectId": { + "type": ["string", "null"] + }, + "text": { + "type": "string" + }, + "source": { + "type": "string" + }, + "createdAt": { + "type": "string" + } + }, + "required": ["id", "projectId", "text", "source", "createdAt"] + } + }, + "invitations": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "projectId": { + "type": "string" + }, + "invitedEmail": { + "type": "string" + }, + "role": { + "type": "string" + }, + "status": { + "type": "string" + }, + "createdAt": { + "type": "string" + }, + "expiresAt": { + "type": "string" + }, + "acceptedAt": { + "type": ["string", "null"] + }, + "declinedAt": { + "type": ["string", "null"] + }, + "cancelledAt": { + "type": ["string", "null"] + } + }, + "required": [ + "id", + "projectId", + "invitedEmail", + "role", + "status", + "createdAt", + "expiresAt", + "acceptedAt", + "declinedAt", + "cancelledAt" + ] + } } }, "required": [ - "id", - "name", - "platform", - "deployment", - "createdAt", - "updatedAt" + "exportedAt", + "profile", + "memberships", + "apiTokens", + "sessions", + "mcpTokens", + "mcpSessions", + "feedback", + "invitations" ] }, - "FlowUpdateResponse": { + "ApiTokenSummary": { "type": "object", "properties": { "id": { "type": "string", - "pattern": "^flow_[a-zA-Z0-9_-]+$", - "example": "flow_a1b2c3d4" + "example": "tok_a1b2c3d4" }, "name": { "type": "string", - "minLength": 1, - "maxLength": 255, - "example": "my-website-flow" + "example": "CI Pipeline" }, - "config": { - "$ref": "#/components/schemas/FlowConfig" + "prefix": { + "type": "string", + "example": "sk-walkeros-abcd" + }, + "origin": { + "type": "string", + "example": "manual" + }, + "projectId": { + "type": ["string", "null"], + "example": null + }, + "scopes": { + "type": ["array", "null"], + "items": { + "type": "string" + }, + "example": null }, "createdAt": { "type": "string", "format": "date-time", "example": "2026-01-26T14:30:00.000Z" }, - "updatedAt": { - "type": "string", + "lastUsedAt": { + "type": ["string", "null"], + "format": "date-time", + "example": "2026-01-26T14:30:00.000Z" + }, + "expiresAt": { + "type": ["string", "null"], + "format": "date-time", + "example": "2026-01-26T14:30:00.000Z" + }, + "revokedAt": { + "type": ["string", "null"], "format": "date-time", "example": "2026-01-26T14:30:00.000Z" } }, - "required": ["id", "name", "config", "createdAt", "updatedAt"] + "required": [ + "id", + "name", + "prefix", + "origin", + "projectId", + "scopes", + "createdAt", + "lastUsedAt", + "expiresAt", + "revokedAt" + ] }, - "CreateProjectResponse": { + "FlowSettingsDetail": { "type": "object", "properties": { "id": { "type": "string", - "pattern": "^proj_[a-zA-Z0-9_-]+$", - "example": "proj_x7y8z9" + "pattern": "^cfg_[a-zA-Z0-9_-]+$", + "example": "cfg_a1b2c3d4" + }, + "name": { + "type": "string" + }, + "platform": { + "type": "string", + "enum": ["web", "server"], + "example": "web" + }, + "config": { + "type": "object", + "additionalProperties": {} + }, + "deployment": { + "type": ["object", "null"], + "properties": { + "id": { + "type": "string" + }, + "status": { + "type": "string" + }, + "type": { + "type": "string" + }, + "containerUrl": { + "type": ["string", "null"] + }, + "publicUrl": { + "type": ["string", "null"] + }, + "errorMessage": { + "type": ["string", "null"] + }, + "createdAt": { + "type": "string", + "format": "date-time", + "example": "2026-01-26T14:30:00.000Z" + }, + "updatedAt": { + "type": "string", + "format": "date-time", + "example": "2026-01-26T14:30:00.000Z" + } + }, + "required": ["id", "status", "type", "createdAt", "updatedAt"] + }, + "createdAt": { + "type": "string", + "format": "date-time", + "example": "2026-01-26T14:30:00.000Z" + }, + "updatedAt": { + "type": "string", + "format": "date-time", + "example": "2026-01-26T14:30:00.000Z" + } + }, + "required": [ + "id", + "name", + "platform", + "config", + "createdAt", + "updatedAt" + ] + }, + "DeploySettingsResponse": { + "type": "object", + "properties": { + "deploymentId": { + "type": "string" + }, + "settingsId": { + "type": "string", + "pattern": "^cfg_[a-zA-Z0-9_-]+$", + "example": "cfg_a1b2c3d4" + }, + "status": { + "type": "string" + } + }, + "required": ["deploymentId", "settingsId", "status"] + }, + "FlowDetailResponse": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^flow_[a-zA-Z0-9_-]+$", + "example": "flow_a1b2c3d4" + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "example": "my-website-flow" + }, + "config": { + "$ref": "#/components/schemas/FlowConfig" + }, + "settings": { + "type": "array", + "items": { + "$ref": "#/components/schemas/FlowSettingsEnriched" + } + }, + "bundleId": { + "type": ["string", "null"] + }, + "createdAt": { + "type": "string", + "format": "date-time", + "example": "2026-01-26T14:30:00.000Z" + }, + "updatedAt": { + "type": "string", + "format": "date-time", + "example": "2026-01-26T14:30:00.000Z" + }, + "deletedAt": { + "type": ["string", "null"], + "format": "date-time" + } + }, + "required": [ + "id", + "name", + "config", + "createdAt", + "updatedAt", + "deletedAt" + ] + }, + "FlowSettingsEnriched": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "platform": { + "type": "string", + "enum": ["web", "server"] + }, + "deployment": { + "type": ["object", "null"], + "properties": { + "id": { + "type": "string" + }, + "slug": { + "type": "string" + }, + "status": { + "type": "string" + }, + "type": { + "type": "string" + }, + "target": { + "type": ["string", "null"] + }, + "containerUrl": { + "type": ["string", "null"] + }, + "createdAt": { + "type": "string", + "format": "date-time", + "example": "2026-01-26T14:30:00.000Z" + }, + "updatedAt": { + "type": ["string", "null"] + } + }, + "required": [ + "id", + "slug", + "status", + "type", + "target", + "containerUrl", + "createdAt", + "updatedAt" + ] + }, + "serving": { + "$ref": "#/components/schemas/ServingStatus" + }, + "latestAttempt": { + "$ref": "#/components/schemas/LatestAttemptStatus" + }, + "deployedAt": { + "type": ["string", "null"] + }, + "createdAt": { + "type": "string", + "format": "date-time", + "example": "2026-01-26T14:30:00.000Z" + }, + "updatedAt": { + "type": "string", + "format": "date-time", + "example": "2026-01-26T14:30:00.000Z" + } + }, + "required": [ + "id", + "name", + "platform", + "deployment", + "serving", + "latestAttempt", + "deployedAt", + "createdAt", + "updatedAt" + ] + }, + "FlowUpdateResponse": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^flow_[a-zA-Z0-9_-]+$", + "example": "flow_a1b2c3d4" + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "example": "my-website-flow" + }, + "config": { + "$ref": "#/components/schemas/FlowConfig" + }, + "createdAt": { + "type": "string", + "format": "date-time", + "example": "2026-01-26T14:30:00.000Z" + }, + "updatedAt": { + "type": "string", + "format": "date-time", + "example": "2026-01-26T14:30:00.000Z" + } + }, + "required": ["id", "name", "config", "createdAt", "updatedAt"] + }, + "CreateProjectResponse": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^proj_[a-zA-Z0-9_-]+$", + "example": "proj_x7y8z9" }, "name": { "type": "string" @@ -854,6 +1233,9 @@ ], "example": "active" }, + "serving": { + "$ref": "#/components/schemas/ServingStatus" + }, "currentVersionNumber": { "type": ["integer", "null"], "exclusiveMinimum": 0 @@ -877,10 +1259,6 @@ "type": "string", "format": "date-time" }, - "traceUntil": { - "type": ["string", "null"], - "format": "date-time" - }, "usageSummary": { "type": "object", "properties": { @@ -903,13 +1281,13 @@ "label", "origin", "status", + "serving", "currentVersionNumber", "url", "flowId", "flowName", "createdAt", - "updatedAt", - "traceUntil" + "updatedAt" ] }, "DeploymentDetailResponse": { @@ -965,6 +1343,45 @@ "error": { "$ref": "#/components/schemas/DeploymentError" }, + "recentErrors": { + "type": ["array", "null"], + "items": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "count": { + "type": "integer" + }, + "firstSeen": { + "type": "string" + }, + "lastSeen": { + "type": "string" + } + }, + "required": ["message", "count", "firstSeen", "lastSeen"] + } + }, + "recentLogs": { + "type": ["array", "null"], + "items": { + "type": "object", + "properties": { + "time": { + "type": "string" + }, + "level": { + "type": "string" + }, + "message": { + "type": "string" + } + }, + "required": ["time", "level", "message"] + } + }, "url": { "type": ["string", "null"] }, @@ -992,7 +1409,7 @@ "healthy" ] }, - "traceUntil": { + "lastHeartbeatAt": { "type": ["string", "null"], "format": "date-time" }, @@ -1018,7 +1435,6 @@ "error", "url", "selfHosted", - "traceUntil", "createdAt", "updatedAt" ] @@ -1176,6 +1592,9 @@ ], "example": "active" }, + "serving": { + "$ref": "#/components/schemas/ServingStatus" + }, "currentVersionNumber": { "type": ["integer", "null"], "exclusiveMinimum": 0 @@ -1199,10 +1618,6 @@ "type": "string", "format": "date-time" }, - "traceUntil": { - "type": ["string", "null"], - "format": "date-time" - }, "usageSummary": { "type": "object", "properties": { @@ -1225,13 +1640,123 @@ "label", "origin", "status", + "serving", "currentVersionNumber", "url", "flowId", "flowName", "createdAt", - "updatedAt", - "traceUntil" + "updatedAt" + ] + }, + "StartDeploymentResponse": { + "anyOf": [ + { + "type": "object", + "properties": { + "deploymentId": { + "type": "string", + "pattern": "^dep_[a-zA-Z0-9_-]+$", + "example": "dep_a1b2c3d4" + }, + "slug": { + "type": "string", + "pattern": "^[a-z0-9]{12}$", + "example": "k7m2x9p4q1w8" + }, + "target": { + "type": ["string", "null"] + }, + "type": { + "type": "string", + "enum": ["web", "server"], + "example": "web" + }, + "status": { + "type": "string", + "enum": ["deploying"] + }, + "settingsId": { + "type": "string" + }, + "versionId": { + "type": "string" + }, + "versionNumber": { + "type": "integer", + "exclusiveMinimum": 0 + } + }, + "required": [ + "deploymentId", + "slug", + "target", + "type", + "status", + "versionId", + "versionNumber" + ] + }, + { + "type": "object", + "properties": { + "deploymentId": { + "type": "string" + }, + "status": { + "type": "string", + "enum": ["already_created"] + } + }, + "required": ["deploymentId", "status"] + } + ] + }, + "DeploymentStreamStatusEvent": { + "type": "object", + "properties": { + "status": { + "type": "string" + }, + "substatus": { + "type": ["string", "null"] + }, + "type": { + "type": "string", + "enum": ["web", "server"], + "example": "web" + }, + "target": { + "type": ["string", "null"] + }, + "containerUrl": { + "type": ["string", "null"] + }, + "errorCode": { + "type": ["string", "null"] + }, + "errorMessage": { + "type": ["string", "null"] + }, + "createdAt": { + "type": "string", + "format": "date-time" + }, + "updatedAt": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "status", + "substatus", + "type", + "target", + "containerUrl", + "errorCode", + "errorMessage", + "createdAt", + "updatedAt" ] }, "ListDeploymentsResponse": { @@ -1651,6 +2176,99 @@ }, "required": ["flowSettingsId"] }, + "ObserveSessionResponse": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^ses_[a-zA-Z0-9_-]+$", + "example": "ses_abc123xyz456" + }, + "projectId": { + "type": "string" + }, + "flowId": { + "type": "string", + "pattern": "^flow_[a-zA-Z0-9_-]+$", + "example": "flow_a1b2c3d4" + }, + "status": { + "type": "string" + }, + "errorMessage": { + "type": ["string", "null"] + }, + "configSnapshot": { + "type": "object", + "additionalProperties": {} + }, + "serverFlowName": { + "type": ["string", "null"] + }, + "serverEndpoint": { + "type": ["string", "null"] + }, + "web": { + "$ref": "#/components/schemas/ObserveSessionWeb" + }, + "createdBy": { + "type": "string" + }, + "createdAt": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "id", + "projectId", + "flowId", + "status", + "errorMessage", + "configSnapshot", + "serverFlowName", + "serverEndpoint", + "web", + "createdBy", + "createdAt" + ] + }, + "ObserveSessionWeb": { + "type": ["object", "null"], + "properties": { + "token": { + "type": "string" + }, + "activationUrl": { + "type": "string" + }, + "bundleUrl": { + "type": "string", + "format": "uri" + } + }, + "required": ["token", "activationUrl", "bundleUrl"] + }, + "CreateObserveSessionRequest": { + "type": "object", + "properties": { + "settingsName": { + "type": "string", + "minLength": 1 + } + }, + "required": ["settingsName"] + }, + "ObserveSessionHeartbeatResponse": { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [true] + } + }, + "required": ["ok"] + }, "SecretName": { "type": "string", "minLength": 1, @@ -2745,29 +3363,6 @@ }, "required": ["ingestToken"] }, - "TraceDeploymentRequest": { - "type": "object", - "properties": { - "minutes": { - "type": "integer", - "minimum": 0 - } - } - }, - "TraceDeploymentResponse": { - "type": "object", - "properties": { - "traceUntil": { - "type": ["string", "null"], - "format": "date-time" - }, - "minutes": { - "type": "integer", - "minimum": 0 - } - }, - "required": ["traceUntil", "minutes"] - }, "DeploymentUsageResponse": { "type": "object", "properties": { @@ -2811,6 +3406,12 @@ "items": { "$ref": "#/components/schemas/UsageBucket" } + }, + "destinations": { + "type": "array", + "items": { + "$ref": "#/components/schemas/UsageDestination" + } } }, "required": [ @@ -2858,6 +3459,42 @@ "instances" ] }, + "UsageDestination": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "count": { + "type": "integer", + "minimum": 0 + }, + "failed": { + "type": "integer", + "minimum": 0 + }, + "duration": { + "type": "number", + "minimum": 0 + }, + "dlqSize": { + "type": "integer", + "minimum": 0 + }, + "dropped": { + "type": "integer", + "minimum": 0 + } + }, + "required": [ + "name", + "count", + "failed", + "duration", + "dlqSize", + "dropped" + ] + }, "CreateCustomDomainRequest": { "type": "object", "properties": { @@ -3930,23 +4567,7 @@ "updatedAt" ] }, - "StartDeploymentResponse": { - "type": "object", - "properties": { - "deploymentId": { - "type": "string" - }, - "type": { - "type": "string", - "enum": ["web", "server"] - }, - "status": { - "type": "string" - } - }, - "required": ["deploymentId", "type", "status"] - }, - "ListSettingsResponse": { + "ListSettingsResponse": { "type": "object", "properties": { "settings": { @@ -4419,6 +5040,14 @@ "duration": { "type": "number", "minimum": 0 + }, + "dlqSize": { + "type": "integer", + "minimum": 0 + }, + "dropped": { + "type": "integer", + "minimum": 0 } }, "required": ["count", "failed", "duration"] @@ -4431,6 +5060,54 @@ "eventsFailed", "destinations" ] + }, + "recentErrors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "message": { + "type": "string", + "maxLength": 256 + }, + "count": { + "type": "integer", + "minimum": 1 + }, + "firstSeen": { + "type": "string", + "format": "date-time" + }, + "lastSeen": { + "type": "string", + "format": "date-time" + } + }, + "required": ["message", "count", "firstSeen", "lastSeen"] + }, + "maxItems": 50 + }, + "recentLogs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "time": { + "type": "string", + "format": "date-time" + }, + "level": { + "type": "string", + "enum": ["error", "warn", "info", "debug"] + }, + "message": { + "type": "string", + "maxLength": 256 + } + }, + "required": ["time", "level", "message"] + }, + "maxItems": 100 } }, "required": ["instanceId", "flowId"] @@ -4465,7 +5142,352 @@ } }, "400": { - "description": "Validation error", + "description": "Validation error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, + "/api/auth/verify": { + "get": { + "tags": ["Auth"], + "summary": "Verify magic link token", + "description": "Verify a magic link token and create an authenticated session. Redirects to the specified URL or home page.", + "parameters": [ + { + "schema": { + "type": "string", + "minLength": 1, + "example": "abc123...", + "description": "Magic link token" + }, + "required": true, + "description": "Magic link token", + "name": "token", + "in": "query" + }, + { + "schema": { + "type": "string", + "example": "/dashboard", + "description": "Redirect URL after verification" + }, + "required": false, + "description": "Redirect URL after verification", + "name": "redirect_to", + "in": "query" + } + ], + "responses": { + "302": { + "description": "Redirect to target URL with session cookie set" + } + } + } + }, + "/api/auth/logout": { + "post": { + "tags": ["Auth"], + "summary": "End session", + "description": "Destroy the current session and clear the session cookie.", + "responses": { + "302": { + "description": "Redirect to login page" + } + } + } + }, + "/api/auth/whoami": { + "get": { + "tags": ["Auth"], + "summary": "Current identity", + "description": "Return the identity of the authenticated user. Supports session cookie and Bearer token.", + "responses": { + "200": { + "description": "Current user identity", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WhoamiResponse" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, + "/api/account": { + "delete": { + "tags": ["Account"], + "summary": "Delete own account", + "description": "Soft-delete the authenticated account, starting the 30-day grace window. Requires a confirmation of the account email in the body. Revokes all sessions, API tokens, and MCP tokens. Blocked with 409 when the caller is the sole owner of a project that still has other members.", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteAccountRequest" + } + } + } + }, + "responses": { + "204": { + "description": "Account scheduled for deletion" + }, + "400": { + "description": "Confirmation email does not match", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "409": { + "description": "Sole owner of a shared project", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteAccountBlocked" + } + } + } + } + } + } + }, + "/api/account/export": { + "get": { + "tags": ["Account"], + "summary": "Export own account data", + "description": "Download a portable JSON export of everything the platform holds about the authenticated account: profile, memberships, token and session metadata, MCP sessions with messages, feedback, and invitations. Metadata only; token hashes and secret values are never included. Served as a file attachment.", + "responses": { + "200": { + "description": "Account data export (file attachment)", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccountExportResponse" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, + "/api/sessions": { + "get": { + "tags": ["Auth"], + "summary": "List sessions", + "description": "List all active sessions for the authenticated user. The current session is marked with isCurrent: true.", + "responses": { + "200": { + "description": "List of active sessions", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListSessionsResponse" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, + "/api/sessions/{id}": { + "delete": { + "tags": ["Auth"], + "summary": "Revoke session", + "description": "Revoke a session by ID. Cannot revoke the current session (use logout instead).", + "parameters": [ + { + "schema": { + "type": "string", + "example": "ses_abc123" + }, + "required": true, + "name": "id", + "in": "path" + } + ], + "responses": { + "204": { + "description": "Session revoked" + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "403": { + "description": "Forbidden", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, + "/api/auth/device/code": { + "post": { + "tags": ["Auth"], + "summary": "Request device code", + "description": "Generate a device code and user code for the device authorization flow.", + "responses": { + "200": { + "description": "Device code generated", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeviceCodeResponse" + } + } + } + } + } + } + }, + "/api/auth/device/approve": { + "post": { + "tags": ["Auth"], + "summary": "Approve device code", + "description": "Approve a device authorization request using the user code. Requires authentication.", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApproveDeviceRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Device approved", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApproveDeviceResponse" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, + "/api/auth/device/token": { + "post": { + "tags": ["Auth"], + "summary": "Poll device token", + "description": "Poll for authorization status using the device code. Returns a token when approved.", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeviceTokenRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Authorization approved", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeviceTokenResponse" + } + } + } + }, + "400": { + "description": "Pending, slow down, or expired", "content": { "application/json": { "schema": { @@ -4477,67 +5499,38 @@ } } }, - "/api/auth/verify": { + "/api/projects": { "get": { - "tags": ["Auth"], - "summary": "Verify magic link token", - "description": "Verify a magic link token and create an authenticated session. Redirects to the specified URL or home page.", + "tags": ["Projects"], + "summary": "List my projects", + "description": "List all projects where the authenticated user is a member.", "parameters": [ { "schema": { - "type": "string", - "minLength": 1, - "example": "abc123...", - "description": "Magic link token" + "type": "string" }, - "required": true, - "description": "Magic link token", - "name": "token", + "required": false, + "name": "cursor", "in": "query" }, { "schema": { - "type": "string", - "example": "/dashboard", - "description": "Redirect URL after verification" + "type": "integer", + "minimum": 1, + "maximum": 100 }, "required": false, - "description": "Redirect URL after verification", - "name": "redirect_to", + "name": "limit", "in": "query" } ], - "responses": { - "302": { - "description": "Redirect to target URL with session cookie set" - } - } - } - }, - "/api/auth/logout": { - "post": { - "tags": ["Auth"], - "summary": "End session", - "description": "Destroy the current session and clear the session cookie.", - "responses": { - "302": { - "description": "Redirect to login page" - } - } - } - }, - "/api/auth/whoami": { - "get": { - "tags": ["Auth"], - "summary": "Current identity", - "description": "Return the identity of the authenticated user. Supports session cookie and Bearer token.", "responses": { "200": { - "description": "Current user identity", + "description": "List of projects", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/WhoamiResponse" + "$ref": "#/components/schemas/ListProjectsResponse" } } } @@ -4553,20 +5546,37 @@ } } } - } - }, - "/api/sessions": { - "get": { - "tags": ["Auth"], - "summary": "List sessions", - "description": "List all active sessions for the authenticated user. The current session is marked with isCurrent: true.", + }, + "post": { + "tags": ["Projects"], + "summary": "Create project", + "description": "Create a new project. The authenticated user becomes the owner.", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateProjectRequest" + } + } + } + }, "responses": { - "200": { - "description": "List of active sessions", + "201": { + "description": "Project created", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ListSessionsResponse" + "$ref": "#/components/schemas/CreateProjectResponse" + } + } + } + }, + "400": { + "description": "Validation error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" } } } @@ -4584,38 +5594,36 @@ } } }, - "/api/sessions/{id}": { - "delete": { - "tags": ["Auth"], - "summary": "Revoke session", - "description": "Revoke a session by ID. Cannot revoke the current session (use logout instead).", + "/api/projects/{projectId}": { + "get": { + "tags": ["Projects"], + "summary": "Get project", + "description": "Get a single project by ID. Requires membership.", "parameters": [ { "schema": { "type": "string", - "example": "ses_abc123" + "pattern": "^proj_[a-zA-Z0-9_-]+$", + "example": "proj_x7y8z9" }, "required": true, - "name": "id", + "name": "projectId", "in": "path" } ], "responses": { - "204": { - "description": "Session revoked" - }, - "401": { - "description": "Unauthorized", + "200": { + "description": "Project details", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "$ref": "#/components/schemas/ProjectDetailResponse" } } } }, - "403": { - "description": "Forbidden", + "401": { + "description": "Unauthorized", "content": { "application/json": { "schema": { @@ -4635,48 +5643,49 @@ } } } - } - }, - "/api/auth/device/code": { - "post": { - "tags": ["Auth"], - "summary": "Request device code", - "description": "Generate a device code and user code for the device authorization flow.", - "responses": { - "200": { - "description": "Device code generated", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DeviceCodeResponse" - } - } - } + }, + "patch": { + "tags": ["Projects"], + "summary": "Update project", + "description": "Update project details. Requires owner role.", + "parameters": [ + { + "schema": { + "type": "string", + "pattern": "^proj_[a-zA-Z0-9_-]+$", + "example": "proj_x7y8z9" + }, + "required": true, + "name": "projectId", + "in": "path" } - } - } - }, - "/api/auth/device/approve": { - "post": { - "tags": ["Auth"], - "summary": "Approve device code", - "description": "Approve a device authorization request using the user code. Requires authentication.", + ], "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ApproveDeviceRequest" + "$ref": "#/components/schemas/UpdateProjectRequest" } } } }, "responses": { "200": { - "description": "Device approved", + "description": "Project updated", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ApproveDeviceResponse" + "$ref": "#/components/schemas/UpdateProjectResponse" + } + } + } + }, + "400": { + "description": "Validation error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" } } } @@ -4691,6 +5700,16 @@ } } }, + "403": { + "description": "Forbidden", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, "404": { "description": "Not found", "content": { @@ -4702,35 +5721,49 @@ } } } - } - }, - "/api/auth/device/token": { - "post": { - "tags": ["Auth"], - "summary": "Poll device token", - "description": "Poll for authorization status using the device code. Returns a token when approved.", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DeviceTokenRequest" - } - } + }, + "delete": { + "tags": ["Projects"], + "summary": "Delete project", + "description": "Delete a project and all its resources. Requires owner role.", + "parameters": [ + { + "schema": { + "type": "string", + "pattern": "^proj_[a-zA-Z0-9_-]+$", + "example": "proj_x7y8z9" + }, + "required": true, + "name": "projectId", + "in": "path" } - }, + ], "responses": { - "200": { - "description": "Authorization approved", + "204": { + "description": "Project deleted" + }, + "401": { + "description": "Unauthorized", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/DeviceTokenResponse" + "$ref": "#/components/schemas/ErrorResponse" } } } }, - "400": { - "description": "Pending, slow down, or expired", + "403": { + "description": "Forbidden", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "404": { + "description": "Not found", "content": { "application/json": { "schema": { @@ -4742,38 +5775,30 @@ } } }, - "/api/projects": { + "/api/projects/{projectId}/members": { "get": { "tags": ["Projects"], - "summary": "List my projects", - "description": "List all projects where the authenticated user is a member.", - "parameters": [ - { - "schema": { - "type": "string" - }, - "required": false, - "name": "cursor", - "in": "query" - }, + "summary": "List members", + "description": "List all members of a project. Requires membership.", + "parameters": [ { "schema": { - "type": "integer", - "minimum": 1, - "maximum": 100 + "type": "string", + "pattern": "^proj_[a-zA-Z0-9_-]+$", + "example": "proj_x7y8z9" }, - "required": false, - "name": "limit", - "in": "query" + "required": true, + "name": "projectId", + "in": "path" } ], "responses": { "200": { - "description": "List of projects", + "description": "List of members", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ListProjectsResponse" + "$ref": "#/components/schemas/ListMembersResponse" } } } @@ -4787,29 +5812,51 @@ } } } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } } } }, "post": { "tags": ["Projects"], - "summary": "Create project", - "description": "Create a new project. The authenticated user becomes the owner.", + "summary": "Add member", + "description": "Add a member to the project by email. Requires owner role.", + "parameters": [ + { + "schema": { + "type": "string", + "pattern": "^proj_[a-zA-Z0-9_-]+$", + "example": "proj_x7y8z9" + }, + "required": true, + "name": "projectId", + "in": "path" + } + ], "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CreateProjectRequest" + "$ref": "#/components/schemas/AddMemberRequest" } } } }, "responses": { "201": { - "description": "Project created", + "description": "Member added", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CreateProjectResponse" + "$ref": "#/components/schemas/Member" } } } @@ -4833,40 +5880,19 @@ } } } - } - } - } - }, - "/api/projects/{projectId}": { - "get": { - "tags": ["Projects"], - "summary": "Get project", - "description": "Get a single project by ID. Requires membership.", - "parameters": [ - { - "schema": { - "type": "string", - "pattern": "^proj_[a-zA-Z0-9_-]+$", - "example": "proj_x7y8z9" - }, - "required": true, - "name": "projectId", - "in": "path" - } - ], - "responses": { - "200": { - "description": "Project details", + }, + "403": { + "description": "Forbidden", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ProjectDetailResponse" + "$ref": "#/components/schemas/ErrorResponse" } } } }, - "401": { - "description": "Unauthorized", + "404": { + "description": "Not found", "content": { "application/json": { "schema": { @@ -4875,8 +5901,8 @@ } } }, - "404": { - "description": "Not found", + "409": { + "description": "Conflict", "content": { "application/json": { "schema": { @@ -4886,11 +5912,13 @@ } } } - }, + } + }, + "/api/projects/{projectId}/members/{userId}": { "patch": { "tags": ["Projects"], - "summary": "Update project", - "description": "Update project details. Requires owner role.", + "summary": "Update member role", + "description": "Update a member's role. Requires owner role.", "parameters": [ { "schema": { @@ -4901,24 +5929,33 @@ "required": true, "name": "projectId", "in": "path" + }, + { + "schema": { + "type": "string", + "example": "user_a1b2c3" + }, + "required": true, + "name": "userId", + "in": "path" } ], "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/UpdateProjectRequest" + "$ref": "#/components/schemas/UpdateMemberRequest" } } } }, "responses": { "200": { - "description": "Project updated", + "description": "Role updated", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/UpdateProjectResponse" + "$ref": "#/components/schemas/Member" } } } @@ -4967,8 +6004,8 @@ }, "delete": { "tags": ["Projects"], - "summary": "Delete project", - "description": "Delete a project and all its resources. Requires owner role.", + "summary": "Remove member", + "description": "Remove a member from the project. Requires owner role.", "parameters": [ { "schema": { @@ -4979,11 +6016,20 @@ "required": true, "name": "projectId", "in": "path" + }, + { + "schema": { + "type": "string", + "example": "user_a1b2c3" + }, + "required": true, + "name": "userId", + "in": "path" } ], "responses": { "204": { - "description": "Project deleted" + "description": "Member removed" }, "401": { "description": "Unauthorized", @@ -5018,11 +6064,11 @@ } } }, - "/api/projects/{projectId}/members": { + "/api/projects/{projectId}/flows": { "get": { - "tags": ["Projects"], - "summary": "List members", - "description": "List all members of a project. Requires membership.", + "tags": ["Flows"], + "summary": "List flows", + "description": "List all flows for a project.", "parameters": [ { "schema": { @@ -5033,15 +6079,63 @@ "required": true, "name": "projectId", "in": "path" + }, + { + "schema": { + "type": "string", + "enum": ["name", "updated_at", "created_at"], + "example": "updated_at" + }, + "required": false, + "name": "sort", + "in": "query" + }, + { + "schema": { + "type": "string", + "enum": ["asc", "desc"], + "example": "desc" + }, + "required": false, + "name": "order", + "in": "query" + }, + { + "schema": { + "type": "string", + "enum": ["true", "false"], + "example": "false" + }, + "required": false, + "name": "include_deleted", + "in": "query" + }, + { + "schema": { + "type": "string" + }, + "required": false, + "name": "cursor", + "in": "query" + }, + { + "schema": { + "type": "integer", + "minimum": 1, + "maximum": 100 + }, + "required": false, + "name": "limit", + "in": "query" } ], "responses": { "200": { - "description": "List of members", + "description": "List of flows", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ListMembersResponse" + "$ref": "#/components/schemas/ListFlowsResponse" } } } @@ -5069,9 +6163,9 @@ } }, "post": { - "tags": ["Projects"], - "summary": "Add member", - "description": "Add a member to the project by email. Requires owner role.", + "tags": ["Flows"], + "summary": "Create flow", + "description": "Create a new flow in the project. Requires member role.", "parameters": [ { "schema": { @@ -5088,18 +6182,18 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AddMemberRequest" + "$ref": "#/components/schemas/CreateFlowRequest" } } } }, "responses": { "201": { - "description": "Member added", + "description": "Flow created", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Member" + "$ref": "#/components/schemas/Flow" } } } @@ -5129,13 +6223,75 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "409": { + "description": "Conflict", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, + "/api/projects/{projectId}/flows/{flowId}": { + "get": { + "tags": ["Flows"], + "summary": "Get flow", + "description": "Get a single flow by ID. Use ?fields to select specific sections (reduces response size).", + "parameters": [ + { + "schema": { + "type": "string", + "pattern": "^proj_[a-zA-Z0-9_-]+$", + "example": "proj_x7y8z9" + }, + "required": true, + "name": "projectId", + "in": "path" + }, + { + "schema": { + "type": "string", + "pattern": "^flow_[a-zA-Z0-9_-]+$", + "example": "flow_a1b2c3d4" + }, + "required": true, + "name": "flowId", + "in": "path" + }, + { + "schema": { + "type": "string", + "description": "Comma-separated dot-paths to select specific fields (e.g., \"config.variables,config.flows.tracking.sources\"). Always includes id, createdAt, updatedAt.", + "example": "config.variables,config.flows.tracking.sources" + }, + "required": false, + "description": "Comma-separated dot-paths to select specific fields (e.g., \"config.variables,config.flows.tracking.sources\"). Always includes id, createdAt, updatedAt.", + "name": "fields", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Flow details", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FlowDetailResponse" } } } }, - "404": { - "description": "Not found", + "401": { + "description": "Unauthorized", "content": { "application/json": { "schema": { @@ -5144,8 +6300,8 @@ } } }, - "409": { - "description": "Conflict", + "404": { + "description": "Not found", "content": { "application/json": { "schema": { @@ -5155,13 +6311,11 @@ } } } - } - }, - "/api/projects/{projectId}/members/{userId}": { + }, "patch": { - "tags": ["Projects"], - "summary": "Update member role", - "description": "Update a member's role. Requires owner role.", + "tags": ["Flows"], + "summary": "Update flow", + "description": "Update an existing flow. Creates a version snapshot before applying changes. Requires member role. Use Content-Type: application/merge-patch+json to send only changed fields (RFC 7386).", "parameters": [ { "schema": { @@ -5176,29 +6330,46 @@ { "schema": { "type": "string", - "example": "user_a1b2c3" + "pattern": "^flow_[a-zA-Z0-9_-]+$", + "example": "flow_a1b2c3d4" }, "required": true, - "name": "userId", + "name": "flowId", "in": "path" + }, + { + "schema": { + "type": "string", + "description": "ETag from a previous GET. Returns 412 if flow was modified since.", + "example": "\"a1b2c3d4e5f6g7h8\"" + }, + "required": false, + "description": "ETag from a previous GET. Returns 412 if flow was modified since.", + "name": "if-match", + "in": "header" } ], "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/UpdateMemberRequest" + "$ref": "#/components/schemas/UpdateFlowRequest" + } + }, + "application/merge-patch+json": { + "schema": { + "$ref": "#/components/schemas/UpdateFlowRequest" } } } }, "responses": { "200": { - "description": "Role updated", + "description": "Flow updated", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Member" + "$ref": "#/components/schemas/FlowUpdateResponse" } } } @@ -5242,13 +6413,26 @@ } } } + }, + "409": { + "description": "Conflict", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "412": { + "description": "ETag mismatch — flow was modified since last read" } } }, "delete": { - "tags": ["Projects"], - "summary": "Remove member", - "description": "Remove a member from the project. Requires owner role.", + "tags": ["Flows"], + "summary": "Soft-delete flow", + "description": "Soft delete a flow (sets deleted_at timestamp). Requires member role.", "parameters": [ { "schema": { @@ -5263,16 +6447,28 @@ { "schema": { "type": "string", - "example": "user_a1b2c3" + "pattern": "^flow_[a-zA-Z0-9_-]+$", + "example": "flow_a1b2c3d4" }, "required": true, - "name": "userId", + "name": "flowId", "in": "path" + }, + { + "schema": { + "type": "string", + "description": "ETag from a previous GET. Returns 412 if flow was modified since.", + "example": "\"a1b2c3d4e5f6g7h8\"" + }, + "required": false, + "description": "ETag from a previous GET. Returns 412 if flow was modified since.", + "name": "if-match", + "in": "header" } ], "responses": { "204": { - "description": "Member removed" + "description": "Flow deleted" }, "401": { "description": "Unauthorized", @@ -5303,15 +6499,18 @@ } } } + }, + "412": { + "description": "ETag mismatch — flow was modified since last read" } } } }, - "/api/projects/{projectId}/flows": { - "get": { + "/api/projects/{projectId}/flows/{flowId}/duplicate": { + "post": { "tags": ["Flows"], - "summary": "List flows", - "description": "List all flows for a project.", + "summary": "Duplicate flow", + "description": "Create a copy of an existing flow with a new ID and no version history. Requires member role.", "parameters": [ { "schema": { @@ -5326,98 +6525,11 @@ { "schema": { "type": "string", - "enum": ["name", "updated_at", "created_at"], - "example": "updated_at" - }, - "required": false, - "name": "sort", - "in": "query" - }, - { - "schema": { - "type": "string", - "enum": ["asc", "desc"], - "example": "desc" - }, - "required": false, - "name": "order", - "in": "query" - }, - { - "schema": { - "type": "string", - "enum": ["true", "false"], - "example": "false" - }, - "required": false, - "name": "include_deleted", - "in": "query" - }, - { - "schema": { - "type": "string" - }, - "required": false, - "name": "cursor", - "in": "query" - }, - { - "schema": { - "type": "integer", - "minimum": 1, - "maximum": 100 - }, - "required": false, - "name": "limit", - "in": "query" - } - ], - "responses": { - "200": { - "description": "List of flows", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ListFlowsResponse" - } - } - } - }, - "401": { - "description": "Unauthorized", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } - }, - "404": { - "description": "Not found", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } - } - } - }, - "post": { - "tags": ["Flows"], - "summary": "Create flow", - "description": "Create a new flow in the project. Requires member role.", - "parameters": [ - { - "schema": { - "type": "string", - "pattern": "^proj_[a-zA-Z0-9_-]+$", - "example": "proj_x7y8z9" + "pattern": "^flow_[a-zA-Z0-9_-]+$", + "example": "flow_a1b2c3d4" }, "required": true, - "name": "projectId", + "name": "flowId", "in": "path" } ], @@ -5425,14 +6537,14 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CreateFlowRequest" + "$ref": "#/components/schemas/DuplicateFlowRequest" } } } }, "responses": { "201": { - "description": "Flow created", + "description": "Flow duplicated", "content": { "application/json": { "schema": { @@ -5471,6 +6583,16 @@ } } }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, "409": { "description": "Conflict", "content": { @@ -5484,11 +6606,11 @@ } } }, - "/api/projects/{projectId}/flows/{flowId}": { + "/api/projects/{projectId}/flows/{flowId}/secrets": { "get": { - "tags": ["Flows"], - "summary": "Get flow", - "description": "Get a single flow by ID. Use ?fields to select specific sections (reduces response size).", + "tags": ["Secrets"], + "summary": "List secrets", + "description": "List a flow's secrets as metadata only (name, id, timestamps). Values are never returned. Requires member role and the secrets entitlement.", "parameters": [ { "schema": { @@ -5509,32 +6631,66 @@ "required": true, "name": "flowId", "in": "path" - }, - { - "schema": { - "type": "string", - "description": "Comma-separated dot-paths to select specific fields (e.g., \"config.variables,config.flows.tracking.sources\"). Always includes id, createdAt, updatedAt.", - "example": "config.variables,config.flows.tracking.sources" - }, - "required": false, - "description": "Comma-separated dot-paths to select specific fields (e.g., \"config.variables,config.flows.tracking.sources\"). Always includes id, createdAt, updatedAt.", - "name": "fields", - "in": "query" } ], "responses": { "200": { - "description": "Flow details", + "description": "Secret metadata list", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "secrets": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "flowId": { + "type": "string" + }, + "createdAt": { + "type": ["string", "null"], + "format": "date-time" + }, + "updatedAt": { + "type": ["string", "null"], + "format": "date-time" + } + }, + "required": [ + "id", + "name", + "flowId", + "createdAt", + "updatedAt" + ] + } + } + }, + "required": ["secrets"] + } + } + } + }, + "401": { + "description": "Unauthorized", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FlowDetailResponse" + "$ref": "#/components/schemas/ErrorResponse" } } } }, - "401": { - "description": "Unauthorized", + "403": { + "description": "Forbidden", "content": { "application/json": { "schema": { @@ -5555,10 +6711,10 @@ } } }, - "patch": { - "tags": ["Flows"], - "summary": "Update flow", - "description": "Update an existing flow. Creates a version snapshot before applying changes. Requires member role. Use Content-Type: application/merge-patch+json to send only changed fields (RFC 7386).", + "post": { + "tags": ["Secrets"], + "summary": "Create secret", + "description": "Create a secret for a flow. The value is encrypted at rest and never returned. Requires member role and the secrets entitlement.", "parameters": [ { "schema": { @@ -5579,40 +6735,58 @@ "required": true, "name": "flowId", "in": "path" - }, - { - "schema": { - "type": "string", - "description": "ETag from a previous GET. Returns 412 if flow was modified since.", - "example": "\"a1b2c3d4e5f6g7h8\"" - }, - "required": false, - "description": "ETag from a previous GET. Returns 412 if flow was modified since.", - "name": "if-match", - "in": "header" } ], "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/UpdateFlowRequest" - } - }, - "application/merge-patch+json": { - "schema": { - "$ref": "#/components/schemas/UpdateFlowRequest" + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "pattern": "^[A-Z_][A-Z0-9_]*$" + }, + "value": { + "type": "string", + "minLength": 1, + "maxLength": 65536 + } + }, + "required": ["name", "value"] } } } }, "responses": { - "200": { - "description": "Flow updated", + "201": { + "description": "Secret created (metadata only)", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FlowUpdateResponse" + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "flowId": { + "type": "string" + }, + "createdAt": { + "type": ["string", "null"], + "format": "date-time" + }, + "updatedAt": { + "type": ["string", "null"], + "format": "date-time" + } + }, + "required": ["id", "name", "flowId", "createdAt", "updatedAt"] } } } @@ -5647,16 +6821,6 @@ } } }, - "404": { - "description": "Not found", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } - }, "409": { "description": "Conflict", "content": { @@ -5666,16 +6830,15 @@ } } } - }, - "412": { - "description": "ETag mismatch — flow was modified since last read" } } - }, - "delete": { - "tags": ["Flows"], - "summary": "Soft-delete flow", - "description": "Soft delete a flow (sets deleted_at timestamp). Requires member role.", + } + }, + "/api/projects/{projectId}/flows/{flowId}/secrets/{secretId}": { + "put": { + "tags": ["Secrets"], + "summary": "Update secret value", + "description": "Rotate a secret's value (re-encrypts). The value is never returned. Requires member role and the secrets entitlement.", "parameters": [ { "schema": { @@ -5700,18 +6863,70 @@ { "schema": { "type": "string", - "description": "ETag from a previous GET. Returns 412 if flow was modified since.", - "example": "\"a1b2c3d4e5f6g7h8\"" + "example": "sec_abc123" }, - "required": false, - "description": "ETag from a previous GET. Returns 412 if flow was modified since.", - "name": "if-match", - "in": "header" + "required": true, + "name": "secretId", + "in": "path" } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "value": { + "type": "string", + "minLength": 1, + "maxLength": 65536 + } + }, + "required": ["value"] + } + } + } + }, "responses": { - "204": { - "description": "Flow deleted" + "200": { + "description": "Secret updated (metadata only)", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "flowId": { + "type": "string" + }, + "createdAt": { + "type": ["string", "null"], + "format": "date-time" + }, + "updatedAt": { + "type": ["string", "null"], + "format": "date-time" + } + }, + "required": ["id", "name", "flowId", "createdAt", "updatedAt"] + } + } + } + }, + "400": { + "description": "Validation error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } }, "401": { "description": "Unauthorized", @@ -5742,18 +6957,13 @@ } } } - }, - "412": { - "description": "ETag mismatch — flow was modified since last read" } } - } - }, - "/api/projects/{projectId}/flows/{flowId}/duplicate": { - "post": { - "tags": ["Flows"], - "summary": "Duplicate flow", - "description": "Create a copy of an existing flow with a new ID and no version history. Requires member role.", + }, + "delete": { + "tags": ["Secrets"], + "summary": "Delete secret", + "description": "Soft-delete a secret. Idempotent: deleting a missing secret returns 204. Requires member role and the secrets entitlement.", "parameters": [ { "schema": { @@ -5774,60 +6984,23 @@ "required": true, "name": "flowId", "in": "path" + }, + { + "schema": { + "type": "string", + "example": "sec_abc123" + }, + "required": true, + "name": "secretId", + "in": "path" } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DuplicateFlowRequest" - } - } - } - }, "responses": { - "201": { - "description": "Flow duplicated", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Flow" - } - } - } - }, - "400": { - "description": "Validation error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } + "204": { + "description": "Secret deleted" }, "401": { - "description": "Unauthorized", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } - }, - "403": { - "description": "Forbidden", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } - }, - "404": { - "description": "Not found", + "description": "Unauthorized", "content": { "application/json": { "schema": { @@ -5836,8 +7009,8 @@ } } }, - "409": { - "description": "Conflict", + "403": { + "description": "Forbidden", "content": { "application/json": { "schema": { @@ -5849,11 +7022,11 @@ } } }, - "/api/projects/{projectId}/flows/{flowId}/secrets": { + "/api/projects/{projectId}/flows/{flowId}/secrets/values": { "get": { "tags": ["Secrets"], - "summary": "List secrets", - "description": "List a flow's secrets as metadata only (name, id, timestamps). Values are never returned. Requires member role and the secrets entitlement.", + "summary": "Get decrypted secret values", + "description": "Return decrypted secret values for a flow as a name-to-value map. Dual auth: a runtime container Bearer token bound to (projectId, flowId) with the `runner:read-secrets` scope returns only the bundle-referenced subset; a session cookie with member role returns all of the flow's secrets for administration. Responses are never cached.", "parameters": [ { "schema": { @@ -5878,46 +7051,20 @@ ], "responses": { "200": { - "description": "Secret metadata list", + "description": "Decrypted secret values keyed by name", "content": { "application/json": { "schema": { "type": "object", "properties": { - "secrets": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "flowId": { - "type": "string" - }, - "createdAt": { - "type": ["string", "null"], - "format": "date-time" - }, - "updatedAt": { - "type": ["string", "null"], - "format": "date-time" - } - }, - "required": [ - "id", - "name", - "flowId", - "createdAt", - "updatedAt" - ] + "values": { + "type": "object", + "additionalProperties": { + "type": "string" } } }, - "required": ["secrets"] + "required": ["values"] } } } @@ -5953,11 +7100,13 @@ } } } - }, + } + }, + "/api/projects/{projectId}/flows/{flowId}/steps/{stepPath}/examples": { "post": { - "tags": ["Secrets"], - "summary": "Create secret", - "description": "Create a secret for a flow. The value is encrypted at rest and never returned. Requires member role and the secrets entitlement.", + "tags": ["Flows"], + "summary": "Add step example", + "description": "Add a named example to a step. Examples are stored as an object map keyed by name. Rejects duplicate names with 409.", "parameters": [ { "schema": { @@ -5978,58 +7127,35 @@ "required": true, "name": "flowId", "in": "path" + }, + { + "schema": { + "type": "string", + "description": "Dot-segmented step path (sectionKey.stepName), e.g. \"destinations.gtag\".", + "example": "destinations.gtag" + }, + "required": true, + "description": "Dot-segmented step path (sectionKey.stepName), e.g. \"destinations.gtag\".", + "name": "stepPath", + "in": "path" } ], "requestBody": { "content": { "application/json": { "schema": { - "type": "object", - "properties": { - "name": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "pattern": "^[A-Z_][A-Z0-9_]*$" - }, - "value": { - "type": "string", - "minLength": 1, - "maxLength": 65536 - } - }, - "required": ["name", "value"] + "$ref": "#/components/schemas/CreateStepExampleRequest" } } } }, "responses": { - "201": { - "description": "Secret created (metadata only)", + "200": { + "description": "Updated examples object map", "content": { "application/json": { "schema": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "flowId": { - "type": "string" - }, - "createdAt": { - "type": ["string", "null"], - "format": "date-time" - }, - "updatedAt": { - "type": ["string", "null"], - "format": "date-time" - } - }, - "required": ["id", "name", "flowId", "createdAt", "updatedAt"] + "$ref": "#/components/schemas/StepExamplesResponse" } } } @@ -6054,8 +7180,8 @@ } } }, - "403": { - "description": "Forbidden", + "404": { + "description": "Not found", "content": { "application/json": { "schema": { @@ -6073,15 +7199,23 @@ } } } + }, + "422": { + "description": "Unprocessable entity", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } } } - } - }, - "/api/projects/{projectId}/flows/{flowId}/secrets/{secretId}": { + }, "put": { - "tags": ["Secrets"], - "summary": "Update secret value", - "description": "Rotate a secret's value (re-encrypts). The value is never returned. Requires member role and the secrets entitlement.", + "tags": ["Flows"], + "summary": "Edit step example", + "description": "Edit an existing named example in place, merging provided fields onto the stored entry. Returns 404 when the named example does not exist.", "parameters": [ { "schema": { @@ -6106,10 +7240,12 @@ { "schema": { "type": "string", - "example": "sec_abc123" + "description": "Dot-segmented step path (sectionKey.stepName), e.g. \"destinations.gtag\".", + "example": "destinations.gtag" }, "required": true, - "name": "secretId", + "description": "Dot-segmented step path (sectionKey.stepName), e.g. \"destinations.gtag\".", + "name": "stepPath", "in": "path" } ], @@ -6117,46 +7253,18 @@ "content": { "application/json": { "schema": { - "type": "object", - "properties": { - "value": { - "type": "string", - "minLength": 1, - "maxLength": 65536 - } - }, - "required": ["value"] + "$ref": "#/components/schemas/EditStepExampleRequest" } } } }, "responses": { "200": { - "description": "Secret updated (metadata only)", + "description": "Updated examples object map", "content": { "application/json": { "schema": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "flowId": { - "type": "string" - }, - "createdAt": { - "type": ["string", "null"], - "format": "date-time" - }, - "updatedAt": { - "type": ["string", "null"], - "format": "date-time" - } - }, - "required": ["id", "name", "flowId", "createdAt", "updatedAt"] + "$ref": "#/components/schemas/StepExamplesResponse" } } } @@ -6166,7 +7274,99 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "422": { + "description": "Unprocessable entity", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + }, + "delete": { + "tags": ["Flows"], + "summary": "Remove step example", + "description": "Remove a named example from a step. Returns 404 when the named example does not exist.", + "parameters": [ + { + "schema": { + "type": "string", + "pattern": "^proj_[a-zA-Z0-9_-]+$", + "example": "proj_x7y8z9" + }, + "required": true, + "name": "projectId", + "in": "path" + }, + { + "schema": { + "type": "string", + "pattern": "^flow_[a-zA-Z0-9_-]+$", + "example": "flow_a1b2c3d4" + }, + "required": true, + "name": "flowId", + "in": "path" + }, + { + "schema": { + "type": "string", + "description": "Dot-segmented step path (sectionKey.stepName), e.g. \"destinations.gtag\".", + "example": "destinations.gtag" + }, + "required": true, + "description": "Dot-segmented step path (sectionKey.stepName), e.g. \"destinations.gtag\".", + "name": "stepPath", + "in": "path" + }, + { + "schema": { + "type": "string", + "minLength": 1, + "description": "Name of the example to remove.", + "example": "product view" + }, + "required": true, + "description": "Name of the example to remove.", + "name": "name", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Updated examples object map", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StepExamplesResponse" } } } @@ -6181,8 +7381,8 @@ } } }, - "403": { - "description": "Forbidden", + "404": { + "description": "Not found", "content": { "application/json": { "schema": { @@ -6191,8 +7391,8 @@ } } }, - "404": { - "description": "Not found", + "422": { + "description": "Unprocessable entity", "content": { "application/json": { "schema": { @@ -6202,11 +7402,13 @@ } } } - }, - "delete": { - "tags": ["Secrets"], - "summary": "Delete secret", - "description": "Soft-delete a secret. Idempotent: deleting a missing secret returns 204. Requires member role and the secrets entitlement.", + } + }, + "/api/projects/{projectId}/flows/{flowId}/deploy": { + "get": { + "tags": ["Deployments"], + "summary": "Get latest deployment", + "description": "Get the latest deployment for a flow.", "parameters": [ { "schema": { @@ -6227,20 +7429,18 @@ "required": true, "name": "flowId", "in": "path" - }, - { - "schema": { - "type": "string", - "example": "sec_abc123" - }, - "required": true, - "name": "secretId", - "in": "path" } ], "responses": { - "204": { - "description": "Secret deleted" + "200": { + "description": "Latest deployment (or null)", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeploymentResponse" + } + } + } }, "401": { "description": "Unauthorized", @@ -6252,8 +7452,8 @@ } } }, - "403": { - "description": "Forbidden", + "404": { + "description": "Not found", "content": { "application/json": { "schema": { @@ -6263,13 +7463,11 @@ } } } - } - }, - "/api/projects/{projectId}/flows/{flowId}/secrets/values": { - "get": { - "tags": ["Secrets"], - "summary": "Get decrypted secret values", - "description": "Return decrypted secret values for a flow as a name-to-value map. Dual auth: a runtime container Bearer token bound to (projectId, flowId) with the `runner:read-secrets` scope returns only the bundle-referenced subset; a session cookie with member role returns all of the flow's secrets for administration. Responses are never cached.", + }, + "post": { + "tags": ["Deployments"], + "summary": "Start deployment", + "description": "Start a new deployment for a flow. The bundle runs asynchronously on the worker. Returns 400 AMBIGUOUS_CONFIG when the flow has multiple named settings (use the per-settings deploy endpoint instead). When an Idempotency-Key replays a prior request, returns 200 with status `already_created`.", "parameters": [ { "schema": { @@ -6294,20 +7492,31 @@ ], "responses": { "200": { - "description": "Decrypted secret values keyed by name", + "description": "Deployment started, or idempotent replay of a prior request", "content": { "application/json": { "schema": { - "type": "object", - "properties": { - "values": { - "type": "object", - "additionalProperties": { - "type": "string" - } - } - }, - "required": ["values"] + "$ref": "#/components/schemas/StartDeploymentResponse" + } + } + } + }, + "201": { + "description": "Deployment started", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StartDeploymentResponse" + } + } + } + }, + "400": { + "description": "Validation error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" } } } @@ -6341,15 +7550,45 @@ } } } + }, + "409": { + "description": "Deployment already in progress", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "429": { + "description": "Rate limited or concurrent deploy limit (Retry-After header)", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "503": { + "description": "Service unavailable", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } } } } }, - "/api/projects/{projectId}/flows/{flowId}/steps/{stepPath}/examples": { - "post": { - "tags": ["Flows"], - "summary": "Add step example", - "description": "Add a named example to a step. Examples are stored as an object map keyed by name. Rejects duplicate names with 409.", + "/api/projects/{projectId}/flows/{flowId}/deploy/{deploymentId}": { + "get": { + "tags": ["Deployments"], + "summary": "Get deployment", + "description": "Get a specific deployment by ID.", "parameters": [ { "schema": { @@ -6374,41 +7613,20 @@ { "schema": { "type": "string", - "description": "Dot-segmented step path (sectionKey.stepName), e.g. \"destinations.gtag\".", - "example": "destinations.gtag" + "example": "dep_abc123" }, "required": true, - "description": "Dot-segmented step path (sectionKey.stepName), e.g. \"destinations.gtag\".", - "name": "stepPath", + "name": "deploymentId", "in": "path" } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CreateStepExampleRequest" - } - } - } - }, "responses": { "200": { - "description": "Updated examples object map", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/StepExamplesResponse" - } - } - } - }, - "400": { - "description": "Validation error", + "description": "Deployment details", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "$ref": "#/components/schemas/DeploymentDetailResponse" } } } @@ -6432,33 +7650,13 @@ } } } - }, - "409": { - "description": "Conflict", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } - }, - "422": { - "description": "Unprocessable entity", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } } } }, - "put": { - "tags": ["Flows"], - "summary": "Edit step example", - "description": "Edit an existing named example in place, merging provided fields onto the stored entry. Returns 404 when the named example does not exist.", + "delete": { + "tags": ["Deployments"], + "summary": "Delete deployment", + "description": "Delete a deployment and its container. Requires owner role.", "parameters": [ { "schema": { @@ -6483,41 +7681,30 @@ { "schema": { "type": "string", - "description": "Dot-segmented step path (sectionKey.stepName), e.g. \"destinations.gtag\".", - "example": "destinations.gtag" + "example": "dep_abc123" }, "required": true, - "description": "Dot-segmented step path (sectionKey.stepName), e.g. \"destinations.gtag\".", - "name": "stepPath", + "name": "deploymentId", "in": "path" } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EditStepExampleRequest" - } - } - } - }, "responses": { "200": { - "description": "Updated examples object map", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/StepExamplesResponse" - } - } - } - }, - "400": { - "description": "Validation error", + "description": "Deployment deleted", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "status": { + "type": "string", + "enum": ["deleted"] + } + }, + "required": ["id", "status"] } } } @@ -6532,8 +7719,8 @@ } } }, - "404": { - "description": "Not found", + "403": { + "description": "Forbidden", "content": { "application/json": { "schema": { @@ -6542,8 +7729,8 @@ } } }, - "422": { - "description": "Unprocessable entity", + "404": { + "description": "Not found", "content": { "application/json": { "schema": { @@ -6553,11 +7740,13 @@ } } } - }, - "delete": { - "tags": ["Flows"], - "summary": "Remove step example", - "description": "Remove a named example from a step. Returns 404 when the named example does not exist.", + } + }, + "/api/projects/{projectId}/flows/{flowId}/settings": { + "get": { + "tags": ["Settings"], + "summary": "List settings", + "description": "List active named settings for a flow.", "parameters": [ { "schema": { @@ -6578,38 +7767,15 @@ "required": true, "name": "flowId", "in": "path" - }, - { - "schema": { - "type": "string", - "description": "Dot-segmented step path (sectionKey.stepName), e.g. \"destinations.gtag\".", - "example": "destinations.gtag" - }, - "required": true, - "description": "Dot-segmented step path (sectionKey.stepName), e.g. \"destinations.gtag\".", - "name": "stepPath", - "in": "path" - }, - { - "schema": { - "type": "string", - "minLength": 1, - "description": "Name of the example to remove.", - "example": "product view" - }, - "required": true, - "description": "Name of the example to remove.", - "name": "name", - "in": "query" } ], "responses": { "200": { - "description": "Updated examples object map", + "description": "List of settings", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/StepExamplesResponse" + "$ref": "#/components/schemas/ListSettingsResponse" } } } @@ -6633,25 +7799,15 @@ } } } - }, - "422": { - "description": "Unprocessable entity", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } } } } }, - "/api/projects/{projectId}/flows/{flowId}/deploy": { + "/api/projects/{projectId}/flows/{flowId}/settings/{settingsId}": { "get": { - "tags": ["Deployments"], - "summary": "Get latest deployment", - "description": "Get the latest deployment for a flow.", + "tags": ["Settings"], + "summary": "Get settings", + "description": "Get a single settings entry with its latest deployment.", "parameters": [ { "schema": { @@ -6672,15 +7828,25 @@ "required": true, "name": "flowId", "in": "path" + }, + { + "schema": { + "type": "string", + "pattern": "^cfg_[a-zA-Z0-9_-]+$", + "example": "cfg_a1b2c3d4" + }, + "required": true, + "name": "settingsId", + "in": "path" } ], "responses": { "200": { - "description": "Latest deployment (or null)", + "description": "Settings details with deployment", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/DeploymentResponse" + "$ref": "#/components/schemas/FlowSettingsDetail" } } } @@ -6706,11 +7872,13 @@ } } } - }, - "post": { - "tags": ["Deployments"], - "summary": "Start deployment", - "description": "Start a new deployment for a flow. Returns 400 AMBIGUOUS_SETTINGS if the flow has multiple named settings — use the per-settings deploy endpoint instead.", + } + }, + "/api/projects/{projectId}/flows/{flowId}/settings/{settingsId}/json": { + "get": { + "tags": ["Settings"], + "summary": "Download settings JSON", + "description": "Download the named flow settings as a self-contained Config JSON file. Includes parent variables and definitions.", "parameters": [ { "schema": { @@ -6731,25 +7899,25 @@ "required": true, "name": "flowId", "in": "path" + }, + { + "schema": { + "type": "string", + "pattern": "^cfg_[a-zA-Z0-9_-]+$", + "example": "cfg_a1b2c3d4" + }, + "required": true, + "name": "settingsId", + "in": "path" } ], "responses": { - "201": { - "description": "Deployment started", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/StartDeploymentResponse" - } - } - } - }, - "400": { - "description": "Validation error", + "200": { + "description": "Flow Config JSON file", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "$ref": "#/components/schemas/FlowConfig" } } } @@ -6773,35 +7941,15 @@ } } } - }, - "409": { - "description": "Deployment already in progress", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } - }, - "503": { - "description": "Service unavailable", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } } } } }, - "/api/projects/{projectId}/flows/{flowId}/deploy/{deploymentId}": { + "/api/projects/{projectId}/flows/{flowId}/settings/{settingsId}/bundle": { "get": { - "tags": ["Deployments"], - "summary": "Get deployment", - "description": "Get a specific deployment by ID.", + "tags": ["Settings"], + "summary": "Download settings bundle", + "description": "Download the compiled JS/MJS for the settings' latest deployment. Redirects to a presigned download URL.", "parameters": [ { "schema": { @@ -6826,26 +7974,30 @@ { "schema": { "type": "string", - "example": "dep_abc123" + "pattern": "^cfg_[a-zA-Z0-9_-]+$", + "example": "cfg_a1b2c3d4" }, "required": true, - "name": "deploymentId", + "name": "settingsId", "in": "path" } ], "responses": { - "200": { - "description": "Deployment details", + "302": { + "description": "Redirect to presigned bundle URL" + }, + "401": { + "description": "Unauthorized", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/DeploymentDetailResponse" + "$ref": "#/components/schemas/ErrorResponse" } } } }, - "401": { - "description": "Unauthorized", + "404": { + "description": "Not found", "content": { "application/json": { "schema": { @@ -6854,8 +8006,8 @@ } } }, - "404": { - "description": "Not found", + "503": { + "description": "Service unavailable", "content": { "application/json": { "schema": { @@ -6865,11 +8017,13 @@ } } } - }, - "delete": { - "tags": ["Deployments"], - "summary": "Delete deployment", - "description": "Delete a deployment and its container. Requires owner role.", + } + }, + "/api/projects/{projectId}/flows/{flowId}/settings/{settingsId}/deploy": { + "post": { + "tags": ["Settings"], + "summary": "Deploy settings", + "description": "Start a deployment for a specific settings entry. Detects platform from the settings.", "parameters": [ { "schema": { @@ -6894,30 +8048,21 @@ { "schema": { "type": "string", - "example": "dep_abc123" + "pattern": "^cfg_[a-zA-Z0-9_-]+$", + "example": "cfg_a1b2c3d4" }, "required": true, - "name": "deploymentId", + "name": "settingsId", "in": "path" } ], "responses": { - "200": { - "description": "Deployment deleted", + "201": { + "description": "Deployment started", "content": { "application/json": { "schema": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "status": { - "type": "string", - "enum": ["deleted"] - } - }, - "required": ["id", "status"] + "$ref": "#/components/schemas/DeploySettingsResponse" } } } @@ -6932,8 +8077,8 @@ } } }, - "403": { - "description": "Forbidden", + "404": { + "description": "Not found", "content": { "application/json": { "schema": { @@ -6942,8 +8087,28 @@ } } }, - "404": { - "description": "Not found", + "409": { + "description": "Deployment already in progress", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "422": { + "description": "Settings orphaned", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "503": { + "description": "Service unavailable", "content": { "application/json": { "schema": { @@ -6953,13 +8118,11 @@ } } } - } - }, - "/api/projects/{projectId}/flows/{flowId}/settings": { + }, "get": { "tags": ["Settings"], - "summary": "List settings", - "description": "List active named settings for a flow.", + "summary": "Get latest settings deployment", + "description": "Get the latest deployment for a specific settings entry.", "parameters": [ { "schema": { @@ -6980,15 +8143,25 @@ "required": true, "name": "flowId", "in": "path" + }, + { + "schema": { + "type": "string", + "pattern": "^cfg_[a-zA-Z0-9_-]+$", + "example": "cfg_a1b2c3d4" + }, + "required": true, + "name": "settingsId", + "in": "path" } ], "responses": { "200": { - "description": "List of settings", + "description": "Latest deployment (or null)", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ListSettingsResponse" + "$ref": "#/components/schemas/SettingsDeploymentResponse" } } } @@ -7016,11 +8189,11 @@ } } }, - "/api/projects/{projectId}/flows/{flowId}/settings/{settingsId}": { + "/api/projects/{projectId}/flows/{flowId}/settings/{settingsId}/deployments/{deploymentId}": { "get": { "tags": ["Settings"], - "summary": "Get settings", - "description": "Get a single settings entry with its latest deployment.", + "summary": "Get settings deployment detail", + "description": "Get a specific deployment by ID, scoped to a settings entry.", "parameters": [ { "schema": { @@ -7051,15 +8224,24 @@ "required": true, "name": "settingsId", "in": "path" + }, + { + "schema": { + "type": "string", + "example": "dep_abc123" + }, + "required": true, + "name": "deploymentId", + "in": "path" } ], "responses": { "200": { - "description": "Settings details with deployment", + "description": "Deployment details", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FlowSettingsDetail" + "$ref": "#/components/schemas/SettingsDeploymentDetailResponse" } } } @@ -7087,11 +8269,11 @@ } } }, - "/api/projects/{projectId}/flows/{flowId}/settings/{settingsId}/json": { - "get": { - "tags": ["Settings"], - "summary": "Download settings JSON", - "description": "Download the named flow settings as a self-contained Config JSON file. Includes parent variables and definitions.", + "/api/projects/{projectId}/flows/{flowId}/previews": { + "post": { + "tags": ["Previews"], + "summary": "Create preview", + "description": "Create a new preview for a web flow settings entry. Bundles the flow and publishes to a unique token-based URL.", "parameters": [ { "schema": { @@ -7112,25 +8294,34 @@ "required": true, "name": "flowId", "in": "path" - }, - { - "schema": { - "type": "string", - "pattern": "^cfg_[a-zA-Z0-9_-]+$", - "example": "cfg_a1b2c3d4" - }, - "required": true, - "name": "settingsId", - "in": "path" } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreatePreviewRequest" + } + } + } + }, "responses": { - "200": { - "description": "Flow Config JSON file", + "201": { + "description": "Preview created", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FlowConfig" + "$ref": "#/components/schemas/PreviewResponse" + } + } + } + }, + "400": { + "description": "Validation error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" } } } @@ -7145,6 +8336,16 @@ } } }, + "403": { + "description": "Forbidden", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, "404": { "description": "Not found", "content": { @@ -7154,15 +8355,33 @@ } } } + }, + "429": { + "description": "Quota exceeded", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "502": { + "description": "Bundle or upload failed", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } } } - } - }, - "/api/projects/{projectId}/flows/{flowId}/settings/{settingsId}/bundle": { + }, "get": { - "tags": ["Settings"], - "summary": "Download settings bundle", - "description": "Download the compiled JS/MJS for the settings' latest deployment. Redirects to a presigned download URL.", + "tags": ["Previews"], + "summary": "List previews", + "description": "List all previews for a flow, ordered by creation date descending.", "parameters": [ { "schema": { @@ -7183,34 +8402,21 @@ "required": true, "name": "flowId", "in": "path" - }, - { - "schema": { - "type": "string", - "pattern": "^cfg_[a-zA-Z0-9_-]+$", - "example": "cfg_a1b2c3d4" - }, - "required": true, - "name": "settingsId", - "in": "path" } ], "responses": { - "302": { - "description": "Redirect to presigned bundle URL" - }, - "401": { - "description": "Unauthorized", + "200": { + "description": "List of previews", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "$ref": "#/components/schemas/ListPreviewsResponse" } } } }, - "404": { - "description": "Not found", + "401": { + "description": "Unauthorized", "content": { "application/json": { "schema": { @@ -7219,8 +8425,8 @@ } } }, - "503": { - "description": "Service unavailable", + "404": { + "description": "Not found", "content": { "application/json": { "schema": { @@ -7232,11 +8438,11 @@ } } }, - "/api/projects/{projectId}/flows/{flowId}/settings/{settingsId}/deploy": { - "post": { - "tags": ["Settings"], - "summary": "Deploy settings", - "description": "Start a deployment for a specific settings entry. Detects platform from the settings.", + "/api/projects/{projectId}/flows/{flowId}/previews/{previewId}": { + "get": { + "tags": ["Previews"], + "summary": "Get preview", + "description": "Get a single preview by ID.", "parameters": [ { "schema": { @@ -7261,21 +8467,21 @@ { "schema": { "type": "string", - "pattern": "^cfg_[a-zA-Z0-9_-]+$", - "example": "cfg_a1b2c3d4" + "pattern": "^prv_[a-z0-9]+$", + "example": "prv_abc123xyz456" }, "required": true, - "name": "settingsId", + "name": "previewId", "in": "path" } ], "responses": { - "201": { - "description": "Deployment started", + "200": { + "description": "Preview details", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/DeploySettingsResponse" + "$ref": "#/components/schemas/PreviewResponse" } } } @@ -7299,43 +8505,13 @@ } } } - }, - "409": { - "description": "Deployment already in progress", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } - }, - "422": { - "description": "Settings orphaned", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } - }, - "503": { - "description": "Service unavailable", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } } } }, - "get": { - "tags": ["Settings"], - "summary": "Get latest settings deployment", - "description": "Get the latest deployment for a specific settings entry.", + "delete": { + "tags": ["Previews"], + "summary": "Delete preview", + "description": "Delete a preview and its S3 bundle. Requires member role.", "parameters": [ { "schema": { @@ -7360,27 +8536,30 @@ { "schema": { "type": "string", - "pattern": "^cfg_[a-zA-Z0-9_-]+$", - "example": "cfg_a1b2c3d4" + "pattern": "^prv_[a-z0-9]+$", + "example": "prv_abc123xyz456" }, "required": true, - "name": "settingsId", + "name": "previewId", "in": "path" } ], "responses": { - "200": { - "description": "Latest deployment (or null)", + "204": { + "description": "Preview deleted" + }, + "401": { + "description": "Unauthorized", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SettingsDeploymentResponse" + "$ref": "#/components/schemas/ErrorResponse" } } } }, - "401": { - "description": "Unauthorized", + "403": { + "description": "Forbidden", "content": { "application/json": { "schema": { @@ -7402,11 +8581,11 @@ } } }, - "/api/projects/{projectId}/flows/{flowId}/settings/{settingsId}/deployments/{deploymentId}": { - "get": { - "tags": ["Settings"], - "summary": "Get settings deployment detail", - "description": "Get a specific deployment by ID, scoped to a settings entry.", + "/api/projects/{projectId}/flows/{flowId}/observe-sessions": { + "post": { + "tags": ["Observe Sessions"], + "summary": "Start observe session", + "description": "Start an Observe session for a flow. Validates the flow topology, inserts the row, and kicks off detached provisioning. Returns the row immediately as arming.", "parameters": [ { "schema": { @@ -7427,34 +8606,34 @@ "required": true, "name": "flowId", "in": "path" - }, - { - "schema": { - "type": "string", - "pattern": "^cfg_[a-zA-Z0-9_-]+$", - "example": "cfg_a1b2c3d4" - }, - "required": true, - "name": "settingsId", - "in": "path" - }, - { - "schema": { - "type": "string", - "example": "dep_abc123" - }, - "required": true, - "name": "deploymentId", - "in": "path" } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateObserveSessionRequest" + } + } + } + }, "responses": { - "200": { - "description": "Deployment details", + "201": { + "description": "Observe session started", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SettingsDeploymentDetailResponse" + "$ref": "#/components/schemas/ObserveSessionResponse" + } + } + } + }, + "400": { + "description": "Validation error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" } } } @@ -7469,6 +8648,16 @@ } } }, + "403": { + "description": "Forbidden", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, "404": { "description": "Not found", "content": { @@ -7478,15 +8667,35 @@ } } } + }, + "409": { + "description": "Flow topology not supported", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "429": { + "description": "Rate limit exceeded", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } } } } }, - "/api/projects/{projectId}/flows/{flowId}/previews": { - "post": { - "tags": ["Previews"], - "summary": "Create preview", - "description": "Create a new preview for a web flow settings entry. Bundles the flow and publishes to a unique token-based URL.", + "/api/projects/{projectId}/flows/{flowId}/observe-sessions/{sessionId}": { + "get": { + "tags": ["Observe Sessions"], + "summary": "Get observe session", + "description": "Get an observe session: status, error message, config snapshot, web activation info, and the live server endpoint when live.", "parameters": [ { "schema": { @@ -7507,34 +8716,25 @@ "required": true, "name": "flowId", "in": "path" + }, + { + "schema": { + "type": "string", + "pattern": "^ses_[a-zA-Z0-9_-]+$", + "example": "ses_abc123xyz456" + }, + "required": true, + "name": "sessionId", + "in": "path" } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CreatePreviewRequest" - } - } - } - }, "responses": { - "201": { - "description": "Preview created", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/PreviewResponse" - } - } - } - }, - "400": { - "description": "Validation error", + "200": { + "description": "Observe session details", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "$ref": "#/components/schemas/ObserveSessionResponse" } } } @@ -7568,33 +8768,13 @@ } } } - }, - "429": { - "description": "Quota exceeded", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } - }, - "502": { - "description": "Bundle or upload failed", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } } } }, - "get": { - "tags": ["Previews"], - "summary": "List previews", - "description": "List all previews for a flow, ordered by creation date descending.", + "delete": { + "tags": ["Observe Sessions"], + "summary": "End observe session", + "description": "End an observe session: tear down the container, revoke credentials, delete the web preview, delete the row. Idempotent.", "parameters": [ { "schema": { @@ -7615,21 +8795,34 @@ "required": true, "name": "flowId", "in": "path" + }, + { + "schema": { + "type": "string", + "pattern": "^ses_[a-zA-Z0-9_-]+$", + "example": "ses_abc123xyz456" + }, + "required": true, + "name": "sessionId", + "in": "path" } ], "responses": { - "200": { - "description": "List of previews", + "204": { + "description": "Observe session ended" + }, + "401": { + "description": "Unauthorized", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ListPreviewsResponse" + "$ref": "#/components/schemas/ErrorResponse" } } } }, - "401": { - "description": "Unauthorized", + "403": { + "description": "Forbidden", "content": { "application/json": { "schema": { @@ -7651,11 +8844,11 @@ } } }, - "/api/projects/{projectId}/flows/{flowId}/previews/{previewId}": { - "get": { - "tags": ["Previews"], - "summary": "Get preview", - "description": "Get a single preview by ID.", + "/api/projects/{projectId}/flows/{flowId}/observe-sessions/{sessionId}/heartbeat": { + "post": { + "tags": ["Observe Sessions"], + "summary": "Heartbeat observe session", + "description": "Keep an observe session warm. The window posts this every 30s while open; a stale session is reaped by the janitor.", "parameters": [ { "schema": { @@ -7680,21 +8873,21 @@ { "schema": { "type": "string", - "pattern": "^prv_[a-z0-9]+$", - "example": "prv_abc123xyz456" + "pattern": "^ses_[a-zA-Z0-9_-]+$", + "example": "ses_abc123xyz456" }, "required": true, - "name": "previewId", + "name": "sessionId", "in": "path" } ], "responses": { "200": { - "description": "Preview details", + "description": "Heartbeat recorded", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/PreviewResponse" + "$ref": "#/components/schemas/ObserveSessionHeartbeatResponse" } } } @@ -7720,11 +8913,13 @@ } } } - }, - "delete": { - "tags": ["Previews"], - "summary": "Delete preview", - "description": "Delete a preview and its S3 bundle. Requires member role.", + } + }, + "/api/projects/{projectId}/flows/{flowId}/observe-sessions/{sessionId}/end": { + "post": { + "tags": ["Observe Sessions"], + "summary": "End observe session (beacon)", + "description": "The navigator.sendBeacon end target for page unload. Mirrors the DELETE end route because sendBeacon cannot send a DELETE. Idempotent.", "parameters": [ { "schema": { @@ -7749,17 +8944,17 @@ { "schema": { "type": "string", - "pattern": "^prv_[a-z0-9]+$", - "example": "prv_abc123xyz456" + "pattern": "^ses_[a-zA-Z0-9_-]+$", + "example": "ses_abc123xyz456" }, "required": true, - "name": "previewId", + "name": "sessionId", "in": "path" } ], "responses": { "204": { - "description": "Preview deleted" + "description": "Observe session ended" }, "401": { "description": "Unauthorized", @@ -7771,16 +8966,6 @@ } } }, - "403": { - "description": "Forbidden", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } - }, "404": { "description": "Not found", "content": { @@ -10475,6 +11660,66 @@ } } }, + "/api/projects/{projectId}/deployments/{deploymentId}/stream": { + "get": { + "tags": ["Deployments"], + "summary": "Stream deployment status (SSE)", + "description": "Server-Sent Events (`text/event-stream`) stream of a deployment's live status. Emits named events: `status` (a snapshot payload, schema below), `done` (terminal, no body), and `timeout`. The CLI consumes this with a raw fetch while waiting for a deploy to finish. Requires member role. The schema documents the JSON `data:` of a `status` event; `errorCode`/`errorMessage` carry the persisted, redacted classification of a failed deploy.", + "parameters": [ + { + "schema": { + "type": "string", + "pattern": "^proj_[a-zA-Z0-9_-]+$", + "example": "proj_x7y8z9" + }, + "required": true, + "name": "projectId", + "in": "path" + }, + { + "schema": { + "type": "string", + "example": "dep_abc123" + }, + "required": true, + "name": "deploymentId", + "in": "path" + } + ], + "responses": { + "200": { + "description": "SSE stream; `status` event payload shape documented here.", + "content": { + "text/event-stream": { + "schema": { + "$ref": "#/components/schemas/DeploymentStreamStatusEvent" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, "/api/projects/{projectId}/deployments/{deploymentId}/versions": { "get": { "tags": ["Deployments"], @@ -10833,96 +12078,6 @@ } } }, - "/api/projects/{projectId}/deployments/{deploymentId}/trace": { - "post": { - "tags": ["Deployments"], - "summary": "Toggle debug tracing", - "description": "Enable or disable debug tracing for a deployment. Owner-only, gated by the debugTrace feature. `minutes` of 0 disables; allowed values are 0, 15, 30, 60. An absent body defaults to 15 minutes.", - "parameters": [ - { - "schema": { - "type": "string", - "pattern": "^proj_[a-zA-Z0-9_-]+$", - "example": "proj_x7y8z9" - }, - "required": true, - "name": "projectId", - "in": "path" - }, - { - "schema": { - "type": "string", - "example": "dep_abc123" - }, - "required": true, - "name": "deploymentId", - "in": "path" - } - ], - "requestBody": { - "required": false, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/TraceDeploymentRequest" - } - } - } - }, - "responses": { - "200": { - "description": "Updated trace window", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/TraceDeploymentResponse" - } - } - } - }, - "400": { - "description": "Validation error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } - }, - "401": { - "description": "Unauthorized", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } - }, - "403": { - "description": "Forbidden", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } - }, - "404": { - "description": "Not found", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - } - } - } - } - }, "/api/projects/{projectId}/deployments/{deploymentId}/usage": { "get": { "tags": ["Deployments"], diff --git a/packages/cli/src/__tests__/integration/bundle/cdn-jsdom.test.ts b/packages/cli/src/__tests__/integration/bundle/cdn-jsdom.test.ts index f3669392f..f15124a6f 100644 --- a/packages/cli/src/__tests__/integration/bundle/cdn-jsdom.test.ts +++ b/packages/cli/src/__tests__/integration/bundle/cdn-jsdom.test.ts @@ -96,17 +96,16 @@ interface LoadedBundle { } describe('CDN bundle — jsdom smoke', () => { + // The three smoke tests build the identical MINIMAL_FLOW cdn bundle with the + // identical overrides, so the esbuild pipeline runs ONCE here and every test + // loads the same script bytes into its own fresh jsdom window. Per-window + // isolation is preserved (the routing test pushes through elb and mutates + // its window), only the expensive build is shared. let tmpDir: string; + let script: string; - beforeEach(async () => { + beforeAll(async () => { tmpDir = await mkdtemp(join(tmpdir(), 'walkeros-jsdom-')); - }); - - afterEach(async () => { - await rm(tmpDir, { recursive: true, force: true }).catch(() => {}); - }); - - async function buildAndLoad(): Promise { const out = join(tmpDir, 'walker.js'); await bundle(MINIMAL_FLOW, { target: 'cdn', @@ -117,8 +116,16 @@ describe('CDN bundle — jsdom smoke', () => { windowElb: 'elb', }, }); - const script = await readFile(out, 'utf8'); + script = await readFile(out, 'utf8'); + }, 120000); + afterAll(async () => { + await rm(tmpDir, { recursive: true, force: true }).catch(() => {}); + }); + + // Load the shared bundle script into a fresh jsdom window with a captured + // fetch. Each call returns an isolated window so tests never share state. + async function loadBundle(): Promise { const virtualConsole = new VirtualConsole(); virtualConsole.on('jsdomError', () => {}); @@ -150,25 +157,25 @@ describe('CDN bundle — jsdom smoke', () => { } it('sets window.elb as a function after load', async () => { - const { dom } = await buildAndLoad(); + const { dom } = await loadBundle(); expect(typeof (dom.window as unknown as { elb: unknown }).elb).toBe( 'function', ); dom.window.close(); - }, 120000); + }); it('sets window.elbLayer as an array after load', async () => { - const { dom } = await buildAndLoad(); + const { dom } = await loadBundle(); expect( Array.isArray((dom.window as unknown as { elbLayer: unknown }).elbLayer), ).toBe(true); dom.window.close(); - }, 120000); + }); // Bug 2 regression: previously elbLayer existed but nothing routed through // to destinations because the browser source had no env.window/env.document. it('elb("page view") reaches the api destination via elbLayer', async () => { - const { dom, fetchCalls } = await buildAndLoad(); + const { dom, fetchCalls } = await loadBundle(); const win = dom.window as unknown as { elb: (event: string, data?: Record) => void; setTimeout: typeof setTimeout; @@ -183,7 +190,7 @@ describe('CDN bundle — jsdom smoke', () => { expect(last.body).toContain('page view'); dom.window.close(); - }, 120000); + }); }); /** diff --git a/packages/cli/src/__tests__/integration/runtime/hot-swap.test.ts b/packages/cli/src/__tests__/integration/runtime/hot-swap.test.ts index d70495609..b7dc623f6 100644 --- a/packages/cli/src/__tests__/integration/runtime/hot-swap.test.ts +++ b/packages/cli/src/__tests__/integration/runtime/hot-swap.test.ts @@ -111,15 +111,25 @@ describe('swapFlow', () => { expect(mockLogger.info).toHaveBeenCalledWith('Flow swapped successfully'); }); - it('keeps old handle if new bundle fails to load', async () => { + it('rolls back to the old handle (no throw) if new bundle fails to load', async () => { const oldHandle = await loadFlow(BUNDLE_V1, { port: 8080 }, mockLogger); + const oldCommand = jest.fn(); + oldHandle.collector.command = oldCommand; - await expect( - swapFlow(oldHandle, BUNDLE_INVALID, { port: 8080 }, mockLogger), - ).rejects.toThrow('Invalid bundle'); + // Atomic load-then-swap: a failed load returns the OLD handle unchanged + // instead of throwing, so the old flow keeps serving (no wedge). + const result = await swapFlow( + oldHandle, + BUNDLE_INVALID, + { port: 8080 }, + mockLogger, + ); - // Old handle should still be valid + expect(result).toBe(oldHandle); expect(oldHandle.collector).toBeDefined(); + // Old collector is left entirely untouched on a failed swap: not shut down, + // not commanded at all. + expect(oldCommand).toHaveBeenCalledTimes(0); }); }); diff --git a/packages/cli/src/__tests__/unit/bundle/bundler-codegen.test.ts b/packages/cli/src/__tests__/unit/bundle/bundler-codegen.test.ts index b6b2c0b60..1f0e97268 100644 --- a/packages/cli/src/__tests__/unit/bundle/bundler-codegen.test.ts +++ b/packages/cli/src/__tests__/unit/bundle/bundler-codegen.test.ts @@ -10,6 +10,7 @@ jest.mock('../../../commands/bundle/bundler.js', () => { }); import { + buildDataPayload, buildSplitConfigObject, createEntryPoint, detectStepPackages, @@ -1270,3 +1271,98 @@ describe('bundle() target resolution', () => { expect(buildOptions.withDev).toBe(false); }); }); + +describe('data payload byte stability', () => { + // Representative flow covering every payload branch: split source props, + // inline-code step (skipped), code-marker props (kept in code layer), + // transformers, stores, and a plain collector. + const flowSettings: Flow = { + config: { platform: 'server' }, + sources: { + api: { + package: '@walkeros/server-source-express', + config: { settings: { port: 8080 } }, + next: 'enrich', + }, + inline: { + code: { type: 'test', push: '$code:(e) => e' }, + config: { settings: { tag: 'inline' } }, + }, + }, + destinations: { + demo: { + package: '@walkeros/destination-demo', + config: { + settings: { name: 'Demo A' }, + mapping: { product: { view: { name: 'view_item' } } }, + }, + }, + coded: { + package: '@walkeros/destination-custom', + config: { settings: { fn: '$code:() => true' } }, + }, + }, + transformers: { + enrich: { + package: '@walkeros/server-transformer-fingerprint', + config: { settings: { output: 'user.hash' } }, + }, + }, + stores: { + memory: { + package: '@walkeros/server-store-fs', + config: { settings: { maxSize: 1000 } }, + }, + }, + collector: { globals: { tenant: 'a' } }, + }; + + // The exact payload object the splitter accumulates, in insertion order + // (sources, destinations, transformers, stores, collector). Steps whose + // props all carry code markers (`coded`) and inline-code steps (`inline`) + // contribute nothing. + const expectedPayload = { + sources: { + api: { + config: { settings: { port: 8080 } }, + next: 'enrich', + }, + }, + destinations: { + demo: { + config: { + settings: { name: 'Demo A' }, + mapping: { product: { view: { name: 'view_item' } } }, + }, + }, + }, + transformers: { + enrich: { + config: { settings: { output: 'user.hash' } }, + }, + }, + stores: { + memory: { + config: { settings: { maxSize: 1000 } }, + }, + }, + collector: { globals: { tenant: 'a' } }, + }; + + it('emits the data payload string byte-identical to the pinned form', () => { + const result = buildSplitConfigObject(flowSettings, new Map()); + expect(result.dataPayload).toBe(JSON.stringify(expectedPayload, null, 2)); + }); + + it('returns the payload object alongside the identical string', () => { + const result = buildSplitConfigObject(flowSettings, new Map()); + expect(result.dataPayloadObj).toEqual(expectedPayload); + expect(result.dataPayload).toBe( + JSON.stringify(result.dataPayloadObj, null, 2), + ); + }); + + it('buildDataPayload builds the same object the bundler bakes', () => { + expect(buildDataPayload(flowSettings)).toEqual(expectedPayload); + }); +}); diff --git a/packages/cli/src/__tests__/unit/commands/run-command-guard.test.ts b/packages/cli/src/__tests__/unit/commands/run-command-guard.test.ts index 291061ea9..9f0059ca4 100644 --- a/packages/cli/src/__tests__/unit/commands/run-command-guard.test.ts +++ b/packages/cli/src/__tests__/unit/commands/run-command-guard.test.ts @@ -7,6 +7,7 @@ * the health-check window and gets killed with no log). */ +import { Level } from '@walkeros/core'; import type { RunCommandOptions } from '../../../commands/run/types.js'; const prepareBundleForRun = jest.fn(); @@ -46,6 +47,7 @@ jest.mock('../../../core/cli-logger.js', () => ({ json: jest.fn(), scope: jest.fn().mockReturnValue({ info: jest.fn() }), }), + createCLILoggerConfig: () => ({ level: Level.DEBUG, handler: jest.fn() }), })); import { runCommand } from '../../../commands/run/index.js'; diff --git a/packages/cli/src/__tests__/unit/commands/run-env-file-wiring.test.ts b/packages/cli/src/__tests__/unit/commands/run-env-file-wiring.test.ts index 25542349d..23837520d 100644 --- a/packages/cli/src/__tests__/unit/commands/run-env-file-wiring.test.ts +++ b/packages/cli/src/__tests__/unit/commands/run-env-file-wiring.test.ts @@ -6,6 +6,7 @@ * - Without it: no auto-discovery, nothing is loaded. */ +import { Level } from '@walkeros/core'; import { mkdtempSync, writeFileSync, chmodSync, rmSync } from 'fs'; import { tmpdir } from 'os'; import { join } from 'path'; @@ -45,6 +46,7 @@ jest.mock('../../../core/cli-logger.js', () => ({ json: jest.fn(), scope: jest.fn().mockReturnValue({ info: jest.fn() }), }), + createCLILoggerConfig: () => ({ level: Level.DEBUG, handler: jest.fn() }), })); import { runCommand } from '../../../commands/run/index.js'; diff --git a/packages/cli/src/__tests__/unit/commands/run-pipeline-frozen.test.ts b/packages/cli/src/__tests__/unit/commands/run-pipeline-frozen.test.ts new file mode 100644 index 000000000..dec0e85f8 --- /dev/null +++ b/packages/cli/src/__tests__/unit/commands/run-pipeline-frozen.test.ts @@ -0,0 +1,231 @@ +/** + * Frozen-config run mode tests for the runtime pipeline. + * + * `WALKEROS_CONFIG_FROZEN` ('1' or 'true') marks the bundle the pipeline + * serves as an immutable snapshot: secrets are still injected at boot and the + * heartbeat keeps running so the operator retains observability, but the + * config hot-swap poller is NOT constructed. Freeze disables the in-container + * re-bundle only, never the heartbeat. Without the env var, the api-enabled + * pipeline behaves as before (secrets + heartbeat + poller). The flag value + * convention matches the package's existing boolean env flags + * (WALKEROS_TELEMETRY_DISABLED): exactly '1' or 'true' enables it, anything + * else (including '0') is off. + */ +import type { PipelineOptions } from '../../../commands/run/pipeline.js'; + +jest.mock('../../../runtime/health-server.js', () => ({ + createHealthServer: jest.fn().mockResolvedValue({ + server: {}, + setFlowHandler: jest.fn(), + setReady: jest.fn(), + setFailed: jest.fn(), + close: jest.fn().mockResolvedValue(undefined), + }), +})); + +jest.mock('../../../runtime/runner.js', () => ({ + loadFlow: jest.fn().mockResolvedValue({ + collector: { command: jest.fn(), status: undefined }, + file: '/tmp/bundle.mjs', + }), + swapFlow: jest.fn().mockResolvedValue({ + collector: { command: jest.fn(), status: undefined }, + file: '/tmp/bundle.mjs', + }), +})); + +jest.mock('../../../runtime/heartbeat.js', () => ({ + createHeartbeat: jest.fn().mockReturnValue({ + start: jest.fn(), + stop: jest.fn(), + sendOnce: jest.fn(), + updateConfigVersion: jest.fn(), + }), + getInstanceId: jest.fn().mockReturnValue('test-instance'), +})); + +jest.mock('../../../runtime/poller.js', () => ({ + createPoller: jest.fn().mockReturnValue({ + start: jest.fn(), + stop: jest.fn(), + pollOnce: jest.fn(), + }), +})); + +jest.mock('../../../runtime/trace-poller.js', () => ({ + createTracePoller: jest.fn().mockReturnValue({ + start: jest.fn(), + stop: jest.fn(), + pollOnce: jest.fn(), + }), +})); + +jest.mock('../../../runtime/secrets-fetcher.js', () => ({ + fetchSecrets: jest.fn().mockResolvedValue({}), + SecretsHttpError: class extends Error { + status: number; + constructor(status: number, text: string) { + super(`${status} ${text}`); + this.status = status; + } + }, +})); + +jest.mock('../../../runtime/cache.js', () => ({ + writeCache: jest.fn(), +})); + +jest.mock('../../../version.js', () => ({ VERSION: '0.0.0-test' })); + +import { loadFlow } from '../../../runtime/runner.js'; +import { createHeartbeat } from '../../../runtime/heartbeat.js'; +import { createPoller } from '../../../runtime/poller.js'; +import { fetchSecrets } from '../../../runtime/secrets-fetcher.js'; + +async function waitFor(cond: () => boolean, timeoutMs = 1000): Promise { + const start = Date.now(); + while (!cond()) { + if (Date.now() - start > timeoutMs) throw new Error('waitFor timed out'); + await new Promise((r) => setTimeout(r, 5)); + } +} + +const mockLogger = { + info: jest.fn(), + error: jest.fn(), + warn: jest.fn(), + debug: jest.fn(), + scope: jest.fn().mockReturnThis(), +}; + +const frozenLogLine = + 'Config frozen: hot-swap disabled (heartbeat still active)'; + +function hasFrozenLog(): boolean { + return mockLogger.info.mock.calls.some( + (call) => typeof call[0] === 'string' && call[0].includes(frozenLogLine), + ); +} + +describe('runPipeline frozen-config mode', () => { + let originalProcessOn: typeof process.on; + let runPipeline: typeof import('../../../commands/run/pipeline.js').runPipeline; + + beforeEach(async () => { + jest.clearAllMocks(); + originalProcessOn = process.on; + process.on = jest.fn() as never; + delete process.env.WALKEROS_CONFIG_FROZEN; + delete process.env.WALKEROS_OBSERVER_URL; + delete process.env.WALKEROS_INGEST_TOKEN; + delete process.env.WALKEROS_DEPLOYMENT_ID; + delete process.env.WALKEROS_OBSERVE_LEVEL; + const mod = await import('../../../commands/run/pipeline.js'); + runPipeline = mod.runPipeline; + }); + + afterEach(() => { + process.on = originalProcessOn; + delete process.env.WALKEROS_CONFIG_FROZEN; + }); + + const baseOptions: PipelineOptions = { + bundlePath: '/tmp/test-bundle.mjs', + port: 8080, + logger: mockLogger as never, + }; + + const makeApiOptions = (): PipelineOptions => ({ + ...baseOptions, + api: { + appUrl: 'https://app.walkeros.io', + token: 'test-token', + projectId: 'proj_123', + flowId: 'flow_456', + deploymentId: 'dep_789', + heartbeatIntervalMs: 60000, + pollIntervalMs: 30000, + cacheDir: '/tmp/cache', + prepareBundleForRun: jest.fn().mockResolvedValue({ + bundlePath: '/tmp/new-bundle.mjs', + cleanup: jest.fn().mockResolvedValue(undefined), + }), + }, + }); + + it('frozen=1 with full api: injects secrets at boot, keeps the heartbeat, but constructs no poller', async () => { + process.env.WALKEROS_CONFIG_FROZEN = '1'; + + void runPipeline(makeApiOptions()); + await waitFor(() => jest.mocked(createHeartbeat).mock.calls.length > 0); + + expect(fetchSecrets).toHaveBeenCalledWith({ + appUrl: 'https://app.walkeros.io', + token: 'test-token', + projectId: 'proj_123', + flowId: 'flow_456', + }); + expect(loadFlow).toHaveBeenCalledTimes(1); + expect(hasFrozenLog()).toBe(true); + // Freeze keeps the heartbeat so the operator retains observability. + expect(createHeartbeat).toHaveBeenCalled(); + // Freeze disables only the in-container re-bundle poller. + expect(createPoller).not.toHaveBeenCalled(); + }); + + it("frozen='true' with full api: also keeps the heartbeat and disables the poller", async () => { + process.env.WALKEROS_CONFIG_FROZEN = 'true'; + + void runPipeline(makeApiOptions()); + await waitFor(() => jest.mocked(createHeartbeat).mock.calls.length > 0); + + expect(hasFrozenLog()).toBe(true); + expect(createHeartbeat).toHaveBeenCalled(); + expect(createPoller).not.toHaveBeenCalled(); + }); + + it('without the env var, the api pipeline behaves as today: secrets + heartbeat + poller', async () => { + void runPipeline(makeApiOptions()); + await waitFor( + () => + jest.mocked(createHeartbeat).mock.calls.length > 0 && + jest.mocked(createPoller).mock.calls.length > 0, + ); + + expect(fetchSecrets).toHaveBeenCalled(); + expect(createHeartbeat).toHaveBeenCalledWith( + expect.objectContaining({ intervalMs: 60000 }), + mockLogger, + ); + expect(createPoller).toHaveBeenCalledWith( + expect.objectContaining({ intervalMs: 30000 }), + mockLogger, + ); + expect(hasFrozenLog()).toBe(false); + }); + + it("frozen='0' is NOT frozen: heartbeat and poller start", async () => { + process.env.WALKEROS_CONFIG_FROZEN = '0'; + + void runPipeline(makeApiOptions()); + await waitFor( + () => + jest.mocked(createHeartbeat).mock.calls.length > 0 && + jest.mocked(createPoller).mock.calls.length > 0, + ); + + expect(hasFrozenLog()).toBe(false); + }); + + it('frozen=1 without api: standalone run unchanged, frozen line still logged for self-explanatory logs', async () => { + process.env.WALKEROS_CONFIG_FROZEN = '1'; + + void runPipeline(baseOptions); + await waitFor(() => hasFrozenLog()); + + expect(loadFlow).toHaveBeenCalledTimes(1); + expect(fetchSecrets).not.toHaveBeenCalled(); + expect(createHeartbeat).not.toHaveBeenCalled(); + expect(createPoller).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli/src/__tests__/unit/commands/run-pipeline-telemetry.test.ts b/packages/cli/src/__tests__/unit/commands/run-pipeline-telemetry.test.ts index ffd6565c3..197df3e35 100644 --- a/packages/cli/src/__tests__/unit/commands/run-pipeline-telemetry.test.ts +++ b/packages/cli/src/__tests__/unit/commands/run-pipeline-telemetry.test.ts @@ -12,10 +12,29 @@ * The active trace window arrives via the trace-poller, which writes the * shared `traceUntil` holder in @walkeros/core; the pipeline supplier reads * it per emit. + * + * WALKEROS_OBSERVE_LEVEL sets the baseline telemetry level (off | standard | + * trace) fed into the resolver's `observe` block; `traceUntil` keeps higher + * priority. A trace baseline also skips the trace poller. */ -import { setTraceUntil } from '@walkeros/core'; +import { resolveTelemetryOptions, setTraceUntil } from '@walkeros/core'; +import type { FlowState, ObserverFn } from '@walkeros/core'; import type { PipelineOptions } from '../../../commands/run/pipeline.js'; +// Partial core mock: keep the real resolver semantics (spied for call +// inspection) but replace the batched poster with a captured emit so +// invoking an observer in tests performs no HTTP and no batching timers. +const mockEmit = jest.fn(); +jest.mock('@walkeros/core', () => { + const actual = + jest.requireActual('@walkeros/core'); + return { + ...actual, + createBatchedPoster: jest.fn(() => mockEmit), + resolveTelemetryOptions: jest.fn(actual.resolveTelemetryOptions), + }; +}); + jest.mock('../../../runtime/health-server.js', () => ({ createHealthServer: jest.fn().mockResolvedValue({ server: {}, @@ -81,6 +100,7 @@ jest.mock('../../../runtime/cache.js', () => ({ jest.mock('../../../version.js', () => ({ VERSION: '0.0.0-test' })); import { loadFlow } from '../../../runtime/runner.js'; +import { createTracePoller } from '../../../runtime/trace-poller.js'; async function waitFor(cond: () => boolean, timeoutMs = 1000): Promise { const start = Date.now(); @@ -109,6 +129,7 @@ describe('runPipeline telemetry wiring', () => { delete process.env.WALKEROS_OBSERVER_URL; delete process.env.WALKEROS_INGEST_TOKEN; delete process.env.WALKEROS_DEPLOYMENT_ID; + delete process.env.WALKEROS_OBSERVE_LEVEL; setTraceUntil(null); const mod = await import('../../../commands/run/pipeline.js'); runPipeline = mod.runPipeline; @@ -119,6 +140,7 @@ describe('runPipeline telemetry wiring', () => { delete process.env.WALKEROS_OBSERVER_URL; delete process.env.WALKEROS_INGEST_TOKEN; delete process.env.WALKEROS_DEPLOYMENT_ID; + delete process.env.WALKEROS_OBSERVE_LEVEL; setTraceUntil(null); }); @@ -184,4 +206,139 @@ describe('runPipeline telemetry wiring', () => { const call = (loadFlow as jest.Mock).mock.calls[0]; expect(call[5]).toBeUndefined(); }); + + describe('WALKEROS_OBSERVE_LEVEL', () => { + const setObserverEnv = () => { + process.env.WALKEROS_OBSERVER_URL = 'https://observer.example.com'; + process.env.WALKEROS_INGEST_TOKEN = 'tok_test'; + process.env.WALKEROS_DEPLOYMENT_ID = 'dep_42'; + }; + + const makeState = (): FlowState => ({ + flowId: 'flow', + stepId: 'destination.test', + stepType: 'destination', + phase: 'in', + eventId: 'evt-1', + timestamp: '2026-06-11T00:00:00.000Z', + elapsedMs: 1, + inEvent: { name: 'page view' }, + }); + + async function startAndGetObserver(): Promise { + void runPipeline(baseOptions); + await waitFor(() => jest.mocked(loadFlow).mock.calls.length > 0); + const firstCall = jest.mocked(loadFlow).mock.calls[0]; + if (!firstCall) throw new Error('loadFlow was not called'); + const observers = firstCall[5]; + const observer = observers?.[0]; + if (!observer || observers.length !== 1) { + throw new Error('expected a single telemetry observer'); + } + return observer; + } + + it('trace: per-emit supplier passes observe { level: trace } to the resolver and payloads survive projection', async () => { + setObserverEnv(); + process.env.WALKEROS_OBSERVE_LEVEL = 'trace'; + + const observer = await startAndGetObserver(); + observer(makeState()); + + const resolveSpy = jest.mocked(resolveTelemetryOptions); + expect(resolveSpy).toHaveBeenCalledWith( + expect.objectContaining({ observe: { level: 'trace' } }), + ); + expect(resolveSpy).toHaveLastReturnedWith( + expect.objectContaining({ level: 'trace' }), + ); + expect(mockEmit).toHaveBeenCalledWith( + expect.objectContaining({ inEvent: { name: 'page view' } }), + ); + }); + + it('off: telemetry resolves to null and nothing is emitted', async () => { + setObserverEnv(); + process.env.WALKEROS_OBSERVE_LEVEL = 'off'; + + const observer = await startAndGetObserver(); + observer(makeState()); + + expect(jest.mocked(resolveTelemetryOptions)).toHaveLastReturnedWith(null); + expect(mockEmit).not.toHaveBeenCalled(); + }); + + it('unset: no observe block, standard level, traceUntil still elevates', async () => { + setObserverEnv(); + + const observer = await startAndGetObserver(); + observer(makeState()); + + const resolveSpy = jest.mocked(resolveTelemetryOptions); + const firstResolveCall = resolveSpy.mock.calls[0]; + if (!firstResolveCall) throw new Error('resolver was not called'); + expect(firstResolveCall[0].observe).toBeUndefined(); + expect(resolveSpy).toHaveLastReturnedWith( + expect.objectContaining({ level: 'standard' }), + ); + expect(mockEmit).toHaveBeenCalledWith( + expect.not.objectContaining({ inEvent: expect.anything() }), + ); + + setTraceUntil(new Date(Date.now() + 60_000).toISOString()); + observer(makeState()); + expect(resolveSpy).toHaveLastReturnedWith( + expect.objectContaining({ level: 'trace' }), + ); + }); + + it('invalid value: warns once, treated as unset, poller still starts', async () => { + setObserverEnv(); + process.env.WALKEROS_OBSERVE_LEVEL = 'verbose'; + + const observer = await startAndGetObserver(); + expect(mockLogger.warn).toHaveBeenCalledWith( + expect.stringContaining('WALKEROS_OBSERVE_LEVEL'), + ); + + observer(makeState()); + expect(jest.mocked(resolveTelemetryOptions)).toHaveLastReturnedWith( + expect.objectContaining({ level: 'standard' }), + ); + + await waitFor(() => jest.mocked(createTracePoller).mock.calls.length > 0); + const pollerResult = jest.mocked(createTracePoller).mock.results[0]; + if (!pollerResult || pollerResult.type !== 'return') { + throw new Error('trace poller was not created'); + } + expect(pollerResult.value.start).toHaveBeenCalled(); + }); + + it('trace: skips starting the trace poller (nothing to elevate)', async () => { + setObserverEnv(); + process.env.WALKEROS_OBSERVE_LEVEL = 'trace'; + + void runPipeline(baseOptions); + await waitFor(() => + mockLogger.info.mock.calls.some( + (call) => + typeof call[0] === 'string' && + call[0].includes('Trace poller: skipped'), + ), + ); + expect(createTracePoller).not.toHaveBeenCalled(); + }); + + it('unset: trace poller starts when observer env is present', async () => { + setObserverEnv(); + + void runPipeline(baseOptions); + await waitFor(() => jest.mocked(createTracePoller).mock.calls.length > 0); + const pollerResult = jest.mocked(createTracePoller).mock.results[0]; + if (!pollerResult || pollerResult.type !== 'return') { + throw new Error('trace poller was not created'); + } + expect(pollerResult.value.start).toHaveBeenCalled(); + }); + }); }); diff --git a/packages/cli/src/__tests__/unit/runtime/heartbeat-counters.test.ts b/packages/cli/src/__tests__/unit/runtime/heartbeat-counters.test.ts index f6aa72ff8..cacd0a222 100644 --- a/packages/cli/src/__tests__/unit/runtime/heartbeat-counters.test.ts +++ b/packages/cli/src/__tests__/unit/runtime/heartbeat-counters.test.ts @@ -10,8 +10,14 @@ describe('computeCounterDelta', () => { out: 95, failed: 5, destinations: { - ga4: { count: 50, failed: 2, duration: 1000 }, - mixpanel: { count: 45, failed: 3, duration: 800 }, + ga4: { count: 50, failed: 2, duration: 1000, dlqSize: 0, dropped: 0 }, + mixpanel: { + count: 45, + failed: 3, + duration: 800, + dlqSize: 0, + dropped: 0, + }, }, }; const last: CounterSnapshot = { @@ -34,7 +40,7 @@ describe('computeCounterDelta', () => { out: 95, failed: 5, destinations: { - ga4: { count: 50, failed: 2, duration: 1000 }, + ga4: { count: 50, failed: 2, duration: 1000, dlqSize: 0, dropped: 0 }, }, }; const current: CounterSnapshot = { @@ -42,7 +48,7 @@ describe('computeCounterDelta', () => { out: 240, failed: 10, destinations: { - ga4: { count: 120, failed: 5, duration: 2500 }, + ga4: { count: 120, failed: 5, duration: 2500, dlqSize: 0, dropped: 0 }, }, }; const delta = computeCounterDelta(current, last); @@ -66,7 +72,13 @@ describe('computeCounterDelta', () => { out: 190, failed: 10, destinations: { - newDest: { count: 95, failed: 5, duration: 500 }, + newDest: { + count: 95, + failed: 5, + duration: 500, + dlqSize: 0, + dropped: 0, + }, }, }; const delta = computeCounterDelta(current, last); @@ -81,7 +93,7 @@ describe('computeCounterDelta', () => { out: 50, failed: 0, destinations: { - ga4: { count: 50, failed: 0, duration: 500 }, + ga4: { count: 50, failed: 0, duration: 500, dlqSize: 0, dropped: 0 }, }, }; const delta = computeCounterDelta(snapshot, snapshot); diff --git a/packages/cli/src/__tests__/unit/runtime/resolve-bundle.test.ts b/packages/cli/src/__tests__/unit/runtime/resolve-bundle.test.ts index ea0db4a0e..b28dc9c67 100644 --- a/packages/cli/src/__tests__/unit/runtime/resolve-bundle.test.ts +++ b/packages/cli/src/__tests__/unit/runtime/resolve-bundle.test.ts @@ -256,13 +256,64 @@ describe('resolveBundle', () => { ).rejects.toThrow('empty'); }); - it('should throw on network error', async () => { - isStdinPiped.mockReturnValue(false); - mockFetch.mockRejectedValue(new Error('ECONNREFUSED')); + it('should throw after retries are exhausted on a persistent network error', async () => { + // fetchOk now retries transient failures; a coded connection error is + // retried up to 3 times, then the exhaustion error surfaces. Fake timers + // drain the backoff sleeps so the test does not wait in real time. + jest.useFakeTimers(); + try { + isStdinPiped.mockReturnValue(false); + const connError = Object.assign(new Error('connect ECONNREFUSED'), { + code: 'ECONNREFUSED', + }); + mockFetch.mockRejectedValue(connError); + + const pending = resolveBundle('https://s3.example.com/bundle.mjs').then( + () => ({ ok: true as const }), + (error: unknown) => ({ ok: false as const, error }), + ); + await jest.runAllTimersAsync(); + const result = await pending; + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error).toBeInstanceOf(Error); + if (result.error instanceof Error) { + expect(result.error.message).toMatch(/after 3 attempts/); + } + } + expect(mockFetch).toHaveBeenCalledTimes(3); + } finally { + jest.useRealTimers(); + } + }); - await expect( - resolveBundle('https://s3.example.com/bundle.mjs'), - ).rejects.toThrow('ECONNREFUSED'); + it('retries a transient failure then resolves the bundle', async () => { + jest.useFakeTimers(); + try { + isStdinPiped.mockReturnValue(false); + const connError = Object.assign(new Error('connect ECONNRESET'), { + code: 'ECONNRESET', + }); + mockFetch + .mockRejectedValueOnce(connError) + .mockResolvedValueOnce( + textResponse('export default function() {}', true, 200, 'OK'), + ); + + const pending = resolveBundle('https://s3.example.com/bundle.mjs'); + await jest.runAllTimersAsync(); + const result = await pending; + + expect(result.source).toBe('url'); + expect(result.path).toBe(DEFAULT_WRITE_PATH); + expect(mockFetch).toHaveBeenCalledTimes(2); + expect(readFileSync(DEFAULT_WRITE_PATH, 'utf-8')).toBe( + 'export default function() {}', + ); + } finally { + jest.useRealTimers(); + } }); }); diff --git a/packages/cli/src/commands/bundle/__tests__/config-classifier.test.ts b/packages/cli/src/commands/bundle/__tests__/config-classifier.test.ts index a654fe259..25990e49e 100644 --- a/packages/cli/src/commands/bundle/__tests__/config-classifier.test.ts +++ b/packages/cli/src/commands/bundle/__tests__/config-classifier.test.ts @@ -127,3 +127,14 @@ describe('classifyStepProperties', () => { expect(dataProps).toEqual({}); }); }); + +describe('package-root exports', () => { + // Prod bundling and simulate-time data injection must share ONE + // classification authority: consumers import these from the package root. + it('exposes the classifier and payload builder from the package root', async () => { + const root = await import('../../../index.js'); + expect(root.containsCodeMarkers).toBe(containsCodeMarkers); + expect(root.classifyStepProperties).toBe(classifyStepProperties); + expect(typeof root.buildDataPayload).toBe('function'); + }); +}); diff --git a/packages/cli/src/commands/bundle/__tests__/package-manager-resilience.test.ts b/packages/cli/src/commands/bundle/__tests__/package-manager-resilience.test.ts new file mode 100644 index 000000000..3d21ddb42 --- /dev/null +++ b/packages/cli/src/commands/bundle/__tests__/package-manager-resilience.test.ts @@ -0,0 +1,399 @@ +import pacote from 'pacote'; +import { createMockLogger } from '@walkeros/core'; +import { + extractWithResilience, + manifestWithResilience, + PACOTE_RETRY_ATTEMPTS, +} from '../package-manager'; + +/** + * Build a fully-typed pacote manifest result so tests never cast. The shape is + * `AbbreviatedManifest & ManifestResult`; only `name`/`version` are read by + * callers, the rest satisfy the type. + */ +function makeManifest( + name: string, + version: string, +): pacote.AbbreviatedManifest & pacote.ManifestResult { + return { + name, + version, + deprecated: undefined, + dist: { + tarball: `https://registry.npmjs.org/${name}/-/${name}-${version}.tgz`, + shasum: '0'.repeat(40), + }, + _from: `${name}@${version}`, + _resolved: `https://registry.npmjs.org/${name}/-/${name}-${version}.tgz`, + _integrity: `sha512-${'0'.repeat(86)}==`, + _id: `${name}@${version}`, + }; +} + +/** Build a typed pacote extract result. */ +function makeFetchResult(name: string, version: string): pacote.FetchResult { + return { + from: `${name}@${version}`, + resolved: `https://registry.npmjs.org/${name}/-/${name}-${version}.tgz`, + integrity: `sha512-${'0'.repeat(86)}==`, + }; +} + +/** A timeout-shaped abort error, like AbortSignal.timeout/abort surfaces. */ +function timeoutError(): Error { + const err = new Error('The operation was aborted'); + err.name = 'AbortError'; + return err; +} + +/** A libuv-style transient network error. */ +function networkError(code: string): Error & { code: string } { + const err = new Error(`network failure: ${code}`); + return Object.assign(err, { code }); +} + +/** A permanent npm error carrying a `code` (E404, ETARGET, ...). */ +function npmError(code: string): Error & { code: string } { + const err = new Error(`npm error ${code}`); + return Object.assign(err, { code }); +} + +/** + * Read the `signal` off pacote opts without casting. `signal` is forwarded by + * pacote at runtime but absent from `@types/pacote`'s Options, so reflect it. + */ +function readSignal(opts: pacote.Options | undefined): AbortSignal | undefined { + if (!opts) return undefined; + const signal = Reflect.get(opts, 'signal'); + return signal instanceof AbortSignal ? signal : undefined; +} + +/** + * Small, fast retry options so backoff sleeps and per-attempt timeouts advance + * quickly under fake timers. Keeps 3 attempts and a budget large enough that the + * attempts (not the budget) are what bound the loop. + */ +const FAST_RETRY = { + attempts: PACOTE_RETRY_ATTEMPTS, + perAttemptTimeoutMs: 50, + maxTotalMs: 30_000, +}; + +/** + * Drive every pending fake timer (per-attempt abort timers + backoff sleeps) to + * completion, flushing microtasks between callbacks. `runAllTimersAsync` + * advances through timers scheduled by earlier callbacks too, so the whole + * bounded retry loop settles in one call. The retry loop is finite (attempts and + * the wall-clock budget both bound it), so this always terminates. + */ +async function flushRetries(): Promise { + await jest.runAllTimersAsync(); +} + +describe('package-manager resilient downloads', () => { + const logger = createMockLogger(); + + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + jest.restoreAllMocks(); + }); + + describe('extractWithResilience', () => { + it('retries a transient timeout then succeeds (2 calls)', async () => { + const spy = jest + .spyOn(pacote, 'extract') + .mockRejectedValueOnce(timeoutError()) + .mockResolvedValueOnce(makeFetchResult('core-js', '3.41.0')); + + const promise = extractWithResilience( + 'core-js@3.41.0', + '/dest', + {}, + logger, + FAST_RETRY, + ); + await flushRetries(); + await expect(promise).resolves.toBeDefined(); + expect(spy).toHaveBeenCalledTimes(2); + }); + + it('retries a transient ECONNRESET then succeeds', async () => { + const spy = jest + .spyOn(pacote, 'extract') + .mockRejectedValueOnce(networkError('ECONNRESET')) + .mockResolvedValueOnce(makeFetchResult('arrow', '21.0.0')); + + const promise = extractWithResilience( + 'arrow@21.0.0', + '/dest', + {}, + logger, + FAST_RETRY, + ); + await flushRetries(); + await expect(promise).resolves.toBeDefined(); + expect(spy).toHaveBeenCalledTimes(2); + }); + + it('fails fast on E404 (1 call, no retry)', async () => { + const spy = jest + .spyOn(pacote, 'extract') + .mockRejectedValue(npmError('E404')); + + const promise = extractWithResilience( + 'missing@1.0.0', + '/dest', + {}, + logger, + FAST_RETRY, + ); + const assertion = expect(promise).rejects.toThrow(); + await flushRetries(); + await assertion; + expect(spy).toHaveBeenCalledTimes(1); + }); + + it('fails fast on E403 (1 call, no retry)', async () => { + const spy = jest + .spyOn(pacote, 'extract') + .mockRejectedValue(npmError('E403')); + + const promise = extractWithResilience( + 'forbidden@1.0.0', + '/dest', + {}, + logger, + FAST_RETRY, + ); + const assertion = expect(promise).rejects.toThrow(); + await flushRetries(); + await assertion; + expect(spy).toHaveBeenCalledTimes(1); + }); + + it('throws after 3 attempts on persistent timeout (3 calls)', async () => { + const spy = jest + .spyOn(pacote, 'extract') + .mockRejectedValue(timeoutError()); + + const promise = extractWithResilience( + 'slow@1.0.0', + '/dest', + {}, + logger, + FAST_RETRY, + ); + const assertion = expect(promise).rejects.toThrow(/after 3 attempts/); + await flushRetries(); + await assertion; + expect(spy).toHaveBeenCalledTimes(PACOTE_RETRY_ATTEMPTS); + }); + + it('resolves immediately on success (1 call, no delay)', async () => { + const spy = jest + .spyOn(pacote, 'extract') + .mockResolvedValue(makeFetchResult('ok', '1.0.0')); + + const promise = extractWithResilience( + 'ok@1.0.0', + '/dest', + {}, + logger, + FAST_RETRY, + ); + await flushRetries(); + await expect(promise).resolves.toBeDefined(); + expect(spy).toHaveBeenCalledTimes(1); + }); + + it('passes an AbortSignal in the opts and aborting rejects the attempt', async () => { + let observed: AbortSignal | undefined; + const spy = jest + .spyOn(pacote, 'extract') + .mockImplementation((_spec, _dest, opts) => { + const signal = readSignal(opts); + observed = signal; + return new Promise((_resolve, reject) => { + if (signal) { + signal.addEventListener('abort', () => { + const err = new Error('aborted'); + err.name = 'AbortError'; + reject(err); + }); + } + }); + }); + + const promise = extractWithResilience( + 'hang@1.0.0', + '/dest', + {}, + logger, + FAST_RETRY, + ); + const assertion = expect(promise).rejects.toThrow(/after 3 attempts/); + await flushRetries(); + await assertion; + expect(observed).toBeInstanceOf(AbortSignal); + expect(spy).toHaveBeenCalledTimes(PACOTE_RETRY_ATTEMPTS); + }); + + it('stays within the total wall-clock budget on persistent failure', async () => { + const spy = jest + .spyOn(pacote, 'extract') + .mockRejectedValue(timeoutError()); + + // A budget too small for 3 full attempts must stop early, never overrun. + const tightBudget = { + attempts: PACOTE_RETRY_ATTEMPTS, + perAttemptTimeoutMs: 50, + maxTotalMs: 3_000, + }; + const start = Date.now(); + const promise = extractWithResilience( + 'slow@1.0.0', + '/dest', + {}, + logger, + tightBudget, + ); + const assertion = expect(promise).rejects.toThrow(); + await flushRetries(); + await assertion; + // Total elapsed (advanced by fake timers) respects the budget; with the + // first backoff (~2s) consuming most of a 3s budget, the loop stops before + // a third attempt rather than overrunning. + expect(Date.now() - start).toBeLessThanOrEqual(3_000); + expect(spy.mock.calls.length).toBeLessThanOrEqual(PACOTE_RETRY_ATTEMPTS); + }); + }); + + describe('manifestWithResilience', () => { + it('retries a transient timeout then succeeds (2 calls)', async () => { + const spy = jest + .spyOn(pacote, 'manifest') + .mockRejectedValueOnce(timeoutError()) + .mockResolvedValueOnce(makeManifest('core-js', '3.41.0')); + + const promise = manifestWithResilience( + 'core-js@^3.41.0', + {}, + logger, + FAST_RETRY, + ); + await flushRetries(); + const result = await promise; + expect(result.version).toBe('3.41.0'); + expect(spy).toHaveBeenCalledTimes(2); + }); + + it('retries a transient ECONNRESET then succeeds', async () => { + const spy = jest + .spyOn(pacote, 'manifest') + .mockRejectedValueOnce(networkError('ECONNRESET')) + .mockResolvedValueOnce(makeManifest('arrow', '21.0.0')); + + const promise = manifestWithResilience( + 'arrow@^21.0.0', + {}, + logger, + FAST_RETRY, + ); + await flushRetries(); + await expect(promise).resolves.toBeDefined(); + expect(spy).toHaveBeenCalledTimes(2); + }); + + it('fails fast on E404 (1 call, no retry)', async () => { + const spy = jest + .spyOn(pacote, 'manifest') + .mockRejectedValue(npmError('E404')); + + const promise = manifestWithResilience( + 'missing@1.0.0', + {}, + logger, + FAST_RETRY, + ); + const assertion = expect(promise).rejects.toThrow(); + await flushRetries(); + await assertion; + expect(spy).toHaveBeenCalledTimes(1); + }); + + it('fails fast on E403 (1 call, no retry)', async () => { + const spy = jest + .spyOn(pacote, 'manifest') + .mockRejectedValue(npmError('E403')); + + const promise = manifestWithResilience( + 'forbidden@1.0.0', + {}, + logger, + FAST_RETRY, + ); + const assertion = expect(promise).rejects.toThrow(); + await flushRetries(); + await assertion; + expect(spy).toHaveBeenCalledTimes(1); + }); + + it('throws after 3 attempts on persistent timeout (3 calls)', async () => { + const spy = jest + .spyOn(pacote, 'manifest') + .mockRejectedValue(timeoutError()); + + const promise = manifestWithResilience( + 'slow@1.0.0', + {}, + logger, + FAST_RETRY, + ); + const assertion = expect(promise).rejects.toThrow(/after 3 attempts/); + await flushRetries(); + await assertion; + expect(spy).toHaveBeenCalledTimes(PACOTE_RETRY_ATTEMPTS); + }); + + it('resolves immediately on success (1 call, no delay)', async () => { + const spy = jest + .spyOn(pacote, 'manifest') + .mockResolvedValue(makeManifest('ok', '1.0.0')); + + const promise = manifestWithResilience( + 'ok@1.0.0', + {}, + logger, + FAST_RETRY, + ); + await flushRetries(); + await expect(promise).resolves.toBeDefined(); + expect(spy).toHaveBeenCalledTimes(1); + }); + + it('passes an AbortSignal in the opts (3rd arg)', async () => { + let observed: AbortSignal | undefined; + const spy = jest + .spyOn(pacote, 'manifest') + .mockImplementation((_spec, opts) => { + observed = readSignal(opts); + return Promise.resolve(makeManifest('ok', '1.0.0')); + }); + + const promise = manifestWithResilience( + 'ok@1.0.0', + {}, + logger, + FAST_RETRY, + ); + await flushRetries(); + await expect(promise).resolves.toBeDefined(); + expect(observed).toBeInstanceOf(AbortSignal); + expect(spy).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/packages/cli/src/commands/bundle/bundler.ts b/packages/cli/src/commands/bundle/bundler.ts index 8bc49bd30..b8c42f78c 100644 --- a/packages/cli/src/commands/bundle/bundler.ts +++ b/packages/cli/src/commands/bundle/bundler.ts @@ -1397,11 +1397,31 @@ function createBuildError(buildError: EsbuildError, code: string): Error { ); } +/** + * Build the wireConfig data payload for a flow as a plain object. + * + * This is the same object the bundler bakes into a skeleton as + * `__configData` (section, step id, data-layer props), produced by the + * same classification pass the bundler uses. Callers that inject data at + * simulate time (the `data` option of the simulate functions) must build + * the FULL payload from the FULL config with this helper; injection + * replaces the baked payload, it does not merge. + * + * Mirrors the bundler's own call path (`detectNamedImports` feeding + * `buildSplitConfigObject`) so the result stays identical to the baked + * payload by construction. + */ +export function buildDataPayload(flowSettings: Flow): Record { + return buildSplitConfigObject(flowSettings, detectNamedImports(flowSettings)) + .dataPayloadObj; +} + /** * Build split config object from flow configuration. - * Produces TWO outputs: + * Produces THREE outputs: * - codeConfigObject: skeleton with code references and __data.* placeholders - * - dataPayload: plain JSON-serializable object with settings, mappings, etc. + * - dataPayloadObj: plain JSON-serializable object with settings, mappings, etc. + * - dataPayload: the stringified form of dataPayloadObj the codegen embeds * * Inline code steps bypass classification and go entirely to the code skeleton. * Package-based steps are split via classifyStepProperties. @@ -1412,6 +1432,7 @@ export function buildSplitConfigObject( ): { storesDeclaration: string; codeConfigObject: string; + dataPayloadObj: Record; dataPayload: string; } { const sources = flowSettings.sources || {}; @@ -1626,7 +1647,7 @@ ${destinationsEntries.join(',\n')} const dataPayload = JSON.stringify(dataPayloadObj, null, 2); - return { storesDeclaration, codeConfigObject, dataPayload }; + return { storesDeclaration, codeConfigObject, dataPayloadObj, dataPayload }; } /** diff --git a/packages/cli/src/commands/bundle/package-manager.ts b/packages/cli/src/commands/bundle/package-manager.ts index b395fe7e6..92dbddde8 100644 --- a/packages/cli/src/commands/bundle/package-manager.ts +++ b/packages/cli/src/commands/bundle/package-manager.ts @@ -9,8 +9,6 @@ import type { Logger } from '@walkeros/core'; import { getPackageCacheKey } from '../../core/cache-utils.js'; import { getTmpPath } from '../../core/tmp.js'; -export const PACKAGE_DOWNLOAD_TIMEOUT_MS = 60000; - export interface NpmConfig { registry: string; [key: string]: unknown; @@ -75,20 +73,204 @@ export async function loadNpmConfigForPacote( return { ...merged, registry, preferOnline: true }; } -export async function withTimeout( - promise: Promise, - ms: number, - errorMessage: string, -): Promise { - let timer: ReturnType; - const timeout = new Promise((_, reject) => { - timer = setTimeout(() => reject(new Error(errorMessage)), ms); +// ============================================================ +// Resilient pacote downloads +// ============================================================ + +/** + * Bounded, classified retry around a single pacote call (`extract`/`manifest`). + * + * pacote 21.x forwards `opts.signal` end-to-end (npm-registry-fetch → + * make-fetch-happen → minipass-fetch). Passing a per-attempt AbortController + * both bounds the call AND cancels the in-flight fetch, so a timed-out attempt + * does not leave a detached download running. The previous `withTimeout` race + * only rejected the wrapper while the real fetch kept running; the signal path + * replaces it. + * + * Only TRANSIENT failures (timeouts, network blips) are retried. Deterministic + * npm failures (404, auth, bad spec, integrity) fail fast. A total wall-clock + * budget clamps every attempt to the remaining time so a sequential download + * loop never holds far longer than the budget. The budget pattern mirrors + * `runtime/fetch-retry.ts` but is copied here to avoid coupling the bundler to + * the runtime. + */ + +/** Number of attempts (the first try plus retries). */ +export const PACOTE_RETRY_ATTEMPTS = 3; + +/** Per-attempt timeout before the attempt is aborted. */ +const PACOTE_PER_ATTEMPT_TIMEOUT_MS = 60_000; + +/** Total wall-clock budget across all attempts (including backoff sleeps). */ +const PACOTE_MAX_TOTAL_MS = 90_000; + +/** + * Floor of remaining budget below which starting another attempt is pointless: + * a sub-second timeout would abort before the socket connects. + */ +const MIN_ATTEMPT_BUDGET_MS = 1_000; + +/** Base backoff before retry #2 and #3. The last entry repeats if needed. */ +const BASE_BACKOFF_MS: readonly number[] = [2_000, 5_000]; + +/** Jitter band applied to each backoff: ±20%. */ +const JITTER = 0.2; + +/** + * npm error codes that are permanent: an outer retry cannot help, so fail fast. + * `EINTEGRITY` is permanent at this layer because pacote already cache-busts and + * retries it once internally; a second outer pass just re-throws. + */ +const PERMANENT_ERROR_CODES: ReadonlySet = new Set([ + 'E404', + 'ETARGET', + 'EINVALIDPACKAGENAME', + 'E401', + 'E403', + 'EINTEGRITY', + 'Z_DATA_ERROR', +]); + +/** Read an optional string `code` off an unknown error without casting. */ +function readErrorCode(value: unknown): string | undefined { + if (typeof value !== 'object' || value === null) return undefined; + const code = Reflect.get(value, 'code'); + return typeof code === 'string' ? code : undefined; +} + +/** A permanent error should not be retried (fail fast). */ +function isPermanentError(error: unknown): boolean { + const code = readErrorCode(error); + return code !== undefined && PERMANENT_ERROR_CODES.has(code); +} + +/** Backoff delay (with jitter) before the retry following attempt index `i`. */ +function backoffForAttempt(index: number): number { + const base = + BASE_BACKOFF_MS[Math.min(index, BASE_BACKOFF_MS.length - 1)] ?? 0; + const spread = base * JITTER; + return base + (Math.random() * 2 - 1) * spread; +} + +/** Promise-based sleep that fake timers can drive in tests. */ +function delay(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(resolve, ms); }); - try { - return await Promise.race([promise, timeout]); - } finally { - clearTimeout(timer!); +} + +/** Short, safe description of a thrown error for the exhaustion message. */ +function describeError(error: unknown): string { + if (error instanceof Error) return error.message; + return String(error); +} + +/** + * Attach an `AbortSignal` to pacote options. `signal` is forwarded by pacote at + * runtime (npm-registry-fetch → make-fetch-happen) but is not part of the + * `@types/pacote` Options surface; widen via intersection rather than casting. + */ +function withSignal( + opts: pacote.Options, + signal: AbortSignal, +): pacote.Options & { signal: AbortSignal } { + return { ...opts, signal }; +} + +export interface PacoteRetryOptions { + attempts?: number; + perAttemptTimeoutMs?: number; + maxTotalMs?: number; +} + +/** + * Run a pacote operation with bounded retry, a per-attempt abort, and a total + * wall-clock budget. `fn` receives the per-attempt `AbortSignal` to forward as + * `opts.signal`. Throws after attempts/budget are spent, or immediately on a + * permanent error. + */ +export async function withPacoteRetry( + fn: (signal: AbortSignal) => Promise, + label: string, + options: PacoteRetryOptions = {}, +): Promise { + const attempts = options.attempts ?? PACOTE_RETRY_ATTEMPTS; + const perAttemptTimeoutMs = + options.perAttemptTimeoutMs ?? PACOTE_PER_ATTEMPT_TIMEOUT_MS; + const maxTotalMs = options.maxTotalMs ?? PACOTE_MAX_TOTAL_MS; + + const start = Date.now(); + let lastError: unknown; + let made = 0; + + for (let attempt = 0; attempt < attempts; attempt++) { + // Clamp each attempt to the remaining budget so the total wall-clock is + // genuinely bounded. Stop if too little budget remains for a real attempt. + const remaining = maxTotalMs - (Date.now() - start); + if (remaining <= MIN_ATTEMPT_BUDGET_MS) break; + const attemptTimeoutMs = Math.min(perAttemptTimeoutMs, remaining); + + made = attempt + 1; + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), attemptTimeoutMs); + try { + return await fn(controller.signal); + } catch (error) { + lastError = error; + // Permanent failures never benefit from a retry. + if (isPermanentError(error)) throw error; + } finally { + clearTimeout(timeoutId); + } + + const isLastAttempt = attempt === attempts - 1; + const budgetSpent = Date.now() - start >= maxTotalMs; + if (isLastAttempt || budgetSpent) break; + + const sleepMs = Math.min( + backoffForAttempt(attempt), + maxTotalMs - (Date.now() - start), + ); + if (sleepMs <= 0) break; + await delay(sleepMs); } + + throw new Error( + `Failed ${label} after ${made} attempts: ${describeError(lastError)}`, + ); +} + +/** + * `pacote.extract` with resilient retry. Retrying into the same dir is safe: + * pacote empties `dest` before each extract, so no redundant pre-clean is + * needed. The `cache` opt (when present) is preserved across attempts. + */ +export function extractWithResilience( + spec: string, + dest: string, + opts: pacote.Options, + _logger: Logger.Instance, + retryOptions?: PacoteRetryOptions, +): Promise { + return withPacoteRetry( + (signal) => pacote.extract(spec, dest, withSignal(opts, signal)), + `download ${spec}`, + retryOptions, + ); +} + +/** `pacote.manifest` with resilient retry. */ +export function manifestWithResilience( + spec: string, + opts: pacote.Options, + _logger: Logger.Instance, + retryOptions?: PacoteRetryOptions, +): Promise { + return withPacoteRetry( + (signal) => pacote.manifest(spec, withSignal(opts, signal)), + `fetch manifest ${spec}`, + retryOptions, + ); } export interface Package { @@ -296,14 +478,17 @@ export async function collectAllSpecs( } let manifest: ManifestWithMeta; try { - manifest = await withTimeout( - pacote.manifest(`${item.name}@${item.spec}`, npmConfig), - PACKAGE_DOWNLOAD_TIMEOUT_MS, - `Manifest fetch timed out: ${item.name}@${item.spec}`, + manifest = await manifestWithResilience( + `${item.name}@${item.spec}`, + npmConfig, + logger, ); } catch (error) { - logger.debug( - `Failed to fetch manifest for ${item.name}@${item.spec}: ${error}`, + // A persistent manifest miss silently under-resolves the dependency + // graph (the consumer's transitive deps are never queued). Escalate to + // warn so the gap is visible, then continue resolving the rest. + logger.warn( + `Failed to fetch manifest for ${item.name}@${item.spec} after retries; its dependencies may be unresolved: ${error instanceof Error ? error.message : String(error)}`, ); continue; } @@ -464,9 +649,10 @@ export async function resolveVersionConflicts( concreteVersion = chosenVersion; } else { try { - const manifest = await pacote.manifest( + const manifest = await manifestWithResilience( `${name}@${chosenVersion}`, npmConfig, + logger, ); concreteVersion = manifest.version; } catch (err) { @@ -648,13 +834,13 @@ async function downloadPackagesImpl( await fs.ensureDir(path.dirname(packageDir)); const cacheDir = process.env.NPM_CACHE_DIR || getTmpPath(tmpDir, 'cache', 'npm'); - await withTimeout( - pacote.extract(packageSpec, packageDir, { - ...npmConfig, - cache: cacheDir, - }), - PACKAGE_DOWNLOAD_TIMEOUT_MS, - `Package download timed out after ${PACKAGE_DOWNLOAD_TIMEOUT_MS / 1000}s: ${packageSpec}`, + // Retrying into the same dir is safe: pacote empties `dest` before each + // extract, so no redundant pre-clean is needed here. + await extractWithResilience( + packageSpec, + packageDir, + { ...npmConfig, cache: cacheDir }, + logger, ); if (userSpecifiedPackages.has(name)) { @@ -683,10 +869,10 @@ async function downloadPackagesImpl( if (!semver.valid(nestedPkg.version)) { try { - const manifest = await withTimeout( - pacote.manifest(resolvedSpec, npmConfig), - PACKAGE_DOWNLOAD_TIMEOUT_MS, - `Manifest fetch timed out: ${resolvedSpec}`, + const manifest = await manifestWithResilience( + resolvedSpec, + npmConfig, + logger, ); resolvedSpec = `${nestedPkg.name}@${manifest.version}`; } catch (error) { @@ -706,13 +892,11 @@ async function downloadPackagesImpl( await fs.ensureDir(path.dirname(nestedDir)); const cacheDir = process.env.NPM_CACHE_DIR || getTmpPath(tmpDir, 'cache', 'npm'); - await withTimeout( - pacote.extract(resolvedSpec, nestedDir, { - ...npmConfig, - cache: cacheDir, - }), - PACKAGE_DOWNLOAD_TIMEOUT_MS, - `Nested package download timed out: ${resolvedSpec}`, + await extractWithResilience( + resolvedSpec, + nestedDir, + { ...npmConfig, cache: cacheDir }, + logger, ); logger.debug(`Nested: ${resolvedSpec} under ${consumer}`); } catch (error) { diff --git a/packages/cli/src/commands/push/__tests__/simulate-data-injection.test.ts b/packages/cli/src/commands/push/__tests__/simulate-data-injection.test.ts new file mode 100644 index 000000000..0d5a8cfcf --- /dev/null +++ b/packages/cli/src/commands/push/__tests__/simulate-data-injection.test.ts @@ -0,0 +1,168 @@ +/** + * Data-injection seam for simulate functions. + * + * A skeleton bundle bakes its split config data as `__configData`. The + * `data` option lets callers execute the same skeleton with a different + * data payload (the wireConfig payload shape) without rebundling. The + * injected payload REPLACES the baked one, so callers must provide the + * full payload built from the full config. + */ + +import fs from 'fs-extra'; +import os from 'os'; +import path from 'path'; +import { bundleCore } from '../../bundle/bundler.js'; +import { loadBundleConfig } from '../../../config/loader.js'; +import { simulateDestination, simulateCollector } from '../index.js'; +import { createCLILogger } from '../../../core/cli-logger.js'; +import type { Flow, Logger } from '@walkeros/core'; + +const configPath = path.resolve( + __dirname, + '../../../../examples/flow-simple.json', +); + +async function bundleFlow( + config: Flow.Json, + outputPath: string, + logger: Logger.Instance, +): Promise { + const { flowSettings, buildOptions } = loadBundleConfig(config, { + configPath, + }); + buildOptions.output = outputPath; + buildOptions.skipWrapper = true; + buildOptions.format = 'esm'; + buildOptions.cache = false; + buildOptions.minify = false; + await bundleCore(flowSettings, buildOptions, logger); +} + +describe('simulate data injection', () => { + let tmpDir: string; + let logger: Logger.Instance; + let config: Flow.Json; + let destinationBundlePath: string; + let collectorConfig: Flow.Json; + let collectorBundlePath: string; + + beforeAll(async () => { + tmpDir = path.join( + os.tmpdir(), + `simulate-data-injection-${Date.now()}-${Math.random().toString(36).slice(2)}`, + ); + await fs.ensureDir(tmpDir); + logger = createCLILogger({ silent: true }); + + // Bundle A: flow-simple as-is, baked destination name "Simple Demo" + config = (await fs.readJSON(configPath)) as Flow.Json; + destinationBundlePath = path.join(tmpDir, 'destination-bundle.mjs'); + await bundleFlow(config, destinationBundlePath, logger); + + // Bundle B: flow-simple plus a plain collector section (baked globals) + collectorConfig = structuredClone(config); + const flow = collectorConfig.flows.default; + if (!flow) throw new Error('flow-simple.json must define a default flow'); + flow.collector = { globals: { tenant: 'baked-a' } }; + collectorBundlePath = path.join(tmpDir, 'collector-bundle.mjs'); + await bundleFlow(collectorConfig, collectorBundlePath, logger); + }, 180000); + + beforeEach(() => { + jest.spyOn(console, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + afterAll(async () => { + await fs.remove(tmpDir); + }); + + it('executes the baked data when no data option is given', async () => { + const result = await simulateDestination( + config, + { name: 'entity action', data: { key: 'value' } }, + { + destinationId: 'demo', + bundlePath: destinationBundlePath, + silent: true, + }, + ); + + expect(result.error).toBeUndefined(); + const logs = result.calls.map((call) => String(call.args[0])); + expect(logs.some((line) => line.includes('[Simple Demo]'))).toBe(true); + }, 60000); + + it('simulateDestination executes the injected data payload instead of the baked one', async () => { + const result = await simulateDestination( + config, + { name: 'entity action', data: { key: 'value' } }, + { + destinationId: 'demo', + bundlePath: destinationBundlePath, + silent: true, + data: { + destinations: { + demo: { + config: { + settings: { + name: 'Injected Demo', + values: ['name', 'data'], + }, + }, + }, + }, + }, + }, + ); + + expect(result.error).toBeUndefined(); + const logs = result.calls.map((call) => String(call.args[0])); + expect(logs.some((line) => line.includes('[Injected Demo]'))).toBe(true); + expect(logs.some((line) => line.includes('[Simple Demo]'))).toBe(false); + }, 60000); + + it('simulateCollector executes the injected data payload instead of the baked one', async () => { + const baked = await simulateCollector( + collectorConfig, + { name: 'product view' }, + { + collectorName: 'default', + bundlePath: collectorBundlePath, + silent: true, + }, + ); + expect(baked.error).toBeUndefined(); + expect(baked.events[0]?.globals).toMatchObject({ tenant: 'baked-a' }); + + const injected = await simulateCollector( + collectorConfig, + { name: 'product view' }, + { + collectorName: 'default', + bundlePath: collectorBundlePath, + silent: true, + data: { + destinations: { + demo: { + config: { + settings: { + name: 'Simple Demo', + values: ['name', 'data', 'user', 'consent'], + }, + }, + }, + }, + collector: { globals: { tenant: 'injected-b' } }, + }, + }, + ); + expect(injected.error).toBeUndefined(); + expect(injected.events[0]?.globals).toMatchObject({ + tenant: 'injected-b', + }); + }, 60000); +}); diff --git a/packages/cli/src/commands/push/__tests__/simulate-mapping-key.test.ts b/packages/cli/src/commands/push/__tests__/simulate-mapping-key.test.ts new file mode 100644 index 000000000..323894369 --- /dev/null +++ b/packages/cli/src/commands/push/__tests__/simulate-mapping-key.test.ts @@ -0,0 +1,158 @@ +/** + * Matched mapping rule key on the destination simulate result. + * + * The destination push site emits per-event FlowState records carrying the + * mappingKey computed by the runtime's processEventMapping. simulateDestination + * observes those emissions, so the result reports the same rule the runtime + * matched, including against injected data payloads. + */ + +import fs from 'fs-extra'; +import os from 'os'; +import path from 'path'; +import { bundleCore } from '../../bundle/bundler.js'; +import { loadBundleConfig } from '../../../config/loader.js'; +import { simulateDestination } from '../index.js'; +import { createCLILogger } from '../../../core/cli-logger.js'; +import type { Flow, Logger } from '@walkeros/core'; + +const configPath = path.resolve( + __dirname, + '../../../../examples/flow-simple.json', +); + +const demoSettings = { + name: 'Simple Demo', + values: ['name', 'data'], +}; + +async function bundleFlow( + config: Flow.Json, + outputPath: string, + logger: Logger.Instance, +): Promise { + const { flowSettings, buildOptions } = loadBundleConfig(config, { + configPath, + }); + buildOptions.output = outputPath; + buildOptions.skipWrapper = true; + buildOptions.format = 'esm'; + buildOptions.cache = false; + buildOptions.minify = false; + await bundleCore(flowSettings, buildOptions, logger); +} + +describe('simulateDestination mappingKey', () => { + let tmpDir: string; + let logger: Logger.Instance; + let config: Flow.Json; + let bundlePath: string; + + beforeAll(async () => { + tmpDir = path.join( + os.tmpdir(), + `simulate-mapping-key-${Date.now()}-${Math.random().toString(36).slice(2)}`, + ); + await fs.ensureDir(tmpDir); + logger = createCLILogger({ silent: true }); + + // flow-simple with a two-rule mapping baked onto the demo destination. + config = (await fs.readJSON(configPath)) as Flow.Json; + const flow = config.flows.default; + if (!flow) throw new Error('flow-simple.json must define a default flow'); + flow.destinations = { + demo: { + package: '@walkeros/destination-demo', + config: { + settings: demoSettings, + mapping: { + product: { + view: {}, + add: {}, + }, + }, + }, + }, + }; + bundlePath = path.join(tmpDir, 'mapping-key-bundle.mjs'); + await bundleFlow(config, bundlePath, logger); + }, 180000); + + beforeEach(() => { + jest.spyOn(console, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + afterAll(async () => { + await fs.remove(tmpDir); + }); + + it('reports the matched rule key when the second rule matches', async () => { + const result = await simulateDestination( + config, + { name: 'product add', data: { id: 'sku-1' } }, + { + destinationId: 'demo', + bundlePath, + silent: true, + }, + ); + + expect(result.error).toBeUndefined(); + expect(result.mappingKey).toBe('product add'); + }, 60000); + + it('leaves mappingKey undefined and the result well-formed when no rule matches', async () => { + const result = await simulateDestination( + config, + { name: 'order complete' }, + { + destinationId: 'demo', + bundlePath, + silent: true, + }, + ); + + expect(result.error).toBeUndefined(); + expect(result.mappingKey).toBeUndefined(); + expect(result.step).toBe('destination'); + expect(result.name).toBe('demo'); + expect(Array.isArray(result.events)).toBe(true); + expect(Array.isArray(result.calls)).toBe(true); + expect(typeof result.duration).toBe('number'); + }, 60000); + + it('follows the injected mapping when options.data changes which rule matches', async () => { + const result = await simulateDestination( + config, + { name: 'product add', data: { id: 'sku-1' } }, + { + destinationId: 'demo', + bundlePath, + silent: true, + data: { + destinations: { + demo: { + config: { + settings: demoSettings, + mapping: { + product: { + '*': {}, + }, + }, + }, + }, + }, + }, + }, + ); + + expect(result.error).toBeUndefined(); + // The baked mapping would match 'product add'; the injected mapping only + // has the wildcard action rule, so the runtime must report 'product *'. + expect(result.mappingKey).toBe('product *'); + }, 60000); +}); diff --git a/packages/cli/src/commands/push/__tests__/simulate-result-shape.test.ts b/packages/cli/src/commands/push/__tests__/simulate-result-shape.test.ts index 61a7fc21a..5259833b3 100644 --- a/packages/cli/src/commands/push/__tests__/simulate-result-shape.test.ts +++ b/packages/cli/src/commands/push/__tests__/simulate-result-shape.test.ts @@ -112,3 +112,27 @@ describe('transformer captured→events mapping (output-only convention)', () => expect(result.calls).toEqual([]); }); }); + +describe('destination mappingKey passthrough', () => { + it('carries the matched mapping rule key when provided', () => { + const result = buildSimulationResult({ + step: 'destination', + name: 'demo', + startTime: Date.now(), + mappingKey: 'product add', + }); + + expect(result.mappingKey).toBe('product add'); + }); + + it('omits mappingKey entirely when none was captured', () => { + const result = buildSimulationResult({ + step: 'destination', + name: 'demo', + startTime: Date.now(), + }); + + expect(result.mappingKey).toBeUndefined(); + expect('mappingKey' in result).toBe(false); + }); +}); diff --git a/packages/cli/src/commands/push/index.ts b/packages/cli/src/commands/push/index.ts index f0e72a0a3..58f979576 100644 --- a/packages/cli/src/commands/push/index.ts +++ b/packages/cli/src/commands/push/index.ts @@ -5,6 +5,7 @@ import { getPlatform, getNextSteps, buildCacheContext, + stepId, } from '@walkeros/core'; import { enrichEvent, @@ -23,6 +24,7 @@ import { import type { Flow, + FlowState, Ingest, Logger, Simulation, @@ -616,7 +618,29 @@ function buildFailureSummary(failedIds: string[], collector: unknown): string { return `Push failed for ${noun}: ${labels.join(', ')}`; } -export interface SimulateSourceOptions { +/** + * Shared data-injection seam for all simulate functions. + */ +export interface SimulateDataOptions { + /** + * Wire-config data payload to execute instead of the bundle's baked + * `__configData`. Shape: the split-config data payload the bundler + * emits (section, step id, data-layer props), as built by + * `buildDataPayload`. + * + * The payload REPLACES the baked data, there is no deep-merge: build + * the full payload from the full config. Injection granularity follows + * the skeleton's `__data` references, which are emitted per TOP-LEVEL + * step prop. Changed values for any nested key under an existing + * top-level data prop (e.g. a new entity-action rule inside an existing + * `mapping`) take effect without a rebundle. An entirely NEW top-level + * data prop on a step has no `__data` reference in the skeleton, so it + * is IGNORED by injection and requires a rebundle. + */ + data?: Record; +} + +export interface SimulateSourceOptions extends SimulateDataOptions { sourceId: string; bundlePath?: string; flow?: string; @@ -718,7 +742,9 @@ export async function simulateSource( ); } - const flowConfig = module.wireConfig(module.__configData ?? undefined); + const flowConfig = module.wireConfig( + options.data ?? module.__configData ?? undefined, + ); applyOverrides(flowConfig, prepared.overrides); // Capture events at the collector.push boundary via prePush hook. @@ -786,7 +812,7 @@ export async function simulateSource( } } -export interface SimulateTransformerOptions { +export interface SimulateTransformerOptions extends SimulateDataOptions { transformerId: string; bundlePath?: string; flow?: string; @@ -887,7 +913,9 @@ export async function simulateTransformer( networkCalls, }, async (module) => { - const flowConfig = module.wireConfig(module.__configData ?? undefined); + const flowConfig = module.wireConfig( + options.data ?? module.__configData ?? undefined, + ); applyOverrides(flowConfig, prepared.overrides); // Don't initialize sources or destinations during transformer simulation. @@ -1027,7 +1055,7 @@ export async function simulateTransformer( } } -export interface SimulateCollectorOptions { +export interface SimulateCollectorOptions extends SimulateDataOptions { collectorName: string; bundlePath?: string; flow?: string; @@ -1123,7 +1151,9 @@ export async function simulateCollector( networkCalls, }, async (module) => { - const flowConfig = module.wireConfig(module.__configData ?? undefined); + const flowConfig = module.wireConfig( + options.data ?? module.__configData ?? undefined, + ); applyOverrides(flowConfig, prepared.overrides); // Don't initialize sources or destinations during collector enrichment. @@ -1185,7 +1215,7 @@ export async function simulateCollector( } } -export interface SimulateDestinationOptions { +export interface SimulateDestinationOptions extends SimulateDataOptions { destinationId: string; bundlePath?: string; flow?: string; @@ -1277,7 +1307,9 @@ export async function simulateDestination( networkCalls, }, async (module) => { - const flowConfig = module.wireConfig(module.__configData ?? undefined); + const flowConfig = module.wireConfig( + options.data ?? module.__configData ?? undefined, + ); applyOverrides(flowConfig, prepared.overrides); // Read env from bundled __devExports @@ -1341,6 +1373,27 @@ export async function simulateDestination( logger.info(`Simulating destination: ${options.destinationId}`); + // Capture the matched mapping rule key from the runtime's own + // FlowState emissions: the destination push site emits per-event + // records carrying the mappingKey that processEventMapping computed. + // Observing the wired execution keeps a single rule-matching + // authority and automatically reflects injected data payloads. + let mappingKey: string | undefined; + const targetStepId = stepId('destination', options.destinationId); + // The in, out, and error phases can all carry the key, so capture on + // presence rather than pinning a phase: a throwing destination still + // reports which rule matched. + const captureMappingKey = (state: FlowState): void => { + if (state.stepId === targetStepId && state.mappingKey) { + mappingKey = state.mappingKey; + } + }; + // Guarded: a prebuilt bundle may carry a collector without an + // observer channel; degrade to an undefined key instead of throwing. + if (collector.observers instanceof Set) { + collector.observers.add(captureMappingKey); + } + // Full pipeline: consent, mapping, enrichment, before chains // include filter ensures only the target destination receives the event await collector.push(event, { @@ -1356,6 +1409,7 @@ export async function simulateDestination( usage: trackedCalls.length ? { [options.destinationId]: trackedCalls } : undefined, + mappingKey, }); }, (error) => diff --git a/packages/cli/src/commands/push/simulation-result.ts b/packages/cli/src/commands/push/simulation-result.ts index e00decb99..fd3172b87 100644 --- a/packages/cli/src/commands/push/simulation-result.ts +++ b/packages/cli/src/commands/push/simulation-result.ts @@ -24,6 +24,8 @@ export interface BuildSimulationResultArgs { startTime: number; captured?: CapturedEntry[]; usage?: Record; + /** Entity-action key of the matched mapping rule (destination simulations). */ + mappingKey?: string; error?: unknown; } @@ -48,7 +50,7 @@ function toError(error: unknown): Error { export function buildSimulationResult( args: BuildSimulationResultArgs, ): Simulation.Result { - const { step, name, startTime, captured, usage, error } = args; + const { step, name, startTime, captured, usage, mappingKey, error } = args; const events: WalkerOS.DeepPartialEvent[] = (captured ?? []) .filter(hasEvent) @@ -66,6 +68,7 @@ export function buildSimulationResult( events, calls, duration: Date.now() - startTime, + ...(mappingKey !== undefined ? { mappingKey } : {}), ...(error !== undefined ? { error: toError(error) } : {}), }; } diff --git a/packages/cli/src/commands/run/__tests__/pipeline.test.ts b/packages/cli/src/commands/run/__tests__/pipeline.test.ts new file mode 100644 index 000000000..d12a25707 --- /dev/null +++ b/packages/cli/src/commands/run/__tests__/pipeline.test.ts @@ -0,0 +1,174 @@ +import { createLogger, Level, type Logger } from '@walkeros/core'; +import type { PipelineOptions } from '../pipeline.js'; +import { + handleUnhandledRejection, + handleUncaughtException, + registerProcessGuards, + resolveInitialEtag, + shouldStartHeartbeat, + shouldStartPoller, +} from '../pipeline.js'; + +const fakeApi: NonNullable = { + appUrl: 'https://app.example', + token: 'token', + projectId: 'project', + flowId: 'flow', + heartbeatIntervalMs: 30_000, + pollIntervalMs: 30_000, + cacheDir: '/tmp/cache', + prepareBundleForRun: async () => ({ + bundlePath: '/tmp/bundle.js', + cleanup: async () => {}, + }), +}; + +describe('resolveInitialEtag', () => { + const original = process.env.WALKEROS_CONFIG_ETAG; + + afterEach(() => { + if (original === undefined) { + delete process.env.WALKEROS_CONFIG_ETAG; + } else { + process.env.WALKEROS_CONFIG_ETAG = original; + } + }); + + it('returns the boot etag when present, ignoring the env var', () => { + process.env.WALKEROS_CONFIG_ETAG = '"env"'; + expect(resolveInitialEtag('"boot"')).toBe('"boot"'); + }); + + it('falls back to WALKEROS_CONFIG_ETAG when no boot etag is given', () => { + process.env.WALKEROS_CONFIG_ETAG = '"env"'; + expect(resolveInitialEtag(undefined)).toBe('"env"'); + }); + + it('returns undefined when neither boot etag nor env var is set', () => { + delete process.env.WALKEROS_CONFIG_ETAG; + expect(resolveInitialEtag(undefined)).toBeUndefined(); + }); +}); + +describe('shouldStartHeartbeat', () => { + it('is true when api is present (not frozen)', () => { + expect(shouldStartHeartbeat(fakeApi)).toBe(true); + }); + + it('is true when api is present even when frozen', () => { + // The heartbeat must keep running under freeze so the operator retains + // observability; freezing only disables the poller. + expect(shouldStartHeartbeat(fakeApi)).toBe(true); + }); + + it('is false when api is absent', () => { + expect(shouldStartHeartbeat(undefined)).toBe(false); + }); +}); + +describe('shouldStartPoller', () => { + it('is true only when api is present and not frozen', () => { + expect(shouldStartPoller(fakeApi, false)).toBe(true); + }); + + it('is false when frozen, even with api present', () => { + expect(shouldStartPoller(fakeApi, true)).toBe(false); + }); + + it('is false when api is absent', () => { + expect(shouldStartPoller(undefined, false)).toBe(false); + expect(shouldStartPoller(undefined, true)).toBe(false); + }); +}); + +describe('process error guards', () => { + function makeLogger(): { logger: Logger.Instance; errors: string[] } { + const errors: string[] = []; + const base = createLogger({ level: Level.DEBUG }); + const logger: Logger.Instance = { + ...base, + error: (message) => { + errors.push(typeof message === 'string' ? message : message.message); + }, + }; + return { logger, errors }; + } + + describe('handleUnhandledRejection', () => { + it('logs the rejection reason and keeps the process serving', () => { + const { logger, errors } = makeLogger(); + const exit = jest.fn(); + + handleUnhandledRejection(new Error('stray reject'), { logger, exit }); + + expect(errors.some((line) => line.includes('stray reject'))).toBe(true); + expect(exit).not.toHaveBeenCalled(); + }); + + it('logs non-Error reasons without throwing', () => { + const { logger, errors } = makeLogger(); + const exit = jest.fn(); + + handleUnhandledRejection('plain string reason', { logger, exit }); + + expect(errors.some((line) => line.includes('plain string reason'))).toBe( + true, + ); + expect(exit).not.toHaveBeenCalled(); + }); + }); + + describe('handleUncaughtException', () => { + it('logs the error and keeps the process serving for non-fatal cases', () => { + const { logger, errors } = makeLogger(); + const exit = jest.fn(); + + handleUncaughtException(new Error('stray throw'), { logger, exit }); + + expect(errors.some((line) => line.includes('stray throw'))).toBe(true); + expect(exit).not.toHaveBeenCalled(); + }); + }); + + describe('registerProcessGuards', () => { + afterEach(() => { + process.removeAllListeners('unhandledRejection'); + process.removeAllListeners('uncaughtException'); + }); + + it('registers one listener per event', () => { + const { logger } = makeLogger(); + const before = { + rejection: process.listenerCount('unhandledRejection'), + exception: process.listenerCount('uncaughtException'), + }; + + registerProcessGuards(logger); + + expect(process.listenerCount('unhandledRejection')).toBe( + before.rejection + 1, + ); + expect(process.listenerCount('uncaughtException')).toBe( + before.exception + 1, + ); + }); + + it('does not double-register when called more than once', () => { + const { logger } = makeLogger(); + + registerProcessGuards(logger); + const afterFirst = { + rejection: process.listenerCount('unhandledRejection'), + exception: process.listenerCount('uncaughtException'), + }; + registerProcessGuards(logger); + + expect(process.listenerCount('unhandledRejection')).toBe( + afterFirst.rejection, + ); + expect(process.listenerCount('uncaughtException')).toBe( + afterFirst.exception, + ); + }); + }); +}); diff --git a/packages/cli/src/commands/run/index.ts b/packages/cli/src/commands/run/index.ts index 4a0ef5577..cf30f6648 100644 --- a/packages/cli/src/commands/run/index.ts +++ b/packages/cli/src/commands/run/index.ts @@ -9,8 +9,13 @@ import path from 'path'; import { writeFileSync } from 'fs'; import { homedir } from 'os'; import { join } from 'path'; -import { createCLILogger } from '../../core/cli-logger.js'; +import { Level } from '@walkeros/core'; +import { + createCLILogger, + createCLILoggerConfig, +} from '../../core/cli-logger.js'; import { createTimer, getErrorMessage } from '../../core/index.js'; +import { ErrorRing, LogRing } from '../../runtime/index.js'; import { getTmpPath } from '../../core/tmp.js'; import { resolveAppUrl } from '../../lib/config-file.js'; import { resolveRunToken } from '../../core/auth.js'; @@ -45,7 +50,30 @@ export async function runCommand(options: RunCommandOptions): Promise { const timer = createTimer(); timer.start(); - const logger = createCLILogger(options); + const errorRing = new ErrorRing(20); + const logRing = new LogRing(100); + + const LEVEL_NAME = { + [Level.ERROR]: 'error', + [Level.WARN]: 'warn', + [Level.INFO]: 'info', + [Level.DEBUG]: 'debug', + } as const; + + const onLine = (level: Level, message: string) => { + if (level === Level.ERROR) errorRing.add(message); + logRing.add({ time: Date.now(), level: LEVEL_NAME[level], message }); + }; + + const logger = createCLILogger({ ...options, onLine }); + + // The deployed bundle's collector builds its own logger from this config + // (`context.logger`), so its destination errors flow through the SAME + // `onLine` ring tap as the runner CLI logger above. Without this, production + // (no --verbose) passes no `context.logger`, the collector's createLogger has + // no handler, and destination "Push failed" errors never reach the ErrorRing + // (the heartbeat would report "No errors reported" despite failed deliveries). + const collectorLoggerConfig = createCLILoggerConfig({ ...options, onLine }); try { // Opt-in dotenv: load BEFORE config resolution/bundling so $env/$secret @@ -123,7 +151,7 @@ export async function runCommand(options: RunCommandOptions): Promise { } // Resolve bundle path - const bundlePath = await resolveBundlePath( + const { bundlePath, bootEtag } = await resolveBundlePath( options.config, apiConfig, logger, @@ -133,10 +161,13 @@ export async function runCommand(options: RunCommandOptions): Promise { logger.info('Starting flow...'); await runPipeline({ bundlePath, + bootEtag, port, logger: logger.scope('runner'), - loggerConfig: options.verbose ? { level: 0 } : undefined, + loggerConfig: collectorLoggerConfig, api: apiConfig, + errorRing, + logRing, }); } catch (error) { const duration = timer.getElapsed() / 1000; @@ -162,12 +193,21 @@ export async function runCommand(options: RunCommandOptions): Promise { * 1. Local file (provided via CLI arg or BUNDLE env) * 2. Remote config fetch (when apiConfig is provided and no local file) * 3. Cached bundle (fallback when remote fetch fails) + * + * `bootEtag` is the etag of the config this process booted with, returned + * only on the remote-fetch path (Case 2) where it is known. It seeds the + * poller so the first poll can 304 instead of re-bundling on every restart. */ +interface ResolvedBundle { + bundlePath: string; + bootEtag?: string; +} + async function resolveBundlePath( configInput: string | undefined, apiConfig: PipelineOptions['api'] | undefined, logger: ReturnType, -): Promise { +): Promise { // Case 1: Local file or URL bundle if (configInput) { const resolved = await resolveBundle(configInput); @@ -181,7 +221,7 @@ async function resolveBundlePath( } if (isPreBuiltConfig(resolved.path)) { - return path.resolve(resolved.path); + return { bundlePath: path.resolve(resolved.path) }; } // JSON config — needs bundling @@ -192,7 +232,7 @@ async function resolveBundlePath( silent: true, flowName: apiConfig?.flowName, }); - return result.bundlePath; + return { bundlePath: result.bundlePath }; } // Runner guard: managed flow containers are started with @@ -255,7 +295,7 @@ async function resolveBundlePath( logger.debug('Cache write failed (non-critical)'); } - return bundleResult.bundlePath; + return { bundlePath: bundleResult.bundlePath, bootEtag: result.etag }; } } catch (error) { logger.error( @@ -266,7 +306,7 @@ async function resolveBundlePath( const cached = readCache(apiConfig.cacheDir); if (cached) { logger.info(`Using cached bundle (version: ${cached.version})`); - return cached.bundlePath; + return { bundlePath: cached.bundlePath }; } throw new Error( @@ -278,7 +318,7 @@ async function resolveBundlePath( // Case 3: Default — look for server-collect.mjs const defaultFile = 'server-collect.mjs'; logger.debug(`No config specified, using default: ${defaultFile}`); - return path.resolve(defaultFile); + return { bundlePath: path.resolve(defaultFile) }; } /** diff --git a/packages/cli/src/commands/run/pipeline.ts b/packages/cli/src/commands/run/pipeline.ts index 37e813616..a2b8d3a8a 100644 --- a/packages/cli/src/commands/run/pipeline.ts +++ b/packages/cli/src/commands/run/pipeline.ts @@ -4,11 +4,18 @@ * Used by both `walkeros run` (CLI) and Docker containers. * Creates health server, loads flow, and optionally enables * heartbeat/polling/secrets when API config is provided. + * + * Env surface: `WALKEROS_OBSERVER_URL` + `WALKEROS_INGEST_TOKEN` + + * `WALKEROS_DEPLOYMENT_ID` together gate telemetry and the trace poller; + * `WALKEROS_OBSERVE_LEVEL` sets the baseline telemetry level (a `trace` + * baseline also skips the trace poller); `WALKEROS_CONFIG_FROZEN` pins the + * served bundle as an immutable snapshot (secrets still injected, no + * hot-swap, no heartbeat). */ import { writeFileSync } from 'fs'; import fs from 'fs-extra'; -import type { Logger, ObserverFn } from '@walkeros/core'; +import type { Logger, ObserverFn, TelemetryLevel } from '@walkeros/core'; import { createBatchedPoster, createTelemetryObserver, @@ -38,13 +45,31 @@ import { SecretsHttpError, } from '../../runtime/secrets-fetcher.js'; import { writeCache } from '../../runtime/cache.js'; +import type { ErrorRing, LogRing } from '../../runtime/index.js'; import { VERSION } from '../../version.js'; export interface PipelineOptions { bundlePath: string; port: number; + /** + * Etag of the config this process booted with, used to seed the poller so + * the first poll can 304 instead of re-bundling the just-booted config. + * Sourced (by the caller) from the boot-time config fetch, or from + * `WALKEROS_CONFIG_ETAG` for the prebuilt-archive deploy path where the + * in-container boot never fetched the config. + */ + bootEtag?: string; logger: Logger.Instance; + /** + * Logger config handed to the deployed bundle's collector as + * `context.logger`. Its handler taps the same ErrorRing/LogRing as the + * runner CLI logger, so the collector's destination errors land in the ring + * (and the heartbeat report) even without `--verbose`. The level is DEBUG so + * the handler controls visibility; ERROR is always emitted into the ring. + */ loggerConfig?: Logger.Config; + errorRing?: ErrorRing; + logRing?: LogRing; api?: { appUrl: string; token: string; @@ -70,6 +95,7 @@ export interface PipelineOptions { export async function runPipeline(options: PipelineOptions): Promise { const { bundlePath, port, logger, loggerConfig, api } = options; let configVersion: string | undefined; + const configFrozen = readConfigFrozen(); // Inject secrets before loading flow if (api) { @@ -78,6 +104,9 @@ export async function runPipeline(options: PipelineOptions): Promise { logger.info(`walkeros/flow v${VERSION}`); logger.info(`Instance: ${getInstanceId()}`); + if (configFrozen) { + logger.info('Config frozen: hot-swap disabled (heartbeat still active)'); + } // Health server (always on) const healthServer = await createHealthServer(port, logger); @@ -87,8 +116,14 @@ export async function runPipeline(options: PipelineOptions): Promise { // API) results in a no-op telemetry path. The active trace window arrives // via the trace-poller below (which writes the shared `traceUntil` holder); // the per-emit supplier reads it, so trace flips on and off at runtime - // without a redeploy. - const telemetryObservers = buildTelemetryObservers(api?.flowId ?? 'flow'); + // without a redeploy. WALKEROS_OBSERVE_LEVEL sets the baseline telemetry + // level for the process; `traceUntil` keeps higher priority in the + // resolver, so a standard (or off) baseline can still be elevated to trace. + const observeLevel = readObserveLevel(logger); + const telemetryObservers = buildTelemetryObservers( + api?.flowId ?? 'flow', + observeLevel, + ); // Load flow const runtimeConfig: RuntimeConfig = { port }; @@ -132,23 +167,31 @@ export async function runPipeline(options: PipelineOptions): Promise { const ingestToken = process.env.WALKEROS_INGEST_TOKEN; const deploymentId = process.env.WALKEROS_DEPLOYMENT_ID; if (observerBase && ingestToken && deploymentId) { - tracePoller = createTracePoller( - { - url: `${observerBase}/trace/${deploymentId}`, - token: ingestToken, - intervalMs: 15_000, - }, - logger, - ); - tracePoller.start(); - logger.info('Trace poller: active (every 15s)'); + if (observeLevel === 'trace') { + // The poller only exists to elevate the level via `traceUntil`. With a + // trace baseline there is nothing to elevate, so polling would burn a + // request every interval for no effect. + logger.info('Trace poller: skipped (observe level is trace)'); + } else { + tracePoller = createTracePoller( + { + url: `${observerBase}/trace/${deploymentId}`, + token: ingestToken, + intervalMs: 15_000, + }, + logger, + ); + tracePoller.start(); + logger.info('Trace poller: active (every 15s)'); + } } // Track temp files for cleanup on hot-swap and shutdown let currentBundleCleanup: (() => Promise) | undefined; let currentConfigPath: string | undefined; - if (api) { + if (shouldStartHeartbeat(api)) { + // Heartbeat runs even under freeze so the operator keeps observability. heartbeat = createHeartbeat( { appUrl: api.appUrl, @@ -159,12 +202,16 @@ export async function runPipeline(options: PipelineOptions): Promise { configVersion, intervalMs: api.heartbeatIntervalMs, getCounters: () => handle.collector.status, + getErrors: () => options.errorRing?.snapshot() ?? [], + getLogs: () => options.logRing?.snapshot() ?? [], }, logger, ); heartbeat.start(); logger.info(`Heartbeat: active (every ${api.heartbeatIntervalMs / 1000}s)`); + } + if (api && shouldStartPoller(api, configFrozen)) { poller = createPoller( { fetchOptions: { @@ -174,6 +221,7 @@ export async function runPipeline(options: PipelineOptions): Promise { flowId: api.flowId, }, intervalMs: api.pollIntervalMs, + initialEtag: resolveInitialEtag(options.bootEtag), onUpdate: async (content, version) => { // Refresh secrets before hot-swap try { @@ -201,10 +249,13 @@ export async function runPipeline(options: PipelineOptions): Promise { flowName: api.flowName, }); - // Keep /ready honest across the swap: not ready until the new - // collector is constructed. If swapFlow throws, readiness stays off. - healthServer.setReady(false); - handle = await swapFlow( + // swapFlow is atomic with rollback: it loads the new bundle into a + // fresh handle first and only mounts it on success, otherwise it + // returns the OLD handle unchanged and the old flow keeps serving. + // So readiness must NOT drop before the swap — a failed swap must + // leave /ready true (no wedge). When the swap succeeds the handler is + // already mounted atomically; readiness was never lost. + const swapped = await swapFlow( handle, newBundleResult.bundlePath, runtimeConfig, @@ -213,7 +264,15 @@ export async function runPipeline(options: PipelineOptions): Promise { healthServer, telemetryObservers, ); - healthServer.setReady(true); + + // Rollback case: handle unchanged. Skip cache write and version + // bookkeeping so a failed swap doesn't record the unbuilt version. + if (swapped === handle) { + await newBundleResult.cleanup().catch(() => {}); + await fs.remove(tmpConfigPath).catch(() => {}); + return; + } + handle = swapped; writeCache( api.cacheDir, @@ -280,10 +339,158 @@ export async function runPipeline(options: PipelineOptions): Promise { process.on('SIGTERM', () => shutdown('SIGTERM')); process.on('SIGINT', () => shutdown('SIGINT')); + // Process-level safety net: a stray unhandled rejection or uncaught + // exception (e.g. from the dynamically imported bundle) must degrade, not + // crash the container. The guards log into the error ring via the logger and + // keep the process serving; the orchestrator's own /ready gate still governs + // traffic. Registration is idempotent in case runPipeline runs more than once. + registerProcessGuards(logger); + // Keep process alive await new Promise(() => {}); } +/** + * Dependencies the process guards need, injectable so the handler bodies are + * unit-testable without registering real `process` listeners or exiting. + */ +export interface ProcessGuardDeps { + logger: Logger.Instance; + exit: (code: number) => void; +} + +/** + * Handle an `unhandledRejection`: log the reason into the error ring (via the + * logger) and keep serving. A stray rejection is treated as non-fatal — the + * container degrades instead of crash-looping. + */ +export function handleUnhandledRejection( + reason: unknown, + deps: ProcessGuardDeps, +): void { + deps.logger.error( + `Unhandled rejection (continuing): ${ + reason instanceof Error ? reason.message : String(reason) + }`, + ); +} + +/** + * Handle an `uncaughtException`: log the error into the error ring (via the + * logger) and keep serving for non-fatal cases. `process.exit` is reserved for + * genuinely unrecoverable state (handled by the shutdown orchestrator on + * signals), not for a single stray throw. + */ +export function handleUncaughtException( + error: Error, + deps: ProcessGuardDeps, +): void { + deps.logger.error(`Uncaught exception (continuing): ${error.message}`); +} + +let processGuardsRegistered = false; + +/** + * Register the process-level error guards exactly once per process. Guards + * against double-registration so a second `runPipeline` call in the same + * process does not stack listeners (which would multiply log lines). + */ +export function registerProcessGuards(logger: Logger.Instance): void { + if (processGuardsRegistered) return; + processGuardsRegistered = true; + + const deps: ProcessGuardDeps = { + logger, + exit: (code) => process.exit(code), + }; + + process.on('unhandledRejection', (reason) => + handleUnhandledRejection(reason, deps), + ); + process.on('uncaughtException', (error) => + handleUncaughtException(error, deps), + ); +} + +/** + * Resolve the poller's seed etag. The boot-time config fetch (Case 2 of the + * run command) knows the etag and wins; the prebuilt-archive deploy path + * never fetches the config in-container, so it falls back to + * `WALKEROS_CONFIG_ETAG`. Neither present means the first poll sends no + * `If-None-Match` (the safe legacy behavior). + */ +export function resolveInitialEtag(bootEtag?: string): string | undefined { + return bootEtag ?? process.env.WALKEROS_CONFIG_ETAG; +} + +/** + * The heartbeat runs whenever the flow has API credentials, regardless of the + * frozen gate. Freezing a deployed flow disables the in-container re-bundle + * (the poller); it must NOT silence the heartbeat, or the operator loses all + * runner observability (counters, errors, logs) on frozen production flows. + */ +export function shouldStartHeartbeat( + api: PipelineOptions['api'], +): api is NonNullable { + return Boolean(api); +} + +/** + * The poller (config hot-swap via in-container re-bundle) runs only when the + * flow has API credentials AND is not frozen. `WALKEROS_CONFIG_FROZEN` exists + * to stop the re-bundle crash-loop, so it gates the poller alone. + */ +export function shouldStartPoller( + api: PipelineOptions['api'], + configFrozen: boolean, +): boolean { + return Boolean(api) && !configFrozen; +} + +const OBSERVE_LEVELS: ReadonlyArray = [ + 'off', + 'standard', + 'trace', +]; + +/** + * Read `WALKEROS_CONFIG_FROZEN` once at pipeline start (`'1'` or `'true'` + * enables it, matching the package's boolean env convention; anything else, + * including `'0'`, is off). + * + * Frozen mode is the immutable-bundle contract: a runtime serving a config + * snapshot must never hot-swap itself to a newer config, so the config + * poller is not constructed. The heartbeat is skipped for the same reason: + * it reports the config-version lifecycle of a hot-swappable runtime, which + * an immutable snapshot does not have. Secrets are still injected at boot, + * and the health server, telemetry, and trace poller are unaffected. + */ +function readConfigFrozen(): boolean { + const raw = process.env.WALKEROS_CONFIG_FROZEN; + return raw === '1' || raw === 'true'; +} + +/** + * Read and validate `WALKEROS_OBSERVE_LEVEL` once at boot. The value sets + * the runtime's baseline telemetry level (`off` | `standard` | `trace`). + * The resolver gives the runtime `traceUntil` window higher priority, so a + * baseline of `off` or `standard` can still be elevated to trace at runtime. + * Unset or empty means the resolver's standard default applies. Invalid + * values are logged and ignored (treated as unset). + */ +function readObserveLevel(logger: Logger.Instance): TelemetryLevel | undefined { + const raw = process.env.WALKEROS_OBSERVE_LEVEL; + if (raw === undefined || raw === '') return undefined; + const level = OBSERVE_LEVELS.find((candidate) => candidate === raw); + if (!level) { + logger.warn( + `Ignoring invalid WALKEROS_OBSERVE_LEVEL "${raw}" (expected off, standard, or trace)`, + ); + return undefined; + } + return level; +} + /** * Build the telemetry observer array the runtime forwards through the * bundle context. Returns undefined when telemetry is disabled (missing env @@ -296,10 +503,13 @@ export async function runPipeline(options: PipelineOptions): Promise { * the batch buffer. Projection-level opts (`level`, `sample`, * `includeIn`/`Out`/`MappingKey`, plus the active `traceUntil`) are * re-resolved per emit through the supplier so the trace-poller's writes to - * the shared holder reach the projection. + * the shared holder reach the projection. The optional `observeLevel` + * baseline (from `WALKEROS_OBSERVE_LEVEL`) feeds the resolver's `observe` + * block; `traceUntil` keeps its higher priority. */ function buildTelemetryObservers( flowId: string, + observeLevel?: TelemetryLevel, ): Array | undefined { const base = process.env.WALKEROS_OBSERVER_URL; const token = process.env.WALKEROS_INGEST_TOKEN; @@ -308,9 +518,11 @@ function buildTelemetryObservers( const url = `${base}/ingest/${deploymentId}`; const emit = createBatchedPoster({ url, token }); + const observe = + observeLevel !== undefined ? { level: observeLevel } : undefined; return [ createTelemetryObserver(emit, () => - resolveTelemetryOptions({ flowId, traceUntil: getTraceUntil() }), + resolveTelemetryOptions({ flowId, observe, traceUntil: getTraceUntil() }), ), ]; } diff --git a/packages/cli/src/core/__tests__/cli-logger-config.test.ts b/packages/cli/src/core/__tests__/cli-logger-config.test.ts new file mode 100644 index 000000000..788d9b832 --- /dev/null +++ b/packages/cli/src/core/__tests__/cli-logger-config.test.ts @@ -0,0 +1,118 @@ +import { createLogger, Level } from '@walkeros/core'; +import { createCLILoggerConfig } from '../cli-logger.js'; +import { ErrorRing, LogRing } from '../../runtime/index.js'; + +/** + * D1: the collector logger config must feed the SAME ring tap the runner CLI + * logger uses, so a deployed bundle's destination errors land in the + * ErrorRing (and therefore the heartbeat report). These tests drive the + * config through `createLogger` exactly as the collector does. + */ +describe('createCLILoggerConfig (collector ring tap)', () => { + let errorSpy: jest.SpyInstance; + let logSpy: jest.SpyInstance; + + beforeEach(() => { + errorSpy = jest.spyOn(console, 'error').mockImplementation(() => undefined); + logSpy = jest.spyOn(console, 'log').mockImplementation(() => undefined); + }); + + afterEach(() => { + errorSpy.mockRestore(); + logSpy.mockRestore(); + }); + + it('routes a scoped destination error into the error ring (no --verbose)', () => { + const errorRing = new ErrorRing(20); + const logRing = new LogRing(100); + + const config = createCLILoggerConfig({ + verbose: false, + silent: true, + onLine: (level, message) => { + if (level === Level.ERROR) errorRing.add(message); + logRing.add({ time: Date.now(), level: 'error', message }); + }, + }); + + // Drive the collector path: scoped destination logger emits "Push failed". + const collectorLogger = createLogger(config); + collectorLogger + .scope('bq') + .error('Push failed', { event: 'order complete' }); + + const snapshot = errorRing.snapshot(); + expect(snapshot).toHaveLength(1); + expect(snapshot[0].message).toBe('[bq] Push failed'); + expect(snapshot[0].count).toBe(1); + }); + + it('sets the level to at least ERROR so errors emit without --verbose', () => { + const config = createCLILoggerConfig({ verbose: false, silent: true }); + expect(config.level).toBeDefined(); + // Level.ERROR is 0; "at least ERROR" means the gate admits ERROR. + const numericLevel = + typeof config.level === 'number' + ? config.level + : Level[config.level as keyof typeof Level]; + expect(numericLevel).toBeGreaterThanOrEqual(Level.ERROR); + }); + + it('does not pollute the error ring with non-error levels', () => { + const errorRing = new ErrorRing(20); + const logRing = new LogRing(100); + + const config = createCLILoggerConfig({ + verbose: true, + silent: false, + onLine: (level, message) => { + if (level === Level.ERROR) errorRing.add(message); + logRing.add({ + time: Date.now(), + level: + level === Level.ERROR + ? 'error' + : level === Level.WARN + ? 'warn' + : level === Level.INFO + ? 'info' + : 'debug', + message, + }); + }, + }); + + const logger = createLogger(config); + logger.info('just info'); + logger.warn('a warning'); + logger.error('a real error'); + + // Only the error lands in the error ring. + const errors = errorRing.snapshot(); + expect(errors).toHaveLength(1); + expect(errors[0].message).toBe('a real error'); + + // Non-error levels still reach the log ring (existing behavior). + const logs = logRing.snapshot(); + expect(logs.map((l) => l.message)).toEqual([ + 'just info', + 'a warning', + 'a real error', + ]); + }); + + it('uses the identical handler as createCLILogger (no parallel ring)', () => { + // The config's handler IS the same function the CLI logger uses, so a + // single onLine tap feeds both runner-process logs and collector logs. + const lines: Array<{ level: Level; message: string }> = []; + const onLine = (level: Level, message: string) => { + lines.push({ level, message }); + }; + + const config = createCLILoggerConfig({ verbose: false, onLine }); + const collectorLogger = createLogger(config); + collectorLogger.scope('gtag').error('boom'); + + expect(lines).toEqual([{ level: Level.ERROR, message: '[gtag] boom' }]); + }); +}); diff --git a/packages/cli/src/core/__tests__/cli-logger-redact.test.ts b/packages/cli/src/core/__tests__/cli-logger-redact.test.ts new file mode 100644 index 000000000..04914b044 --- /dev/null +++ b/packages/cli/src/core/__tests__/cli-logger-redact.test.ts @@ -0,0 +1,147 @@ +import { Level } from '@walkeros/core'; +import type { Logger } from '@walkeros/core'; +import { createCLILogger, createCLILoggerConfig } from '../cli-logger.js'; + +// No-op default handler to satisfy the 5-arg Handler signature when calling the +// config handler directly. The handler under test ignores it. +const noopDefaultHandler: Logger.DefaultHandler = () => undefined; + +/** + * The CLI logger handler must redact secrets ONCE, before BOTH the `onLine` + * ring tap and the `console.*` output, so stderr (shipped directly by + * Cockpit/Loki) and the heartbeat ring are both scrubbed for every line. + */ +describe('createCLILogger handler redaction', () => { + let errorSpy: jest.SpyInstance; + let logSpy: jest.SpyInstance; + + beforeEach(() => { + errorSpy = jest.spyOn(console, 'error').mockImplementation(() => undefined); + logSpy = jest.spyOn(console, 'log').mockImplementation(() => undefined); + }); + + afterEach(() => { + errorSpy.mockRestore(); + logSpy.mockRestore(); + }); + + function joinedConsoleOutput(): string { + const calls: unknown[][] = [...errorSpy.mock.calls, ...logSpy.mock.calls]; + return calls.map((c) => c.map((a) => String(a)).join(' ')).join('\n'); + } + + it('redacts a PEM block in both console output and the onLine tap', () => { + const captured: string[] = []; + const logger = createCLILogger({ + onLine: (_level, message) => captured.push(message), + }); + + const pem = [ + '-----BEGIN PRIVATE KEY-----', + 'MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7', + '-----END PRIVATE KEY-----', + ].join('\n'); + logger.error(`init failed\n${pem}`); + + const body = 'MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7'; + expect(captured.join('\n')).not.toContain(body); + expect(captured.join('\n')).not.toContain('BEGIN PRIVATE KEY'); + expect(joinedConsoleOutput()).not.toContain(body); + expect(joinedConsoleOutput()).not.toContain('BEGIN PRIVATE KEY'); + }); + + it('redacts a private_key JSON value in both sinks', () => { + const captured: string[] = []; + const logger = createCLILogger({ + onLine: (_level, message) => captured.push(message), + }); + + const secret = 'MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcw=='; + logger.error( + `boom {"private_key":"-----BEGIN PRIVATE KEY-----\\n${secret}\\n-----END PRIVATE KEY-----\\n"}`, + ); + + expect(captured.join('\n')).not.toContain(secret); + expect(joinedConsoleOutput()).not.toContain(secret); + }); + + it('redacts a client_email JSON value in both sinks', () => { + const captured: string[] = []; + const logger = createCLILogger({ + onLine: (_level, message) => captured.push(message), + }); + + const email = 'svc@my-proj.iam.gserviceaccount.com'; + logger.error(`auth failed {"client_email":"${email}"}`); + + expect(captured.join('\n')).not.toContain(email); + expect(joinedConsoleOutput()).not.toContain(email); + }); + + it('redacts a client_x509_cert_url JSON value in both sinks', () => { + const captured: string[] = []; + const logger = createCLILogger({ + onLine: (_level, message) => captured.push(message), + }); + + const url = + 'https://www.googleapis.com/robot/v1/metadata/x509/svc%40my-proj.iam.gserviceaccount.com'; + logger.error(`cert error {"client_x509_cert_url":"${url}"}`); + + expect(captured.join('\n')).not.toContain( + 'svc%40my-proj.iam.gserviceaccount.com', + ); + expect(joinedConsoleOutput()).not.toContain( + 'svc%40my-proj.iam.gserviceaccount.com', + ); + }); + + it('leaves a normal line unchanged in both sinks', () => { + const captured: string[] = []; + const logger = createCLILogger({ + onLine: (_level, message) => captured.push(message), + }); + + logger.info('Connected to BigQuery dataset analytics'); + + expect(captured).toEqual(['Connected to BigQuery dataset analytics']); + expect(joinedConsoleOutput()).toContain( + 'Connected to BigQuery dataset analytics', + ); + }); + + it('redaction runs before the onLine tap (ring snapshot is already scrubbed)', () => { + // A credential routed through the collector path (D1 wiring): the deployed + // bundle builds its logger from createCLILoggerConfig, so its destination + // errors flow through the identical onLine tap. The snapshot must be clean. + const ringMessages: string[] = []; + const config = createCLILoggerConfig({ + verbose: false, + silent: true, + onLine: (level, message) => { + if (level === Level.ERROR) ringMessages.push(message); + }, + }); + + // Drive the collector path exactly as the D1 tests do. + const secret = 'MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcw=='; + config.handler?.( + Level.ERROR, + `BigQuery init failed: {"private_key":"-----BEGIN PRIVATE KEY-----\\n${secret}\\n-----END PRIVATE KEY-----\\n"}`, + {}, + ['bq'], + noopDefaultHandler, + ); + + expect(ringMessages).toHaveLength(1); + expect(ringMessages[0]).not.toContain(secret); + expect(ringMessages[0]).toContain('BigQuery init failed'); + }); + + it('does not over-truncate a long non-secret message in console (legibility)', () => { + const logger = createCLILogger({ verbose: true }); + const long = 'diagnostic detail '.repeat(40); + logger.info(long); + expect(joinedConsoleOutput()).toContain(long.trimEnd()); + }); +}); diff --git a/packages/cli/src/core/__tests__/cli-logger.test.ts b/packages/cli/src/core/__tests__/cli-logger.test.ts new file mode 100644 index 000000000..c170c337c --- /dev/null +++ b/packages/cli/src/core/__tests__/cli-logger.test.ts @@ -0,0 +1,58 @@ +import { Level } from '@walkeros/core'; +import { createCLILogger } from '../cli-logger.js'; + +describe('createCLILogger onLine hook', () => { + let errorSpy: jest.SpyInstance; + let logSpy: jest.SpyInstance; + + beforeEach(() => { + errorSpy = jest.spyOn(console, 'error').mockImplementation(() => undefined); + logSpy = jest.spyOn(console, 'log').mockImplementation(() => undefined); + }); + + afterEach(() => { + errorSpy.mockRestore(); + logSpy.mockRestore(); + }); + + it('captures ERROR, INFO, and scoped-child lines via onLine', () => { + const captured: Array<{ level: Level; message: string }> = []; + + const logger = createCLILogger({ + onLine: (level, message) => { + captured.push({ level, message }); + }, + }); + + logger.error('boom'); + logger.info('hi'); + logger.scope('dest').error('nested'); + + expect(captured).toHaveLength(3); + expect(captured[0]).toEqual({ level: Level.ERROR, message: 'boom' }); + expect(captured[1]).toEqual({ level: Level.INFO, message: 'hi' }); + expect(captured[2]).toEqual({ + level: Level.ERROR, + message: '[dest] nested', + }); + }); + + it('does not break logging if onLine throws', () => { + const logger = createCLILogger({ + onLine: () => { + throw new Error('consumer error'); + }, + }); + + // Should not throw, and console.error should still be called for ERROR level + expect(() => logger.error('boom')).not.toThrow(); + expect(errorSpy).toHaveBeenCalled(); + }); + + it('works normally without onLine', () => { + const logger = createCLILogger({ silent: false }); + + expect(() => logger.info('hello')).not.toThrow(); + expect(logSpy).toHaveBeenCalledWith('hello'); + }); +}); diff --git a/packages/cli/src/core/__tests__/redact-line.test.ts b/packages/cli/src/core/__tests__/redact-line.test.ts new file mode 100644 index 000000000..806e1ff59 --- /dev/null +++ b/packages/cli/src/core/__tests__/redact-line.test.ts @@ -0,0 +1,179 @@ +import { scrubSecrets, redactLine } from '../redact-line.js'; + +describe('scrubSecrets', () => { + it('masks a high-entropy token', () => { + const token = 'AKIAIOSFODNN7EXAMPLEKEY1234ABCD'; + expect(scrubSecrets(`token is ${token} here`)).toBe('token is *** here'); + }); + + it('masks a JSON private_key value', () => { + const json = + '{"type":"service_account","private_key":"-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBg==\\n-----END PRIVATE KEY-----\\n"}'; + const result = scrubSecrets(json); + expect(result).not.toContain('MIIEvQIBADANBg=='); + }); + + it('masks a client_email value but keeps the key name', () => { + const json = + '{"client_email":"svc@my-proj.iam.gserviceaccount.com","type":"service_account"}'; + const result = scrubSecrets(json); + expect(result).not.toContain('svc@my-proj.iam.gserviceaccount.com'); + expect(result).toContain('client_email'); + }); + + it('masks a client_x509_cert_url value', () => { + const json = + '{"client_x509_cert_url":"https://www.googleapis.com/robot/v1/metadata/x509/svc%40my-proj.iam.gserviceaccount.com"}'; + const result = scrubSecrets(json); + expect(result).not.toContain('svc%40my-proj.iam.gserviceaccount.com'); + expect(result).toContain('***'); + }); + + it('removes a multi-line PEM private key block', () => { + const pem = [ + '-----BEGIN PRIVATE KEY-----', + 'MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7', + '-----END PRIVATE KEY-----', + ].join('\n'); + const result = scrubSecrets(`prefix\n${pem}\nsuffix`); + expect(result).not.toContain('BEGIN PRIVATE KEY'); + expect(result).not.toContain( + 'MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7', + ); + expect(result).toContain('prefix'); + expect(result).toContain('suffix'); + }); + + it('leaves a normal diagnostic line unchanged', () => { + const msg = 'Error: BigQuery 403 PERMISSION_DENIED on dataset analytics'; + expect(scrubSecrets(msg)).toBe(msg); + }); + + it('does NOT truncate long non-secret messages (console legibility)', () => { + const long = 'a normal long message '.repeat(50); + // scrubSecrets must preserve the full message; only redactLine truncates. + expect(scrubSecrets(long)).toBe(long); + expect(scrubSecrets(long).length).toBeGreaterThan(256); + }); + + // ReDoS: a BEGIN marker with no END plus a large run must complete fast. + it('handles a BEGIN-without-END PEM with a large run in under 1000ms', () => { + const input = '-----BEGIN PRIVATE KEY-----\n' + 'A'.repeat(200_000); + const start = Date.now(); + const result = scrubSecrets(input); + const elapsed = Date.now() - start; + expect(elapsed).toBeLessThan(1000); + expect(typeof result).toBe('string'); + expect(result).not.toContain('BEGIN PRIVATE KEY'); + }); + + // ── Path/URL legibility: `/` is structural, not a secret signal ─────────── + // These run on every console line now, so ordinary BigQuery/GCS diagnostics + // (FQNs, gs:// paths, request paths, googleapis URLs) must survive UNMASKED. + describe('does not over-redact paths and URLs', () => { + it.each([ + 'analytics.dataset.events_table', + 'gs://my-bucket/path/to/object.json', + 'GET /api/users/12345/settings 200', + 'https://bigquery.googleapis.com/v2/projects/p/datasets/d', + ])('leaves %s unmasked', (line) => { + const result = scrubSecrets(line); + expect(result).toBe(line); + expect(result).not.toContain('***'); + }); + }); + + // ── Real secrets still masked (the path/URL relaxation must not leak) ────── + describe('still masks real secrets', () => { + it('masks a base64 token containing + / = padding', () => { + const token = 'aGVsbG8gd29ybGQgdGhpcy9pcythL3Rlc3Q9'; // has / and = + const result = scrubSecrets(`body ${token} end`); + expect(result).not.toContain(token); + expect(result).toContain('***'); + }); + + it('masks a high-entropy token even when it contains a slash', () => { + // 31-char + 8-char segments split on `/`; the first segment is itself a + // high-entropy secret, so the whole run is masked. + const token = 'kJ8sL2mNqRtVxYzAbCdEfGhIjKlMnOp/QrStUvWx'; + const result = scrubSecrets(`token ${token} ok`); + expect(result).not.toContain('kJ8sL2mNqRtVxYzAbCdEfGhIjKlMnOp'); + expect(result).toContain('***'); + }); + + it('masks an all-hex digest', () => { + const digest = 'deadbeefdeadbeefdeadbeefdeadbeef'; + const result = scrubSecrets(`checksum ${digest}`); + expect(result).not.toContain(digest); + expect(result).toContain('***'); + }); + + it('masks a known-prefix token', () => { + const result = scrubSecrets('key is AKIAIOSFODNN7EXAMPLE done'); + expect(result).not.toContain('AKIAIOSFODNN7EXAMPLE'); + expect(result).toContain('***'); + }); + }); + + // ── PEM + all five SA-JSON fields unaffected by the path relaxation ──────── + describe('PEM and SA-JSON fields remain masked', () => { + it('removes a PEM block', () => { + const pem = [ + '-----BEGIN PRIVATE KEY-----', + 'MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7', + '-----END PRIVATE KEY-----', + ].join('\n'); + const result = scrubSecrets(`x\n${pem}\ny`); + expect(result).not.toContain('BEGIN PRIVATE KEY'); + expect(result).not.toContain( + 'MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7', + ); + }); + + it.each([ + [ + 'private_key', + '-----BEGIN PRIVATE KEY-----\\nMIIEvQ==\\n-----END PRIVATE KEY-----\\n', + 'MIIEvQ==', + ], + [ + 'client_email', + 'svc@my-proj.iam.gserviceaccount.com', + 'svc@my-proj.iam.gserviceaccount.com', + ], + [ + 'private_key_id', + 'a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6', + 'a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6', + ], + ['client_id', '123456789012345678901', '123456789012345678901'], + [ + 'client_x509_cert_url', + 'https://www.googleapis.com/robot/v1/metadata/x509/svc%40my-proj.iam.gserviceaccount.com', + 'svc%40my-proj.iam.gserviceaccount.com', + ], + ])('masks the %s SA-JSON field value', (_field, value, secret) => { + const json = `{"${_field}":"${value}"}`; + const result = scrubSecrets(`init failed ${json}`); + // The secret value is gone and masking is visible. (Field-name legibility + // is handled by the named-field replacement; a name that itself trips the + // token heuristic, like client_x509_cert_url, is harmlessly masked too.) + expect(result).not.toContain(secret); + expect(result).toContain('***'); + }); + }); +}); + +describe('redactLine (truncating wire variant)', () => { + it('still truncates to <= 256 chars for the heartbeat wire contract', () => { + const long = 'Z'.repeat(300); + const result = redactLine(long); + expect(result.length).toBe(256); + expect(result.endsWith('…')).toBe(true); + }); + + it('scrubs secrets as well', () => { + const token = 'AKIAIOSFODNN7EXAMPLEKEY1234ABCD'; + expect(redactLine(`token is ${token} here`)).toBe('token is *** here'); + }); +}); diff --git a/packages/cli/src/core/cli-logger.ts b/packages/cli/src/core/cli-logger.ts index df2a06929..05aa704ce 100644 --- a/packages/cli/src/core/cli-logger.ts +++ b/packages/cli/src/core/cli-logger.ts @@ -2,30 +2,33 @@ import chalk from 'chalk'; import { createLogger, Level } from '@walkeros/core'; import type { Logger } from '@walkeros/core'; +import { scrubSecrets } from './redact-line.js'; export interface CLILoggerOptions { verbose?: boolean; silent?: boolean; json?: boolean; stderr?: boolean; + onLine?: (level: Level, message: string) => void; } /** - * Create a core Logger.Instance with CLI-appropriate behavior. + * Build the `Logger.Config` (level + handler + jsonHandler) that backs the + * CLI logger. Returned separately so the runner can hand the SAME handler + * (including the `onLine` ring tap) to the deployed bundle's collector. * - * Replaces the old CLI logger, adaptLogger, and createCollectorLoggerConfig. - * One factory, one logger type, DRY. + * The collector in a deployed bundle builds its own logger from this config + * (`config.logger`), so its destination errors flow through the identical + * `onLine` tap the runner CLI logger uses, landing in the shared ErrorRing. + * There is no parallel ring: one handler, one tap. * - * Behavior: - * - ERROR: always shown (chalk red, via console.error) unless --json - * - WARN: shown unless --silent or --json - * - INFO: shown unless --silent or --json - * - DEBUG: shown only with --verbose (and not --silent/--json) - * - json(): shown unless --silent + * Level: `Level.DEBUG` so the handler sees every line and controls visibility + * itself (verbose/silent gating). ERROR is therefore always emitted into the + * ring regardless of `--verbose`. */ -export function createCLILogger( +export function createCLILoggerConfig( options: CLILoggerOptions = {}, -): Logger.Instance { +): Logger.Config { const { verbose = false, silent = false, @@ -34,13 +37,30 @@ export function createCLILogger( } = options; const out = stderr ? console.error : console.log; - return createLogger({ - // Let handler control visibility — pass everything through + return { + // Let handler control visibility — pass everything through. With the gate + // at DEBUG, ERROR always reaches the handler (and the ring) even without + // --verbose. level: Level.DEBUG, handler: (level, message, _context, scope) => { // Build formatted message const scopePath = scope.length > 0 ? `[${scope.join(':')}] ` : ''; - const fullMessage = `${scopePath}${message}`; + // Redact secrets ONCE here, before BOTH the onLine ring tap and the + // console.* output. stderr is shipped directly by Cockpit/Loki, so the + // heartbeat-egress redactor alone (runtime/redact.ts) would miss it; doing + // it in the handler scrubs every line routed through this logger + // (collector + steps, via the D1 wiring) on both paths. Length is + // preserved here (no truncation); the heartbeat path applies the 256-char + // wire cap separately as a backstop on already-redacted text. + const fullMessage = scrubSecrets(`${scopePath}${message}`); + + // Tap every line before any early return so no level is dropped from capture. + try { + options.onLine?.(level, fullMessage); + } catch { + // Swallow: a throwing consumer must never break logging (and we must + // not call logger methods here to avoid infinite recursion). + } // ERROR: always shown unless json mode if (level === Level.ERROR) { @@ -64,5 +84,24 @@ export function createCLILogger( jsonHandler: (data) => { if (!silent) out(JSON.stringify(data, null, 2)); }, - }); + }; +} + +/** + * Create a core Logger.Instance with CLI-appropriate behavior. + * + * Replaces the old CLI logger, adaptLogger, and createCollectorLoggerConfig. + * One factory, one logger type, DRY. + * + * Behavior: + * - ERROR: always shown (chalk red, via console.error) unless --json + * - WARN: shown unless --silent or --json + * - INFO: shown unless --silent or --json + * - DEBUG: shown only with --verbose (and not --silent/--json) + * - json(): shown unless --silent + */ +export function createCLILogger( + options: CLILoggerOptions = {}, +): Logger.Instance { + return createLogger(createCLILoggerConfig(options)); } diff --git a/packages/cli/src/core/redact-line.ts b/packages/cli/src/core/redact-line.ts new file mode 100644 index 000000000..545631cea --- /dev/null +++ b/packages/cli/src/core/redact-line.ts @@ -0,0 +1,293 @@ +/** + * Shared secret redactor. Neutral module imported by BOTH the CLI logger + * handler (`core/cli-logger.ts`, scrubbing console + ring before egress) and + * the heartbeat path (`runtime/redact.ts`, scrubbing the ring snapshot before + * the POST). One set of patterns, no duplication. + * + * Two entry points: + * - `scrubSecrets(line)`: mask secrets, keep the message length intact (for + * console/stderr legibility). + * - `redactLine(line)`: `scrubSecrets` plus truncation to the 256-char + * heartbeat wire contract. + * + * All regex are linear (single unambiguous quantifiers, no nested quantifiers), + * so there is no catastrophic backtracking even on a BEGIN-marker-with-no-END + * input or a long no-space line. + */ + +const MAX_LENGTH = 256; + +// Minimum length for a standalone run to be considered a candidate token. +const MIN_TOKEN_LEN = 20; + +// Shannon entropy threshold (bits/char). Random tokens score high (~4.5-6), +// natural-language words and identifiers score low (~2-3.5). +const ENTROPY_THRESHOLD = 4.0; + +// Known secret prefixes that force-mask their run regardless of entropy/shape. +const FORCE_MASK_PREFIXES = [ + 'sk-', + 'sk_', + 'pk_', + 'ghp_', + 'gho_', + 'xoxb-', + 'xoxp-', + 'AKIA', +]; + +// ── Regex constants (all linear — no nested quantifiers) ───────────────────── + +// URL credentials: scheme://user:password@host — keep user, mask password. +const RE_URL_CREDS = /(:\/\/[^/:@\s]+:)[^@\s]+(@)/g; + +// JSON service-account field values — catches service-account blobs. +// `private_key` holds the PEM body; `client_email`, `private_key_id`, +// `client_id`, and `client_x509_cert_url` identify and pair with the leaked +// credential and must not ship either (the cert url embeds the URL-encoded +// client email). Naming them explicitly beats relying on the token-run +// heuristics, which can miss an email or a URL. The value may contain escaped +// newlines (\n as literal backslash-n) which do not split the line. [^"] is +// safe: these values contain no embedded quotes. One alternation over the field +// names keeps the scan single-pass and linear. +const RE_JSON_SA_FIELD = + /("(?:private_key|client_email|private_key_id|client_id|client_x509_cert_url)"\s*:\s*)"[^"]*"/g; + +// PEM block boundaries (structural removal, not regex masking). +// Case-insensitive, tolerate leading whitespace so a lowercase or indented +// `-----begin private key-----` block does not leak its body. +const RE_PEM_BEGIN = /^\s*-----BEGIN [A-Z ]*PRIVATE KEY-----/i; +const RE_PEM_END = /-----END[A-Z -]*-----/i; + +// KEY=value / KEY: value secret pattern. +// ReDoS-safe: the key is anchored to start-of-line or whitespace and bounded to +// 64 chars ({0,63} after the first char), so there is no O(n²) backtracking on +// long no-space lines. The value class includes - and . so hyphenated/dotted +// tokens are captured whole. Two capture groups are preserved in the +// replacement: the leading (^|\s) boundary and the key=. +const TOKEN_VALUE_CLASS = '[A-Za-z0-9+/._-]'; +const RE_KV_SECRET = new RegExp( + `(^|\\s)([A-Za-z_][A-Za-z0-9_.]{0,63}\\s*[=:]\\s*)(${TOKEN_VALUE_CLASS}{12,}={0,2})`, + 'g', +); + +// Standalone candidate runs: any run of token chars (incl. - and .) ≥ 20. +// The decision to mask happens in the callback (shouldMaskToken), keeping the +// regex itself trivial and linear. +const RE_TOKEN_RUN = new RegExp(`${TOKEN_VALUE_CLASS}{${MIN_TOKEN_LEN},}`, 'g'); + +// Force-mask prefix detector for runs that carry a known secret prefix. +// Matches a known prefix followed by token chars (any length), so e.g. +// `ghp_xxxxxxxx` or `xoxb-2384-2384-AbCdEf` is masked even under 20 chars. +// Built from FORCE_MASK_PREFIXES so the list is the single source of truth. +// Prefixes are regex-escaped (only `-`/`_` appear, both literal here, but escape +// defensively) and ordered longest-first so the alternation is unambiguous. +function escapeRegex(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} +const PREFIX_ALTERNATION = [...FORCE_MASK_PREFIXES] + .sort((a, b) => b.length - a.length) + .map(escapeRegex) + .join('|'); +const RE_PREFIXED_TOKEN = new RegExp( + `(?:${PREFIX_ALTERNATION})${TOKEN_VALUE_CLASS}*`, + 'g', +); + +// ── Entropy ────────────────────────────────────────────────────────────────── + +/** Shannon entropy in bits per character. Linear in input length. */ +function shannonEntropy(s: string): number { + if (s.length === 0) return 0; + const counts = new Map(); + for (const ch of s) counts.set(ch, (counts.get(ch) ?? 0) + 1); + let entropy = 0; + for (const count of counts.values()) { + const p = count / s.length; + entropy -= p * Math.log2(p); + } + return entropy; +} + +// ── Token masking decision ───────────────────────────────────────────────── + +const RE_ALL_HEX = /^[0-9a-fA-F]+$/; +const RE_ALL_DIGIT = /^[0-9]+$/; +// Real base64 padding/special chars. `/` is intentionally EXCLUDED: it is a +// structural path/URL separator (FQNs, gs:// paths, request paths, googleapis +// URLs), not a secret signal. Since redaction now runs on every console line, +// treating `/` as a secret char nuked legitimate BigQuery/GCS diagnostics. Real +// base64 secrets still carry `+`/`=` padding, are all-hex, or are high-entropy. +const RE_BASE64_SPECIAL = /[+=]/; +const RE_HAS_DIGIT = /[0-9]/; +const RE_HAS_LETTER = /[A-Za-z]/; + +/** + * Decide whether a candidate run (a single token, no `/`) is a secret to mask. + * + * Mask when ANY of: + * - all-hex (digests, SHA, etc.) + * - all-digit (numeric tokens/ids) + * - contains a base64 special char (+ =) + * - contains both a digit and a letter (mixed token) + * - Shannon entropy ≥ 4.0 bits/char (random all-letter / mixed-case tokens) + */ +function isSecretSegment(run: string): boolean { + if (RE_ALL_HEX.test(run)) return true; + if (RE_ALL_DIGIT.test(run)) return true; + if (RE_BASE64_SPECIAL.test(run)) return true; + if (RE_HAS_DIGIT.test(run) && RE_HAS_LETTER.test(run)) return true; + if (shannonEntropy(run) >= ENTROPY_THRESHOLD) return true; + return false; +} + +/** + * Decide whether a ≥20-char candidate run is a secret to mask. + * + * A run containing `/` is treated as a path/URL structure, not one opaque token. + * Paths and URLs split into short alphanumeric segments (`v2`, `projects`, + * `12345`), each of which trips the digit+letter or entropy heuristic on its own + * even though the whole thing is legible diagnostics. So a `/`-bearing run is + * masked only when one of its long (≥20-char) segments independently looks like + * a secret (e.g. a base64 blob embedded in a URL path). A base64 secret with `/` + * still carries `+`/`=` (handled per-segment) or is hex/high-entropy, so real + * secrets remain masked while FQNs, gs:// paths, request paths, and googleapis + * URLs survive. + * + * ACCEPTED best-effort limitation: a bare all-letters run with no digit, no + * special char, no known prefix, and entropy < 4.0 (e.g. `supercalifragilistic` + * or a long camelCase identifier like `getUserAccountSettings`) may pass through + * unmasked. Masking every such run would destroy log readability by redacting + * ordinary identifiers and words, which defeats the purpose of legible + * diagnostics. This is a deliberate trade-off, not an oversight. + */ +function shouldMaskToken(run: string): boolean { + if (run.includes('/')) { + // Per-segment: only mask if a long segment is itself secret-shaped. Short + // path segments (the bulk of any URL) never qualify, keeping URLs legible. + return run + .split('/') + .filter((segment) => segment.length >= MIN_TOKEN_LEN) + .some(isSecretSegment); + } + return isSecretSegment(run); +} + +// ── PEM block removal ──────────────────────────────────────────────────────── + +/** + * Remove all PEM private-key blocks from an array of lines. + * A block starts at a BEGIN PRIVATE KEY marker and ends at the matching END + * marker (inclusive). If no END is found, everything to end of entry is dropped. + */ +function removePemBlocks(lines: string[]): string[] { + const out: string[] = []; + let inBlock = false; + + for (const line of lines) { + if (!inBlock) { + if (RE_PEM_BEGIN.test(line)) { + inBlock = true; + continue; // start-of-block line dropped + } + out.push(line); + } else { + if (RE_PEM_END.test(line)) { + inBlock = false; + } + // drop every in-block line, including the END marker + } + } + + return out; +} + +// ── Per-line secret masking ────────────────────────────────────────────────── + +/** + * Mask secrets in a single (non-PEM) line. + * All patterns are applied in order; all regex are linear (no nested quantifiers). + */ +function maskLine(line: string): string { + let s = line; + + // 1. URL credentials: ://user:secret@host → ://user:***@host + s = s.replace(RE_URL_CREDS, '$1***$2'); + + // 2. KEY=secret / KEY: secret (anchored, bounded key — ReDoS-safe). + // Mask the value only when it is secret-shaped. This stops `scheme://host/...` + // (matched as `gs:` / `https:` key + `//...` value) from masking legitimate + // URLs/paths, while real `TOKEN=secret` values still mask. The `/`-aware + // shouldMaskToken keeps path-shaped values legible. + s = s.replace( + RE_KV_SECRET, + (match: string, boundary: string, keyPart: string, value: string) => + shouldMaskToken(value) ? `${boundary}${keyPart}***` : match, + ); + + // 3. Known-prefix tokens (force-mask regardless of length/entropy) + s = s.replace(RE_PREFIXED_TOKEN, '***'); + + // 4. Standalone candidate runs ≥ 20 chars — mask by shape/entropy + s = s.replace(RE_TOKEN_RUN, (match) => + shouldMaskToken(match) ? '***' : match, + ); + + return s; +} + +// ── Public API ─────────────────────────────────────────────────────────────── + +/** + * Scrub credential material from a log line (or multi-line entry) WITHOUT + * truncating. This is the shared secret redactor used by the CLI logger handler + * (covers console/stderr and the ring before egress). + * + * Algorithm: + * 1. Mask JSON service-account fields (private_key, client_email, + * private_key_id, client_id, client_x509_cert_url) BEFORE splitting + * (service-account blobs embed PEM blocks as \\n-encoded strings; masking + * first prevents the BEGIN marker from appearing on its own line and dropping + * surrounding fields). + * 2. Split on \\n. + * 3. Remove PEM private-key blocks structurally (BEGIN…END inclusive, + * case-insensitive; a no-END block drops to end of entry). + * 4. For each surviving line: mask URL creds, KEY=secret, prefixed tokens, + * high-entropy/shape-based token runs. + * 5. Rejoin with \\n. + */ +export function scrubSecrets(line: string): string { + // Step 1: JSON service-account field masking before any line splitting + const withoutJsonKey = line.replace(RE_JSON_SA_FIELD, '$1"***"'); + + // Step 2: split + const rawLines = withoutJsonKey.split('\n'); + + // Step 3: remove PEM blocks (any remaining standalone markers after step 1) + const cleanLines = removePemBlocks(rawLines); + + // Step 4: mask each surviving line + const maskedLines = cleanLines.map(maskLine); + + // Step 5: rejoin + return maskedLines.join('\n'); +} + +/** + * Redact a log line and truncate to the heartbeat wire contract (256 chars). + * This is the heartbeat-egress variant: scrub secrets, then truncate AFTER + * masking so a token straddling char 256 is masked, not left partially visible. + * + * The app's heartbeat schema enforces message .max(256); a 257-char string + * would fail zod .parse and drop the whole heartbeat. Reserve one char for the + * ellipsis so the TOTAL length is at most MAX_LENGTH (256). + */ +export function redactLine(line: string): string { + const scrubbed = scrubSecrets(line); + + if (scrubbed.length > MAX_LENGTH) { + return scrubbed.slice(0, MAX_LENGTH - 1) + '…'; + } + + return scrubbed; +} diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index e196f6c5a..69b3f05a3 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -59,6 +59,11 @@ export { // === Programmatic API === // High-level functions for library usage export { bundle } from './commands/bundle/index.js'; +export { buildDataPayload } from './commands/bundle/bundler.js'; +export { + classifyStepProperties, + containsCodeMarkers, +} from './commands/bundle/config-classifier.js'; export { validateFlowStructure } from './commands/bundle/validate-structure.js'; export { wrapSkeleton } from './commands/bundle/wrap.js'; export type { WrapSkeletonOptions } from './commands/bundle/wrap.js'; @@ -70,6 +75,7 @@ export { simulateCollector, } from './commands/push/index.js'; export type { + SimulateDataOptions, SimulateSourceOptions, SimulateTransformerOptions, SimulateDestinationOptions, diff --git a/packages/cli/src/runtime/__tests__/config-fetcher.test.ts b/packages/cli/src/runtime/__tests__/config-fetcher.test.ts index 87ef6bcfe..730978011 100644 --- a/packages/cli/src/runtime/__tests__/config-fetcher.test.ts +++ b/packages/cli/src/runtime/__tests__/config-fetcher.test.ts @@ -28,6 +28,70 @@ describe('fetchConfig', () => { globalThis.fetch = originalFetch; }); + /** + * Drive a fetchConfig call to completion while flushing the retry helper's + * backoff sleeps so a transient-then-success sequence settles without real + * waits. Mirrors the fetch-retry test's fake-timer drain. + */ + async function runWithTimers(promise: Promise): Promise { + const settled = promise.then( + (value) => ({ ok: true as const, value }), + (error: unknown) => ({ ok: false as const, error }), + ); + await jest.runAllTimersAsync(); + const result = await settled; + if (result.ok) return result.value; + throw result.error; + } + + it('retries a transient 503 then succeeds', async () => { + jest.useFakeTimers(); + try { + globalThis.fetch = jest + .fn() + .mockResolvedValueOnce(makeResponse({ status: 503 })) + .mockResolvedValueOnce( + makeResponse({ + status: 200, + body: { config: { version: 3 } }, + }), + ); + + const result = await runWithTimers( + fetchConfig({ + appUrl: 'http://localhost:3000', + token: 'sk-walkeros-test', + projectId: 'proj_1', + flowId: 'cfg_1', + }), + ); + + expect(result.changed).toBe(true); + expect(globalThis.fetch).toHaveBeenCalledTimes(2); + } finally { + jest.useRealTimers(); + } + }); + + it('does NOT retry a 401 (single call) and throws RunnerAuthError', async () => { + const fetchMock = jest + .fn() + .mockResolvedValue( + makeResponse({ status: 401, statusText: 'Unauthorized' }), + ); + globalThis.fetch = fetchMock; + + await expect( + fetchConfig({ + appUrl: 'http://localhost:3000', + token: 'sk-walkeros-test', + projectId: 'proj_1', + flowId: 'cfg_1', + }), + ).rejects.toBeInstanceOf(RunnerAuthError); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + it('throws RunnerAuthError(unauthorised) on 401 response', async () => { globalThis.fetch = jest .fn() @@ -69,21 +133,30 @@ describe('fetchConfig', () => { } }); - it('throws generic error on other HTTP failures', async () => { - globalThis.fetch = jest.fn().mockResolvedValue({ - ok: false, - status: 500, - statusText: 'Internal Server Error', - }); + it('retries a persistent 500 to exhaustion then throws (3 calls)', async () => { + jest.useFakeTimers(); + try { + const fetchMock = jest.fn().mockResolvedValue({ + ok: false, + status: 500, + statusText: 'Internal Server Error', + }); + globalThis.fetch = fetchMock; - await expect( - fetchConfig({ - appUrl: 'http://localhost:3000', - token: 'sk-walkeros-test', - projectId: 'proj_1', - flowId: 'cfg_1', - }), - ).rejects.toThrow(/500/); + await expect( + runWithTimers( + fetchConfig({ + appUrl: 'http://localhost:3000', + token: 'sk-walkeros-test', + projectId: 'proj_1', + flowId: 'cfg_1', + }), + ), + ).rejects.toThrow(/500/); + expect(fetchMock).toHaveBeenCalledTimes(3); + } finally { + jest.useRealTimers(); + } }); it('returns unchanged on 304', async () => { diff --git a/packages/cli/src/runtime/__tests__/fetch-retry.test.ts b/packages/cli/src/runtime/__tests__/fetch-retry.test.ts new file mode 100644 index 000000000..9db777b89 --- /dev/null +++ b/packages/cli/src/runtime/__tests__/fetch-retry.test.ts @@ -0,0 +1,274 @@ +import { fetchWithRetry } from '../fetch-retry.js'; + +/** + * Build the kind of error AbortSignal.timeout() produces when an attempt + * exceeds its per-attempt budget. Node throws a DOMException named + * 'TimeoutError'; modelling it directly keeps the test free of casts. + */ +function timeoutError(): DOMException { + return new DOMException('The operation timed out', 'TimeoutError'); +} + +/** + * Build a network-layer error carrying a libuv-style `code`, matching how + * undici surfaces connection failures (e.g. a rejected fetch with + * cause.code === 'ECONNRESET'). + */ +function networkError(code: string): Error & { code: string } { + return Object.assign(new Error(`connect ${code}`), { code }); +} + +describe('fetchWithRetry', () => { + const originalFetch = globalThis.fetch; + let mockFetch: jest.Mock; + + beforeEach(() => { + jest.useFakeTimers(); + mockFetch = jest.fn(); + globalThis.fetch = mockFetch; + }); + + afterEach(() => { + jest.useRealTimers(); + globalThis.fetch = originalFetch; + }); + + /** + * Run fetchWithRetry to completion while draining every backoff timer the + * retry loop schedules. runAllTimersAsync flushes the setTimeout-based sleeps + * so the promise settles without real waits. + */ + async function runWithTimers(promise: Promise): Promise { + // Keep a settled handle so the rejection is observed (no unhandled + // rejection) while runAllTimersAsync drains the backoff sleeps. + const settled = promise.then( + (value) => ({ ok: true as const, value }), + (error: unknown) => ({ ok: false as const, error }), + ); + await jest.runAllTimersAsync(); + const result = await settled; + if (result.ok) return result.value; + throw result.error; + } + + it('retries a timeout then succeeds (2 calls)', async () => { + mockFetch + .mockRejectedValueOnce(timeoutError()) + .mockResolvedValueOnce(new Response('ok', { status: 200 })); + + const response = await runWithTimers( + fetchWithRetry('https://example.com/bundle'), + ); + + expect(response.status).toBe(200); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it('retries a 503 then succeeds (2 calls)', async () => { + mockFetch + .mockResolvedValueOnce(new Response('busy', { status: 503 })) + .mockResolvedValueOnce(new Response('ok', { status: 200 })); + + const response = await runWithTimers( + fetchWithRetry('https://example.com/bundle'), + ); + + expect(response.status).toBe(200); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it('retries a 429 then succeeds (2 calls)', async () => { + mockFetch + .mockResolvedValueOnce(new Response('slow down', { status: 429 })) + .mockResolvedValueOnce(new Response('ok', { status: 200 })); + + const response = await runWithTimers( + fetchWithRetry('https://example.com/bundle'), + ); + + expect(response.status).toBe(200); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it('retries a network error then succeeds (2 calls)', async () => { + mockFetch + .mockRejectedValueOnce(networkError('ECONNRESET')) + .mockResolvedValueOnce(new Response('ok', { status: 200 })); + + const response = await runWithTimers( + fetchWithRetry('https://example.com/bundle'), + ); + + expect(response.status).toBe(200); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it('does NOT retry a 404 (1 call, returns the 404 Response)', async () => { + mockFetch.mockResolvedValueOnce(new Response('missing', { status: 404 })); + + const response = await runWithTimers( + fetchWithRetry('https://example.com/bundle'), + ); + + expect(response.status).toBe(404); + expect(mockFetch).toHaveBeenCalledTimes(1); + }); + + it('does NOT retry a 200 (1 call, returns the Response)', async () => { + mockFetch.mockResolvedValueOnce(new Response('ok', { status: 200 })); + + const response = await runWithTimers( + fetchWithRetry('https://example.com/bundle'), + ); + + expect(response.status).toBe(200); + expect(mockFetch).toHaveBeenCalledTimes(1); + }); + + it('throws after exhausting attempts on persistent timeout (3 calls)', async () => { + mockFetch + .mockRejectedValueOnce(timeoutError()) + .mockRejectedValueOnce(timeoutError()) + .mockRejectedValueOnce(timeoutError()); + + await expect( + runWithTimers(fetchWithRetry('https://example.com/bundle')), + ).rejects.toThrow(/after 3 attempts/); + + expect(mockFetch).toHaveBeenCalledTimes(3); + }); + + it('names the last cause in the exhaustion error message', async () => { + mockFetch + .mockResolvedValueOnce(new Response('busy', { status: 503 })) + .mockResolvedValueOnce(new Response('busy', { status: 503 })) + .mockResolvedValueOnce(new Response('busy', { status: 503 })); + + await expect( + runWithTimers(fetchWithRetry('https://example.com/bundle')), + ).rejects.toThrow(/503/); + }); + + it('honors a custom attempts count', async () => { + mockFetch + .mockRejectedValueOnce(timeoutError()) + .mockResolvedValueOnce(new Response('ok', { status: 200 })); + + const response = await runWithTimers( + fetchWithRetry('https://example.com/bundle', { attempts: 2 }), + ); + + expect(response.status).toBe(200); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it('passes init and a per-attempt timeout signal to fetch', async () => { + mockFetch.mockResolvedValueOnce(new Response('ok', { status: 200 })); + + await runWithTimers( + fetchWithRetry('https://example.com/bundle', { + init: { headers: { 'x-test': '1' } }, + }), + ); + + expect(mockFetch).toHaveBeenCalledWith( + 'https://example.com/bundle', + expect.objectContaining({ + headers: { 'x-test': '1' }, + signal: expect.any(AbortSignal), + }), + ); + }); + + it('stops retrying once the total budget is exceeded', async () => { + // maxTotalMs allows exactly one attempt: attempt 1 fails, the loop sleeps + // the (clamped) backoff which consumes the rest of the budget, then the + // next iteration sees too little left to start a second attempt. + mockFetch.mockRejectedValue(timeoutError()); + + await expect( + runWithTimers( + fetchWithRetry('https://example.com/bundle', { + attempts: 5, + maxTotalMs: 2_000, + }), + ), + ).rejects.toThrow(/after 1 attempt/); + + expect(mockFetch).toHaveBeenCalledTimes(1); + }); + + it('does not start an attempt when budget is below the floor', async () => { + // Budget under MIN_ATTEMPT_BUDGET_MS: a clamped sub-second attempt could not + // connect, so the loop must not call fetch at all. + mockFetch.mockRejectedValue(timeoutError()); + + await expect( + runWithTimers( + fetchWithRetry('https://example.com/bundle', { + attempts: 5, + maxTotalMs: 500, + }), + ), + ).rejects.toThrow(/after 0 attempts/); + + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('clamps each attempt timeout to the remaining budget', async () => { + // perAttemptTimeoutMs (30s) far exceeds maxTotalMs (5s): the first attempt's + // timeout must be clamped to the remaining 5s so a single attempt cannot + // overrun the total budget. + // The per-attempt timeout is scheduled with + // setTimeout(() => controller.abort(), attemptTimeoutMs) and cleared once + // fetch resolves, so assert on the scheduled delay. + const setTimeoutSpy = jest.spyOn(globalThis, 'setTimeout'); + try { + mockFetch.mockResolvedValueOnce(new Response('ok', { status: 200 })); + + await runWithTimers( + fetchWithRetry('https://example.com/bundle', { + perAttemptTimeoutMs: 30_000, + maxTotalMs: 5_000, + }), + ); + + // Success on the first try schedules only the attempt-timeout timer (no + // backoff sleep), clamped to the 5s budget. + const delays = setTimeoutSpy.mock.calls.map((call) => call[1]); + expect(delays).toEqual([5_000]); + } finally { + setTimeoutSpy.mockRestore(); + } + }); + + it('clamps later attempts as the budget shrinks', async () => { + // After a failed attempt and ~2000ms backoff, the next attempt's timeout + // must be clamped to the budget remaining (~3000ms of the 5000ms cap), not + // the full perAttemptTimeoutMs. + const setTimeoutSpy = jest.spyOn(globalThis, 'setTimeout'); + try { + mockFetch + .mockRejectedValueOnce(timeoutError()) + .mockResolvedValueOnce(new Response('ok', { status: 200 })); + + await runWithTimers( + fetchWithRetry('https://example.com/bundle', { + perAttemptTimeoutMs: 30_000, + maxTotalMs: 5_000, + }), + ); + + // Scheduled timers in order: attempt-1 timeout, the backoff sleep, then + // attempt-2 timeout. Attempt 1 is clamped to the full 5000ms budget; + // attempt 2 is clamped to what remains after the ~2000ms (±20%) backoff. + const delays = setTimeoutSpy.mock.calls.map((call) => call[1]); + expect(delays).toHaveLength(3); + expect(delays[0]).toBe(5_000); + expect(delays[2]).toBeLessThan(5_000); + expect(delays[2]).toBeGreaterThan(0); + } finally { + setTimeoutSpy.mockRestore(); + } + }); +}); diff --git a/packages/cli/src/runtime/__tests__/heartbeat.test.ts b/packages/cli/src/runtime/__tests__/heartbeat.test.ts index e8f5420b7..fbbf52996 100644 --- a/packages/cli/src/runtime/__tests__/heartbeat.test.ts +++ b/packages/cli/src/runtime/__tests__/heartbeat.test.ts @@ -3,18 +3,103 @@ import { computeCounterDelta, type CounterSnapshot, } from '../heartbeat.js'; +import type { Collector, Logger } from '@walkeros/core'; +import type { DedupedError, RingEntry } from '../log-ring.js'; -const mockLogger = { - info: jest.fn(), - error: jest.fn(), - warn: jest.fn(), - debug: jest.fn(), - scope: jest.fn().mockReturnThis(), -}; +/** + * A `Logger.Instance` whose methods are jest spies, so tests can both pass it to + * `createHeartbeat` (typed) and assert on `.error`/`.warn` calls. `scope` + * returns the same instance so scoped logging resolves back to these spies. + */ +function makeSpyLogger(): Logger.Instance { + const logger: Logger.Instance = { + error: jest.fn(), + warn: jest.fn(), + info: jest.fn(), + debug: jest.fn(), + throw: (message: string | Error): never => { + throw new Error(typeof message === 'string' ? message : message.message); + }, + json: jest.fn(), + scope: (_name: string): Logger.Instance => logger, + }; + return logger; +} + +const mockLogger = makeSpyLogger(); + +/** A heartbeat record entry as serialized into the POST body. */ +interface SerializedRecord { + message: string; + count: number; + firstSeen: string; + lastSeen: string; +} + +/** A heartbeat log line as serialized into the POST body. */ +interface SerializedLog { + time: string; + level: string; + message: string; +} + +interface SerializedDestination { + count: number; + failed: number; + duration: number; + dlqSize: number; + dropped: number; +} + +/** The shape of the JSON body the heartbeat POSTs (fields under test). */ +interface HeartbeatBody { + instanceId?: string; + recentErrors?: SerializedRecord[]; + recentLogs?: SerializedLog[]; + counters?: { + eventsIn: number; + eventsOut: number; + eventsFailed: number; + destinations: Record; + }; +} + +/** + * Read and parse the JSON body of the first fetch call from a typed fetch mock, + * without casts. The mock's `.mock.calls` are typed via `jest.fn`, + * so the init arg is `RequestInit | undefined` and its `body` is narrowed to a + * string before parsing. + */ +function readHeartbeatBody( + mock: jest.Mock, Parameters>, +): HeartbeatBody { + const init = mock.mock.calls[0]?.[1]; + const body = init?.body; + if (typeof body !== 'string') { + throw new Error('expected a string request body'); + } + const parsed: HeartbeatBody = JSON.parse(body); + return parsed; +} + +function createFetchMock(): jest.Mock< + ReturnType, + Parameters +> { + return jest.fn, Parameters>(() => + Promise.resolve(new Response(null, { status: 200 })), + ); +} describe('heartbeat', () => { const originalFetch = globalThis.fetch; + beforeEach(() => { + // mockLogger is shared across tests; clear its spies' call history so + // assertions don't see calls from a prior test. + jest.clearAllMocks(); + }); + afterEach(() => { globalThis.fetch = originalFetch; jest.restoreAllMocks(); @@ -30,7 +115,7 @@ describe('heartbeat', () => { projectId: 'proj_1', intervalMs: 60000, }, - mockLogger as any, + mockLogger, ); await heartbeat.sendOnce(); @@ -53,7 +138,7 @@ describe('heartbeat', () => { projectId: 'proj_1', intervalMs: 60000, }, - mockLogger as any, + mockLogger, ); await heartbeat.sendOnce(); @@ -73,7 +158,7 @@ describe('heartbeat', () => { projectId: 'proj_1', intervalMs: 60000, }, - mockLogger as any, + mockLogger, ); await heartbeat.sendOnce(); @@ -84,6 +169,228 @@ describe('heartbeat', () => { }); }); +describe('heartbeat snapshot suppliers', () => { + const originalFetch = globalThis.fetch; + + afterEach(() => { + globalThis.fetch = originalFetch; + jest.restoreAllMocks(); + }); + + function makeTypedLogger(): Logger.Instance { + const logger: Logger.Instance = { + error: jest.fn(), + warn: jest.fn(), + info: jest.fn(), + debug: jest.fn(), + throw: (message: string | Error): never => { + throw new Error( + typeof message === 'string' ? message : message.message, + ); + }, + json: jest.fn(), + scope: (_name: string): Logger.Instance => logger, + }; + return logger; + } + + const typedLogger = makeTypedLogger(); + + it('includes recentErrors and recentLogs in the body when suppliers return non-empty arrays, with secrets redacted', async () => { + const secret = 'sk-1234567890abcdef1234'; + const now = Date.now(); + + const fetchMock = createFetchMock(); + globalThis.fetch = fetchMock; + + const errors: DedupedError[] = [ + { + message: `Connection failed TOKEN=${secret}`, + count: 3, + firstSeen: now - 5000, + lastSeen: now, + }, + ]; + + const logs: RingEntry[] = [ + { + time: now - 1000, + level: 'warn', + message: `Warning: secret=${secret} detected`, + }, + ]; + + const heartbeat = createHeartbeat( + { + appUrl: 'http://localhost:3000', + token: 'bearer-test', + projectId: 'proj_1', + intervalMs: 60000, + getErrors: () => errors, + getLogs: () => logs, + }, + typedLogger, + ); + + await heartbeat.sendOnce(); + + expect(fetchMock.mock.calls).toHaveLength(1); + const body = readHeartbeatBody(fetchMock); + + // Fields are present + expect(body.recentErrors).toBeDefined(); + expect(body.recentLogs).toBeDefined(); + + // Secret is redacted in both + const bodyStr = JSON.stringify(body); + expect(bodyStr).not.toContain(secret); + + // recentErrors structure + const recentErrors = body.recentErrors ?? []; + expect(recentErrors).toHaveLength(1); + expect(recentErrors[0]?.count).toBe(3); + expect(typeof recentErrors[0]?.firstSeen).toBe('string'); // ISO string + expect(typeof recentErrors[0]?.lastSeen).toBe('string'); + + // recentLogs structure + const recentLogs = body.recentLogs ?? []; + expect(recentLogs).toHaveLength(1); + expect(recentLogs[0]?.level).toBe('warn'); + expect(typeof recentLogs[0]?.time).toBe('string'); // ISO string + }); + + it('always sends recentErrors (empty array) so the snapshot can clear, but omits empty recentLogs', async () => { + const fetchMock = createFetchMock(); + globalThis.fetch = fetchMock; + + const heartbeat = createHeartbeat( + { + appUrl: 'http://localhost:3000', + token: 'bearer-test', + projectId: 'proj_1', + intervalMs: 60000, + getErrors: () => [], + getLogs: () => [], + }, + typedLogger, + ); + + await heartbeat.sendOnce(); + + const body = readHeartbeatBody(fetchMock); + + // recentErrors is ALWAYS sent so an empty array clears a stale snapshot. + expect(body.recentErrors).toEqual([]); + // recentLogs stays omit-when-empty (no clear semantics needed). + expect(body.recentLogs).toBeUndefined(); + }); + + it('still sends recentErrors as [] when the supplier is absent, omitting recentLogs', async () => { + const fetchMock = createFetchMock(); + globalThis.fetch = fetchMock; + + const heartbeat = createHeartbeat( + { + appUrl: 'http://localhost:3000', + token: 'bearer-test', + projectId: 'proj_1', + intervalMs: 60000, + // no getErrors, no getLogs + }, + typedLogger, + ); + + await heartbeat.sendOnce(); + + const body = readHeartbeatBody(fetchMock); + + expect(body.recentErrors).toEqual([]); + expect(body.recentLogs).toBeUndefined(); + }); +}); + +describe('heartbeat per-destination breakdown (dlqSize + dropped)', () => { + const originalFetch = globalThis.fetch; + + afterEach(() => { + globalThis.fetch = originalFetch; + jest.restoreAllMocks(); + }); + + function typedLogger(): Logger.Instance { + const logger: Logger.Instance = { + error: jest.fn(), + warn: jest.fn(), + info: jest.fn(), + debug: jest.fn(), + throw: (message: string | Error): never => { + throw new Error( + typeof message === 'string' ? message : message.message, + ); + }, + json: jest.fn(), + scope: (_name: string): Logger.Instance => logger, + }; + return logger; + } + + function statusWith( + destinations: Collector.Status['destinations'], + dropped: Collector.Status['dropped'], + ): Collector.Status { + return { + startedAt: 0, + in: 0, + out: 0, + failed: 0, + sources: {}, + destinations, + dropped, + }; + } + + it('includes per-destination dlqSize (gauge) and dropped (delta) in counters', async () => { + const fetchMock = createFetchMock(); + globalThis.fetch = fetchMock; + + const status = statusWith( + { + bigquery: { + count: 10, + failed: 4, + duration: 200, + queuePushSize: 0, + dlqSize: 3, + }, + }, + { 'destination.bigquery': { dlq: 2, queue: 1 } }, + ); + + const heartbeat = createHeartbeat( + { + appUrl: 'http://localhost:3000', + token: 'bearer-test', + projectId: 'proj_1', + intervalMs: 60000, + getCounters: () => status, + }, + typedLogger(), + ); + + await heartbeat.sendOnce(); + + const body = readHeartbeatBody(fetchMock); + + const dest = body.counters?.destinations['bigquery']; + expect(dest).toBeDefined(); + expect(dest?.failed).toBe(4); + // dlqSize is a point-in-time gauge, not a delta. + expect(dest?.dlqSize).toBe(3); + // dropped sums queue + dlq drops for the destination step (delta from 0). + expect(dest?.dropped).toBe(3); + }); +}); + describe('computeCounterDelta', () => { it('computes correct delta for top-level counters', () => { const current: CounterSnapshot = { @@ -110,7 +417,7 @@ describe('computeCounterDelta', () => { out: 5, failed: 0, destinations: { - demo: { count: 5, failed: 0, duration: 100 }, + demo: { count: 5, failed: 0, duration: 100, dlqSize: 0, dropped: 0 }, }, }; const last: CounterSnapshot = { @@ -118,7 +425,7 @@ describe('computeCounterDelta', () => { out: 2, failed: 0, destinations: { - demo: { count: 2, failed: 0, duration: 40 }, + demo: { count: 2, failed: 0, duration: 40, dlqSize: 0, dropped: 0 }, }, }; const delta = computeCounterDelta(current, last); @@ -131,7 +438,9 @@ describe('computeCounterDelta', () => { in: 1, out: 1, failed: 0, - destinations: { newDest: { count: 1, failed: 0, duration: 10 } }, + destinations: { + newDest: { count: 1, failed: 0, duration: 10, dlqSize: 0, dropped: 0 }, + }, }; const last: CounterSnapshot = { in: 0, @@ -142,4 +451,28 @@ describe('computeCounterDelta', () => { const delta = computeCounterDelta(current, last); expect(delta.destinations.newDest.count).toBe(1); }); + + it('deltas dropped but reports dlqSize as the current gauge', () => { + const current: CounterSnapshot = { + in: 10, + out: 8, + failed: 2, + destinations: { + bq: { count: 10, failed: 2, duration: 50, dlqSize: 5, dropped: 7 }, + }, + }; + const last: CounterSnapshot = { + in: 4, + out: 3, + failed: 1, + destinations: { + bq: { count: 4, failed: 1, duration: 20, dlqSize: 2, dropped: 3 }, + }, + }; + const delta = computeCounterDelta(current, last); + // dropped is monotonic → delta. + expect(delta.destinations.bq.dropped).toBe(4); + // dlqSize is a point-in-time depth → report the current value, not a delta. + expect(delta.destinations.bq.dlqSize).toBe(5); + }); }); diff --git a/packages/cli/src/runtime/__tests__/log-ring.test.ts b/packages/cli/src/runtime/__tests__/log-ring.test.ts new file mode 100644 index 000000000..cc4156da5 --- /dev/null +++ b/packages/cli/src/runtime/__tests__/log-ring.test.ts @@ -0,0 +1,100 @@ +import { LogRing, ErrorRing } from '../log-ring.js'; +import type { RingEntry } from '../log-ring.js'; + +function makeEntry(time: number, message: string): RingEntry { + return { time, level: 'info', message }; +} + +describe('LogRing', () => { + it('starts empty', () => { + const ring = new LogRing(5); + expect(ring.snapshot()).toEqual([]); + }); + + it('keeps only the last N entries', () => { + const ring = new LogRing(5); + for (let i = 0; i < 10; i++) { + ring.add(makeEntry(i, `msg-${i}`)); + } + const snap = ring.snapshot(); + expect(snap).toHaveLength(5); + // oldest dropped: only messages 5..9 remain + expect(snap[0].message).toBe('msg-5'); + expect(snap[4].message).toBe('msg-9'); + }); + + it('snapshot(limit) returns the most recent limit entries oldest-first', () => { + const ring = new LogRing(10); + for (let i = 0; i < 8; i++) { + ring.add(makeEntry(i, `msg-${i}`)); + } + const snap = ring.snapshot(3); + expect(snap).toHaveLength(3); + expect(snap[0].message).toBe('msg-5'); + expect(snap[1].message).toBe('msg-6'); + expect(snap[2].message).toBe('msg-7'); + }); +}); + +describe('ErrorRing', () => { + it('starts empty', () => { + let t = 0; + const ring = new ErrorRing(10, () => t++); + expect(ring.snapshot()).toEqual([]); + }); + + it('deduplicates: same message twice yields count 2, firstSeen unchanged, lastSeen updated', () => { + let t = 100; + const now = () => t; + const ring = new ErrorRing(10, now); + + t = 100; + ring.add('boom'); + t = 200; + ring.add('boom'); + + const snap = ring.snapshot(); + expect(snap).toHaveLength(1); + expect(snap[0].message).toBe('boom'); + expect(snap[0].count).toBe(2); + expect(snap[0].firstSeen).toBe(100); + expect(snap[0].lastSeen).toBe(200); + }); + + it('evicts the least-recently-seen entry when maxUnique is exceeded', () => { + let t = 0; + const ring = new ErrorRing(3, () => t++); + + // add 3 distinct messages at t=0,1,2 + ring.add('a'); // lastSeen=0 + ring.add('b'); // lastSeen=1 + ring.add('c'); // lastSeen=2 + + // touch 'a' again so it's no longer the LRS + ring.add('a'); // lastSeen=3 now + + // 'd' is the 4th distinct message — should evict 'b' (lastSeen=1, oldest) + ring.add('d'); // t=4 + + const snap = ring.snapshot(); + const messages = snap.map((e) => e.message); + expect(messages).not.toContain('b'); + expect(messages).toContain('a'); + expect(messages).toContain('c'); + expect(messages).toContain('d'); + }); + + it('snapshot returns entries sorted by lastSeen descending', () => { + let t = 0; + const ring = new ErrorRing(10, () => t++); + + ring.add('first'); // t=0 + ring.add('second'); // t=1 + ring.add('third'); // t=2 + + const snap = ring.snapshot(); + expect(snap[0].message).toBe('third'); + expect(snap[1].message).toBe('second'); + expect(snap[2].message).toBe('first'); + }); +}); diff --git a/packages/cli/src/runtime/__tests__/poller.test.ts b/packages/cli/src/runtime/__tests__/poller.test.ts new file mode 100644 index 000000000..c5fdd91ff --- /dev/null +++ b/packages/cli/src/runtime/__tests__/poller.test.ts @@ -0,0 +1,158 @@ +import { createPoller } from '../poller.js'; +import { fetchConfig, type ConfigFetchResult } from '../config-fetcher.js'; +import type { Logger } from '@walkeros/core'; + +jest.mock('../config-fetcher.js'); + +const mockedFetchConfig = jest.mocked(fetchConfig); + +function makeTypedLogger(): Logger.Instance { + const logger: Logger.Instance = { + error: jest.fn(), + warn: jest.fn(), + info: jest.fn(), + debug: jest.fn(), + throw: (message: string | Error): never => { + throw new Error(typeof message === 'string' ? message : message.message); + }, + json: jest.fn(), + scope: (_name: string): Logger.Instance => logger, + }; + return logger; +} + +const baseFetchOptions = { + appUrl: 'http://localhost:3000', + token: 'sk-walkeros-test', + projectId: 'proj_1', + flowId: 'cfg_1', +}; + +describe('createPoller', () => { + afterEach(() => { + jest.clearAllMocks(); + }); + + it('seeds lastEtag from initialEtag so the first poll sends If-None-Match and a 304-equivalent unchanged result skips onUpdate', async () => { + const logger = makeTypedLogger(); + const onUpdate = jest.fn(async () => {}); + const unchanged: ConfigFetchResult = { changed: false }; + mockedFetchConfig.mockResolvedValue(unchanged); + + const poller = createPoller( + { + fetchOptions: baseFetchOptions, + intervalMs: 30000, + initialEtag: 'X', + onUpdate, + }, + logger, + ); + + await poller.pollOnce(); + + expect(mockedFetchConfig).toHaveBeenCalledTimes(1); + expect(mockedFetchConfig).toHaveBeenCalledWith({ + ...baseFetchOptions, + lastEtag: 'X', + }); + expect(onUpdate).not.toHaveBeenCalled(); + expect(logger.debug).toHaveBeenCalledWith('Config unchanged'); + }); + + it('sends lastEtag undefined on the first poll when no initialEtag is provided', async () => { + const logger = makeTypedLogger(); + const onUpdate = jest.fn(async () => {}); + mockedFetchConfig.mockResolvedValue({ changed: false }); + + const poller = createPoller( + { + fetchOptions: baseFetchOptions, + intervalMs: 30000, + onUpdate, + }, + logger, + ); + + await poller.pollOnce(); + + expect(mockedFetchConfig).toHaveBeenCalledWith({ + ...baseFetchOptions, + lastEtag: undefined, + }); + }); + + it('calls onUpdate and advances lastEtag on a changed result', async () => { + const logger = makeTypedLogger(); + const onUpdate = jest.fn(async () => {}); + const changed: ConfigFetchResult = { + changed: true, + content: { version: 2 }, + version: 'v2', + etag: '"v2"', + }; + mockedFetchConfig + .mockResolvedValueOnce(changed) + .mockResolvedValueOnce({ changed: false }); + + const poller = createPoller( + { + fetchOptions: baseFetchOptions, + intervalMs: 30000, + initialEtag: 'X', + onUpdate, + }, + logger, + ); + + await poller.pollOnce(); + + expect(onUpdate).toHaveBeenCalledTimes(1); + expect(onUpdate).toHaveBeenCalledWith({ version: 2 }, 'v2'); + + // Next poll should carry the advanced etag, not the initial seed. + await poller.pollOnce(); + + expect(mockedFetchConfig).toHaveBeenNthCalledWith(2, { + ...baseFetchOptions, + lastEtag: '"v2"', + }); + }); + + it('does NOT reset an advanced etag back to the boot seed across stop/start', async () => { + const logger = makeTypedLogger(); + const onUpdate = jest.fn(async () => {}); + mockedFetchConfig + .mockResolvedValueOnce({ + changed: true, + content: { version: 2 }, + version: 'v2', + etag: '"v2"', + }) + .mockResolvedValue({ changed: false }); + + const poller = createPoller( + { + fetchOptions: baseFetchOptions, + intervalMs: 30000, + initialEtag: 'X', + onUpdate, + }, + logger, + ); + + await poller.pollOnce(); + + poller.start(); + poller.stop(); + + await poller.pollOnce(); + + // After advancing past the boot seed, a stop/start must not reset + // lastEtag to the stale 'X' (which would trigger an unnecessary rebundle). + expect(mockedFetchConfig).toHaveBeenNthCalledWith(2, { + ...baseFetchOptions, + lastEtag: '"v2"', + }); + }); +}); diff --git a/packages/cli/src/runtime/__tests__/redact.test.ts b/packages/cli/src/runtime/__tests__/redact.test.ts new file mode 100644 index 000000000..a30427d80 --- /dev/null +++ b/packages/cli/src/runtime/__tests__/redact.test.ts @@ -0,0 +1,408 @@ +import { redactLine, redactErrors, redactLogs } from '../redact.js'; +import type { DedupedError, RingEntry } from '../log-ring.js'; + +describe('redactLine', () => { + // Case 1: High-entropy token (32+ char base64/hex run) + it('masks high-entropy token standalone', () => { + const token = 'AKIAIOSFODNN7EXAMPLEKEY1234ABCD'; + expect(redactLine(`token is ${token} here`)).toBe('token is *** here'); + }); + + it('masks 32-char hex token', () => { + const hex = 'a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6'; + expect(redactLine(`key=${hex}`)).toBe('key=***'); + }); + + // Case 2: URL credentials + it('masks password in URL credentials', () => { + const result = redactLine('postgres://user:p4ssw0rdSecret@host/db'); + expect(result).toBe('postgres://user:***@host/db'); + }); + + it('keeps host visible after URL credential masking', () => { + const result = redactLine( + 'connecting to postgres://admin:s3cr3tP4ss@db.example.com/mydb', + ); + expect(result).toContain('db.example.com'); + expect(result).not.toContain('s3cr3tP4ss'); + }); + + // Case 3: Authorization Bearer token + it('masks Bearer token in Authorization header', () => { + const longToken = + 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0'; + const result = redactLine(`Authorization: Bearer ${longToken}`); + expect(result).not.toContain(longToken); + expect(result).toContain('***'); + }); + + // Case 4: KEY=value secret + it('masks WALKEROS_INGEST_TOKEN value but keeps key name', () => { + const result = redactLine( + 'WALKEROS_INGEST_TOKEN=sk_live_abc123def456ghi789', + ); + expect(result).toBe('WALKEROS_INGEST_TOKEN=***'); + }); + + it('masks KEY: value style secret', () => { + const result = redactLine('SOME_SECRET_KEY: sk_live_abc123def456ghi789012'); + expect(result).toContain('SOME_SECRET_KEY'); + expect(result).not.toContain('sk_live_abc123def456ghi789012'); + expect(result).toContain('***'); + }); + + // Case 5: Multi-line PEM block + it('removes entire PEM private key block — no body lines survive', () => { + const pem = [ + '-----BEGIN PRIVATE KEY-----', + 'MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7', + 'o2FlZXZlciBiZSBhYmxlIHRvIHJlYWQgdGhpcyBkYXRhLg==', + '-----END PRIVATE KEY-----', + ].join('\n'); + const result = redactLine(`some prefix\n${pem}\nsome suffix`); + expect(result).not.toContain('BEGIN PRIVATE KEY'); + expect(result).not.toContain('END PRIVATE KEY'); + expect(result).not.toContain( + 'MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7', + ); + expect(result).not.toContain( + 'o2FlZXZlciBiZSBhYmxlIHRvIHJlYWQgdGhpcyBkYXRhLg==', + ); + expect(result).toContain('some prefix'); + expect(result).toContain('some suffix'); + }); + + it('removes PEM block with no END marker — drops to end of entry', () => { + const input = [ + 'before', + '-----BEGIN PRIVATE KEY-----', + 'MIIEvQIBADANBgkqhkiG9w0BAQEFAASC', + ].join('\n'); + const result = redactLine(input); + expect(result).toContain('before'); + expect(result).not.toContain('BEGIN PRIVATE KEY'); + expect(result).not.toContain('MIIEvQIBADANBgkqhkiG9w0BAQEFAASC'); + }); + + // Case 6: JSON service-account private_key field + it('masks private_key value in JSON service-account fragment', () => { + const json = + '{"type":"service_account","client_email":"x@y.iam.gserviceaccount.com","private_key":"-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBg==\\n-----END PRIVATE KEY-----\\n"}'; + const result = redactLine(json); + expect(result).not.toContain('MIIEvQIBADANBg=='); + // The field key name survives for legibility; its value is masked. + expect(result).toContain('client_email'); + }); + + // Case 6b: service-account identifier fields are masked. + // client_email and private_key_id identify and pair with the leaked SA, so + // they must not ship even though they are not the private key body itself. + it('masks client_email value in a service-account JSON fragment', () => { + const json = + '{"type":"service_account","client_email":"svc@my-proj.iam.gserviceaccount.com","token_uri":"https://oauth2.googleapis.com/token"}'; + const result = redactLine(json); + expect(result).not.toContain('svc@my-proj.iam.gserviceaccount.com'); + expect(result).toContain('client_email'); + expect(result).toContain('***'); + // Non-secret structural fields are untouched. + expect(result).toContain('service_account'); + }); + + it('masks private_key_id value in a service-account JSON fragment', () => { + const json = + '{"private_key_id":"a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0","type":"service_account"}'; + const result = redactLine(json); + expect(result).not.toContain('a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0'); + expect(result).toContain('private_key_id'); + expect(result).toContain('***'); + }); + + // Case 6c: a full SA JSON blob embedded in an error message ships nothing + // sensitive: no private key body, no email, no key id, no cert url. + it('strips a full service-account JSON blob from an error message', () => { + const sa = + '{"type":"service_account","project_id":"my-proj","private_key_id":"a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0",' + + '"private_key":"-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcw==\\n-----END PRIVATE KEY-----\\n",' + + '"client_email":"svc@my-proj.iam.gserviceaccount.com",' + + '"client_x509_cert_url":"https://www.googleapis.com/robot/v1/metadata/x509/svc%40my-proj.iam.gserviceaccount.com"}'; + const result = redactLine(`BigQuery init failed: ${sa}`); + expect(result).not.toContain('MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcw=='); + expect(result).not.toContain('svc@my-proj.iam.gserviceaccount.com'); + expect(result).not.toContain('svc%40my-proj.iam.gserviceaccount.com'); + expect(result).not.toContain('a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0'); + expect(result).toContain('***'); + // Still informative: the operator sees what failed and the project. + expect(result).toContain('BigQuery init failed'); + expect(result).toContain('my-proj'); + }); + + // Case 6d: cert url and client_id are named explicitly, not left to heuristics. + it('masks client_x509_cert_url and client_id JSON values explicitly', () => { + const json = + '{"client_id":"123456789012345678901","client_x509_cert_url":"https://www.googleapis.com/robot/v1/metadata/x509/svc%40my-proj.iam.gserviceaccount.com"}'; + const result = redactLine(json); + expect(result).not.toContain('123456789012345678901'); + expect(result).not.toContain('www.googleapis.com/robot/v1/metadata/x509'); + expect(result).not.toContain('svc%40my-proj.iam.gserviceaccount.com'); + expect(result).toContain('***'); + }); + + // Case 7: Normal diagnostic line passes through + it('passes normal diagnostic messages through unmodified', () => { + const msg = 'Error: BigQuery 403 PERMISSION_DENIED on dataset analytics'; + expect(redactLine(msg)).toBe(msg); + }); + + it('does not over-redact short values or common words', () => { + expect(redactLine('status: ok')).toBe('status: ok'); + expect(redactLine('count=42')).toBe('count=42'); + expect(redactLine('user=alice')).toBe('user=alice'); + }); + + // Case 8: Long message truncated to a total of at most 256 chars (wire contract) + it('truncates messages longer than 256 chars to total length <= 256', () => { + // 'Z' is not a hex char and all-same-char entropy is 0, so this run passes + // through unmasked and exercises the truncation path. + const long = 'Z'.repeat(300); + const result = redactLine(long); + // The app's heartbeat schema enforces message .max(256). The ellipsis must + // fit WITHIN the 256 budget, so total length is exactly 256 here, never 257. + expect(result.length).toBeLessThanOrEqual(256); + expect(result.length).toBe(256); + expect(result.endsWith('…')).toBe(true); + expect(result.startsWith('ZZZ')).toBe(true); + }); + + // Case 9: Secret straddling the 256-char boundary is masked not partially shown + it('masks a secret straddling the 256-char boundary (masking before truncation)', () => { + // Place a high-entropy token starting at position 240 (straddles char 256) + const prefix = 'B'.repeat(240); + const token = 'sk_live_abc123def456ghi789jkl0123456'; // 35 chars, starts at 240, ends at 275 + const input = prefix + token; + expect(input.length).toBeGreaterThan(256); + const result = redactLine(input); + // The token must not appear partially in the output + expect(result).not.toContain('sk_live_abc123def456ghi789'); + expect(result).not.toContain('sk_live'); + expect(result).toContain('***'); + }); + + // Case 10: Adversarial large input completes without hanging + it('handles ~10KB adversarial input without catastrophic backtracking', () => { + // Token is at the end of the large input; after truncation it won't appear in the + // 256-char window. The key assertion is no catastrophic backtracking (time bound) + // and that the raw token value is not visible in the output. 'Z' is non-hex and + // low-entropy so the padding survives unmasked and the result stays truncated. + const bigLine = + 'Z'.repeat(10000) + 'sk_live_abc123def456ghi789jkl012345678'; + const start = Date.now(); + const result = redactLine(bigLine); + const elapsed = Date.now() - start; + // Should complete well under 1 second — proves no ReDoS + expect(elapsed).toBeLessThan(1000); + // The token value must not appear verbatim (it's either masked or truncated away) + expect(result).not.toContain('sk_live_abc123def456ghi789jkl012345678'); + // Result is truncated + expect(result.endsWith('…')).toBe(true); + }); + + // ── Security-review regressions ────────────────────────────────────────── + + // Leak A: lowercase / indented PEM block leaked its body (case-sensitive bug) + it('removes a lowercase PEM block (case-insensitive)', () => { + const pem = [ + '-----begin private key-----', + 'MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7', + '-----end private key-----', + ].join('\n'); + const result = redactLine(`prefix\n${pem}\nsuffix`); + expect(result).not.toContain('begin private key'); + expect(result).not.toContain( + 'MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7', + ); + expect(result).toContain('prefix'); + expect(result).toContain('suffix'); + }); + + it('removes an indented PEM block (leading whitespace tolerated)', () => { + const pem = [ + ' -----BEGIN PRIVATE KEY-----', + 'MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7', + ' -----END PRIVATE KEY-----', + ].join('\n'); + const result = redactLine(pem); + expect(result).not.toContain('PRIVATE KEY'); + expect(result).not.toContain( + 'MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7', + ); + }); + + // Leak B: all-hex digest (no letters mixed with digits in the >=20 sense) leaked + it('masks an all-hex digest', () => { + const digest = 'deadbeefdeadbeefdeadbeefdeadbeef'; // 32 hex chars + const result = redactLine(`checksum ${digest} computed`); + expect(result).not.toContain(digest); + expect(result).toContain('***'); + }); + + // Leak C: all-digit run leaked + it('masks an all-digit token run', () => { + const digits = '12345678901234567890123'; // 23 digits + const result = redactLine(`id=${digits}`); + expect(result).not.toContain(digits); + expect(result).toContain('***'); + }); + + // Leak D: all-letter base64url token (random, high entropy) leaked + it('masks an all-letter high-entropy token', () => { + const token = 'AbCdEfGhIjKlMnOpQrStUvWxYz'; // 26 distinct-ish letters, entropy >= 4.0 + const result = redactLine(`token ${token} ok`); + expect(result).not.toContain(token); + expect(result).toContain('***'); + }); + + // Leak E: Bearer leaked (no digit gate caught it) + it('masks a letter-only Bearer token', () => { + const token = 'AbCdEfGhIjKlMnOpQrStUvWxYzaBcDeFgHiJ'; // random-looking letters + const result = redactLine(`Authorization: Bearer ${token}`); + expect(result).not.toContain(token); + expect(result).toContain('***'); + }); + + // Leak F: Slack hyphenated token leaked because '-' terminated the run + it('masks a hyphenated Slack token (xoxb-...)', () => { + const token = 'xoxb-2384-2384-AbCdEfGhIjKlMnOp'; + const result = redactLine(`SLACK_TOKEN: ${token}`); + expect(result).not.toContain(token); + expect(result).not.toContain('AbCdEfGhIjKlMnOp'); + expect(result).toContain('***'); + }); + + it('masks a standalone hyphenated Slack token without a key prefix', () => { + const token = 'xoxp-2384-2384-AbCdEfGhIjKlMnOp'; + const result = redactLine(`logged token ${token} here`); + expect(result).not.toContain('xoxp-2384'); + expect(result).not.toContain('AbCdEfGhIjKlMnOp'); + expect(result).toContain('***'); + }); + + // Force-mask prefixes are masked regardless of length/entropy + it.each([ + ['sk-shorttoken123', 'sk-'], + ['pk_shorttoken123', 'pk_'], + ['ghp_16charsToken00', 'ghp_'], + ['gho_16charsToken00', 'gho_'], + ['AKIAIOSFODNN7EXAMPLE', 'AKIA'], + ])('force-masks known-prefix token %s', (token) => { + const result = redactLine(`value is ${token} end`); + expect(result).not.toContain(token); + expect(result).toContain('***'); + }); + + // Leak G: JWT only partially masked — middle claims segment survived + it('masks the FULL JWT (no surviving eyJ segment)', () => { + const header = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9'; + const payload = 'eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4ifQ'; + const signature = 'SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c'; + const jwt = `${header}.${payload}.${signature}`; + const result = redactLine(`Authorization: Bearer ${jwt}`); + // No base64url segment of the JWT may survive — the whole thing is one run + expect(result).not.toContain('eyJ'); + expect(result).not.toContain(header); + expect(result).not.toContain(payload); + expect(result).not.toContain(signature); + expect(result).toContain('***'); + }); + + // ReDoS: large no-space line must stay linear (the 10KB test is too small). + // The bound is deliberately loose (2s): the fixed regex runs in single-digit + // ms while the prior O(n^2) backtracking took seconds at this size, so 2s + // cleanly separates linear from quadratic without flaking on slow/contended CI. + it('handles a 60KB no-space line without quadratic backtracking', () => { + // Key-prefixed value with no spaces, 60KB — the prior unbounded key prefix + // backtracked O(n^2) on input like this. + const big = 'KEY=' + 'a'.repeat(60_000); + const start = Date.now(); + const result = redactLine(big); + const elapsed = Date.now() - start; + expect(elapsed).toBeLessThan(2000); + expect(typeof result).toBe('string'); + }); + + it('handles an 80KB key-shaped no-space line without quadratic backtracking', () => { + // No '=' / ':' at all — exercises the key-prefix scan alone at scale. + const big = 'a'.repeat(80_000); + const start = Date.now(); + const result = redactLine(big); + const elapsed = Date.now() - start; + expect(elapsed).toBeLessThan(2000); + expect(typeof result).toBe('string'); + }); +}); + +describe('redactErrors', () => { + it('masks secrets in error messages and converts timestamps to ISO strings', () => { + const errors: DedupedError[] = [ + { + message: 'token AKIAIOSFODNN7EXAMPLEKEY123456 leaked', + count: 3, + firstSeen: 1700000000000, + lastSeen: 1700000060000, + }, + { + message: 'BigQuery 403 PERMISSION_DENIED', + count: 1, + firstSeen: 1700000010000, + lastSeen: 1700000010000, + }, + ]; + + const result = redactErrors(errors); + + expect(result).toHaveLength(2); + + // First entry: secret masked, timestamps as ISO + expect(result[0].message).not.toContain('AKIAIOSFODNN7EXAMPLEKEY123456'); + expect(result[0].message).toContain('***'); + expect(result[0].count).toBe(3); + expect(result[0].firstSeen).toBe(new Date(1700000000000).toISOString()); + expect(result[0].lastSeen).toBe(new Date(1700000060000).toISOString()); + + // Second entry: normal message passes through + expect(result[1].message).toBe('BigQuery 403 PERMISSION_DENIED'); + expect(result[1].firstSeen).toBe(new Date(1700000010000).toISOString()); + expect(result[1].lastSeen).toBe(new Date(1700000010000).toISOString()); + }); +}); + +describe('redactLogs', () => { + it('masks secrets in log entries and converts timestamps to ISO strings', () => { + const entries: RingEntry[] = [ + { + time: 1700000000000, + level: 'info', + message: 'Connected to postgres://admin:s3cr3tP4ss@db.example.com/mydb', + }, + { + time: 1700000030000, + level: 'error', + message: 'Service started successfully', + }, + ]; + + const result = redactLogs(entries); + + expect(result).toHaveLength(2); + + // First entry: password masked, host visible, time as ISO + expect(result[0].message).not.toContain('s3cr3tP4ss'); + expect(result[0].message).toContain('db.example.com'); + expect(result[0].time).toBe(new Date(1700000000000).toISOString()); + expect(result[0].level).toBe('info'); + + // Second entry: normal message, time as ISO + expect(result[1].message).toBe('Service started successfully'); + expect(result[1].time).toBe(new Date(1700000030000).toISOString()); + expect(result[1].level).toBe('error'); + }); +}); diff --git a/packages/cli/src/runtime/__tests__/runner-logger-tap.test.ts b/packages/cli/src/runtime/__tests__/runner-logger-tap.test.ts new file mode 100644 index 000000000..a52d619ff --- /dev/null +++ b/packages/cli/src/runtime/__tests__/runner-logger-tap.test.ts @@ -0,0 +1,117 @@ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { createLogger, Level, type Logger } from '@walkeros/core'; +import { loadFlow } from '../runner.js'; +import { ErrorRing, LogRing } from '../index.js'; +import { createCLILoggerConfig } from '../../core/cli-logger.js'; + +/** + * D1: the runner must forward a collector `Logger.Config` (built from the same + * ring tap as the runner CLI logger) into the bundle factory as + * `context.logger`, so the deployed bundle's collector taps the ErrorRing. + * + * This drives a real on-disk bundle whose factory simulates the production + * generateServerEntry contract: `if (context.logger) config.logger = context.logger`, + * then a collector built with `createLogger(config.logger)` emits a scoped + * "Push failed" error exactly like collector/destination.ts does. + */ +describe('loadFlow forwards collector logger config to the bundle', () => { + const tempDirs: string[] = []; + let errorSpy: jest.SpyInstance; + let logSpy: jest.SpyInstance; + let originalCwd: string; + + beforeEach(() => { + originalCwd = process.cwd(); + errorSpy = jest.spyOn(console, 'error').mockImplementation(() => undefined); + logSpy = jest.spyOn(console, 'log').mockImplementation(() => undefined); + }); + + afterEach(() => { + errorSpy.mockRestore(); + logSpy.mockRestore(); + // loadFlow does process.chdir(flowDir); restore before removing temp dirs + // so the worker's cwd never points at a deleted directory (which would + // make every later suite on this worker fail with ENOENT uv_cwd). + process.chdir(originalCwd); + for (const dir of tempDirs) + fs.rmSync(dir, { recursive: true, force: true }); + tempDirs.length = 0; + jest.clearAllMocks(); + }); + + function createBundle(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'runner-logger-tap-')); + const file = path.join(dir, 'bundle.mjs'); + // Mirror generateServerEntry: only adopt context.logger when truthy, then + // simulate a collector that builds its own logger from config.logger and a + // failing destination push that logs a scoped error. + fs.writeFileSync( + file, + `import { createLogger } from '@walkeros/core'; +export default async function(context = {}) { + const config = {}; + if (context.logger) config.logger = context.logger; + const logger = createLogger({ + level: config.logger?.level, + handler: config.logger?.handler, + }); + return { + collector: { + push: async () => { + // Simulate a failing destination push. + logger.scope('bq').error('Push failed', { event: 'order complete' }); + return {}; + }, + command: async () => {}, + }, + }; +} +`, + 'utf8', + ); + tempDirs.push(dir); + return file; + } + + it('routes the bundle collector error into the runner error ring without --verbose', async () => { + const errorRing = new ErrorRing(20); + const logRing = new LogRing(100); + + const collectorLoggerConfig: Logger.Config = createCLILoggerConfig({ + verbose: false, + silent: true, + onLine: (level, message) => { + if (level === Level.ERROR) errorRing.add(message); + logRing.add({ time: Date.now(), level: 'error', message }); + }, + }); + + const runnerLogger = createLogger({ level: Level.DEBUG }); + const bundle = createBundle(); + + const handle = await loadFlow( + bundle, + { port: 0 }, + runnerLogger, + collectorLoggerConfig, + ); + + // loadFlow only exposes command/status on the handle; drive the push via + // a fresh load to reach the collector. Re-load and call push directly. + const { loadBundle } = await import('../load-bundle.js'); + const loaded = await loadBundle( + bundle, + { logger: collectorLoggerConfig }, + runnerLogger, + ); + await loaded.collector.push(); + + const snapshot = errorRing.snapshot(); + expect(snapshot).toHaveLength(1); + expect(snapshot[0].message).toBe('[bq] Push failed'); + + expect(handle.file).toBe(bundle); + }); +}); diff --git a/packages/cli/src/runtime/__tests__/runner.test.ts b/packages/cli/src/runtime/__tests__/runner.test.ts index 90c3c66e2..df2e285fa 100644 --- a/packages/cli/src/runtime/__tests__/runner.test.ts +++ b/packages/cli/src/runtime/__tests__/runner.test.ts @@ -1,99 +1,260 @@ -import { loadFlow, swapFlow, type FlowHandle } from '../runner'; - -const mockLogger = { - info: jest.fn(), - error: jest.fn(), - debug: jest.fn(), - warn: jest.fn(), - scope: jest.fn().mockReturnThis(), -}; - -describe('swapFlow', () => { - it('calls shutdown command on old flow before loading new one', async () => { - const shutdownOrder: string[] = []; - const commandFn = jest.fn().mockImplementation(async (cmd: string) => { - if (cmd === 'shutdown') shutdownOrder.push('shutdown'); - }); +import http from 'http'; +import { createLogger, Level } from '@walkeros/core'; +import { swapFlow, type FlowHandle, type FlowLoader } from '../runner.js'; +import type { HealthServer } from '../health-server.js'; - const oldHandle: FlowHandle = { - collector: { command: commandFn }, - file: '/old/bundle.mjs', - }; +/** + * In-memory HealthServer double recording the mounted flow handler and the + * functional readiness flag, so the swap's mount/readiness ordering can be + * asserted without binding a real port. + */ +function createFakeHealthServer(): { + healthServer: HealthServer; + getHandler: () => http.RequestListener | null; + isReady: () => boolean; + failures: string[]; +} { + let handler: http.RequestListener | null = null; + let ready = false; + const failures: string[] = []; + + const healthServer: HealthServer = { + server: http.createServer(), + setFlowHandler(next) { + handler = next; + }, + setReady(value) { + ready = value; + }, + setFailed(reason) { + ready = false; + failures.push(reason); + }, + close: async () => {}, + }; + + return { + healthServer, + getHandler: () => handler, + isReady: () => ready, + failures, + }; +} + +type Handler = NonNullable; + +function makeHandle( + file: string, + command: FlowHandle['collector']['command'], + httpHandler?: Handler, +): FlowHandle { + return { collector: { command }, file, httpHandler }; +} - // Mock loadFlow via dynamic import — we test swapFlow's ordering logic - // by verifying shutdown is called before the new handle is returned - const mockModule = { - default: jest.fn().mockImplementation(async () => { - shutdownOrder.push('load'); - return { - collector: { - command: jest.fn(), - }, - }; - }), +// The flow handler is the opaque runner shape `(...args: unknown[]) => void`, +// matching `FlowHandle['httpHandler']` (not http.RequestListener, which the +// runner only widens to when handing off to the health server). The body is +// never invoked here — these tests assert handle identity and mount ordering — +// so each handler is a distinct no-op tagged by `label` only for readability. +function makeHandler(label: string): Handler { + const handler: Handler = () => { + void label; + }; + return handler; +} + +const logger = createLogger({ level: Level.DEBUG }); + +describe('swapFlow — atomic load-then-swap with rollback', () => { + it('rolls back to the old handle when the new bundle fails to load', async () => { + const fake = createFakeHealthServer(); + // Old flow is already mounted and serving. + const oldHandler = makeHandler('old'); + fake.healthServer.setFlowHandler(oldHandler); + fake.healthServer.setReady(true); + + const oldShutdown = jest.fn(async () => {}); + const oldHandle = makeHandle('/old/bundle.mjs', oldShutdown, oldHandler); + + const failingLoader: FlowLoader = async () => { + throw new Error('bundle blew up'); }; - // We can't easily mock dynamic import, so test the ordering contract: - // shutdown must be called, and it must happen before we attempt loading - - // Verify shutdown command is called - try { - await swapFlow( - oldHandle, - '/new/bundle.mjs', - undefined, - mockLogger as any, - ); - } catch { - // loadFlow will fail (no real bundle), but shutdown should have been called - } + const result = await swapFlow( + oldHandle, + '/new/bundle.mjs', + undefined, + logger, + undefined, + fake.healthServer, + undefined, + failingLoader, + ); - expect(commandFn).toHaveBeenCalledWith('shutdown'); + // Old handle returned unchanged (no wedge). + expect(result).toBe(oldHandle); + // Old handler still mounted — setFlowHandler(null) was never called. + expect(fake.getHandler()).toBe(oldHandler); + // Readiness never dropped. + expect(fake.isReady()).toBe(true); + // Old collector NOT shut down on a failed swap. + expect(oldShutdown).not.toHaveBeenCalled(); }); - it('continues loading even if shutdown command throws', async () => { - const commandFn = jest.fn().mockRejectedValue(new Error('shutdown failed')); + it('never detaches the old handler (no setFlowHandler(null)) on failure', async () => { + const fake = createFakeHealthServer(); + const oldHandler = makeHandler('old'); + fake.healthServer.setFlowHandler(oldHandler); + fake.healthServer.setReady(true); - const oldHandle: FlowHandle = { - collector: { command: commandFn }, - file: '/old/bundle.mjs', + const setFlowHandlerSpy = jest.spyOn(fake.healthServer, 'setFlowHandler'); + const oldHandle = makeHandle('/old/bundle.mjs', async () => {}, oldHandler); + + const failingLoader: FlowLoader = async () => { + throw new Error('load failed'); }; - // swapFlow should catch the shutdown error and proceed to loadFlow - try { - await swapFlow( - oldHandle, - '/new/bundle.mjs', - undefined, - mockLogger as any, - ); - } catch (err: any) { - // Should fail on loadFlow (no real bundle), NOT on shutdown - expect(err.message).not.toContain('shutdown failed'); + await swapFlow( + oldHandle, + '/new/bundle.mjs', + undefined, + logger, + undefined, + fake.healthServer, + undefined, + failingLoader, + ); + + // setFlowHandler must not have been called with null at any point. + for (const call of setFlowHandlerSpy.mock.calls) { + expect(call[0]).not.toBeNull(); } + }); + + it('logs the load error on a failed swap', async () => { + const fake = createFakeHealthServer(); + const errorSpy = jest.spyOn(logger, 'error'); + const oldHandle = makeHandle('/old/bundle.mjs', async () => {}); - expect(commandFn).toHaveBeenCalledWith('shutdown'); - expect(mockLogger.debug).toHaveBeenCalledWith( - expect.stringContaining('Shutdown warning'), + const failingLoader: FlowLoader = async () => { + throw new Error('boom-during-load'); + }; + + await swapFlow( + oldHandle, + '/new/bundle.mjs', + undefined, + logger, + undefined, + fake.healthServer, + undefined, + failingLoader, ); + + expect( + errorSpy.mock.calls.some((call) => + String(call[0]).includes('boom-during-load'), + ), + ).toBe(true); + errorSpy.mockRestore(); }); - it('handles missing command function gracefully', async () => { - const oldHandle: FlowHandle = { - collector: {}, - file: '/old/bundle.mjs', + it('mounts the new handler and shuts the old collector down exactly once on success', async () => { + const fake = createFakeHealthServer(); + const oldHandler = makeHandler('old'); + fake.healthServer.setFlowHandler(oldHandler); + fake.healthServer.setReady(true); + + const order: string[] = []; + const oldShutdown = jest.fn(async () => { + order.push('old-shutdown'); + }); + const oldHandle = makeHandle('/old/bundle.mjs', oldShutdown, oldHandler); + + const newHandler = makeHandler('new'); + const newHandle = makeHandle('/new/bundle.mjs', async () => {}, newHandler); + + const successLoader: FlowLoader = async () => { + order.push('load'); + return newHandle; }; - // Should not throw on missing command, only on loadFlow (no real bundle) - try { - await swapFlow( - oldHandle, - '/new/bundle.mjs', - undefined, - mockLogger as any, - ); - } catch (err: any) { - expect(err.message).not.toContain('command'); - } + const result = await swapFlow( + oldHandle, + '/new/bundle.mjs', + undefined, + logger, + undefined, + fake.healthServer, + undefined, + successLoader, + ); + + expect(result).toBe(newHandle); + // New handler is mounted after a successful load. + expect(fake.getHandler()).toBe(newHandler); + expect(fake.isReady()).toBe(true); + // Old collector shut down exactly once. + expect(oldShutdown).toHaveBeenCalledTimes(1); + // Load happens before old shutdown (load-then-swap, not shut-then-load). + expect(order).toEqual(['load', 'old-shutdown']); + }); + + it('mounts the new handler before shutting the old collector down', async () => { + const fake = createFakeHealthServer(); + const order: string[] = []; + + const oldShutdown = jest.fn(async () => { + order.push('shutdown'); + }); + const oldHandle = makeHandle('/old/bundle.mjs', oldShutdown); + + const newHandler = makeHandler('new'); + const newHandle = makeHandle('/new/bundle.mjs', async () => {}, newHandler); + + jest.spyOn(fake.healthServer, 'setFlowHandler').mockImplementation((h) => { + if (h === newHandler) order.push('mount'); + }); + + const successLoader: FlowLoader = async () => newHandle; + + await swapFlow( + oldHandle, + '/new/bundle.mjs', + undefined, + logger, + undefined, + fake.healthServer, + undefined, + successLoader, + ); + + expect(order).toEqual(['mount', 'shutdown']); + }); + + it('swallows an old-collector shutdown error after a successful swap', async () => { + const fake = createFakeHealthServer(); + const newHandler = makeHandler('new'); + const newHandle = makeHandle('/new/bundle.mjs', async () => {}, newHandler); + const oldHandle = makeHandle('/old/bundle.mjs', async () => { + throw new Error('shutdown failed'); + }); + + const successLoader: FlowLoader = async () => newHandle; + + const result = await swapFlow( + oldHandle, + '/new/bundle.mjs', + undefined, + logger, + undefined, + fake.healthServer, + undefined, + successLoader, + ); + + // Swap still succeeds — new flow is mounted, old shutdown error is non-fatal. + expect(result).toBe(newHandle); + expect(fake.getHandler()).toBe(newHandler); }); }); diff --git a/packages/cli/src/runtime/__tests__/secrets-fetcher.test.ts b/packages/cli/src/runtime/__tests__/secrets-fetcher.test.ts index 57a69965f..ba9b5a245 100644 --- a/packages/cli/src/runtime/__tests__/secrets-fetcher.test.ts +++ b/packages/cli/src/runtime/__tests__/secrets-fetcher.test.ts @@ -1,10 +1,27 @@ import { fetchSecrets } from '../secrets-fetcher'; +import { RunnerAuthError } from '../runner-auth-error.js'; const mockFetch = jest.fn(); global.fetch = mockFetch; afterEach(() => mockFetch.mockReset()); +/** + * Drive a fetchSecrets call to completion while flushing the retry helper's + * backoff sleeps so a transient-then-success sequence settles without real + * waits. Mirrors the fetch-retry test's fake-timer drain. + */ +async function runWithTimers(promise: Promise): Promise { + const settled = promise.then( + (value) => ({ ok: true as const, value }), + (error: unknown) => ({ ok: false as const, error }), + ); + await jest.runAllTimersAsync(); + const result = await settled; + if (result.ok) return result.value; + throw result.error; +} + describe('fetchSecrets', () => { const opts = { appUrl: 'https://app.test', @@ -50,13 +67,76 @@ describe('fetchSecrets', () => { await expect(fetchSecrets(opts)).rejects.toThrow('404'); }); - it('throws on 500', async () => { + it('retries a persistent 500 to exhaustion then throws (3 calls)', async () => { + jest.useFakeTimers(); + try { + mockFetch.mockResolvedValue({ + ok: false, + status: 500, + statusText: 'Internal Server Error', + }); + await expect(runWithTimers(fetchSecrets(opts))).rejects.toThrow('500'); + expect(mockFetch).toHaveBeenCalledTimes(3); + } finally { + jest.useRealTimers(); + } + }); + + it('applies a timeout signal to the fetch (no longer unbounded)', async () => { + mockFetch.mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ values: {} }), + }); + await fetchSecrets(opts); + expect(mockFetch).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ + signal: expect.any(AbortSignal), + }), + ); + }); + + it('retries a transient 503 then succeeds', async () => { + jest.useFakeTimers(); + try { + mockFetch + .mockResolvedValueOnce({ + ok: false, + status: 503, + statusText: 'Service Unavailable', + }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ values: { API_KEY: 'secret123' } }), + }); + const result = await runWithTimers(fetchSecrets(opts)); + expect(result).toEqual({ API_KEY: 'secret123' }); + expect(mockFetch).toHaveBeenCalledTimes(2); + } finally { + jest.useRealTimers(); + } + }); + + it('does NOT retry a 403 (single call) and throws RunnerAuthError', async () => { mockFetch.mockResolvedValue({ ok: false, - status: 500, - statusText: 'Internal Server Error', + status: 403, + statusText: 'Forbidden', + headers: new Headers(), + clone() { + return { + status: 403, + statusText: 'Forbidden', + json: async () => ({ + error: { code: 'FORBIDDEN_SCOPE', message: 'no scope' }, + }), + }; + }, }); - await expect(fetchSecrets(opts)).rejects.toThrow('500'); + await expect(fetchSecrets(opts)).rejects.toBeInstanceOf(RunnerAuthError); + expect(mockFetch).toHaveBeenCalledTimes(1); }); it('URL-encodes path parameters', async () => { diff --git a/packages/cli/src/runtime/config-fetcher.ts b/packages/cli/src/runtime/config-fetcher.ts index b1eb6e026..e17ec8c21 100644 --- a/packages/cli/src/runtime/config-fetcher.ts +++ b/packages/cli/src/runtime/config-fetcher.ts @@ -1,4 +1,5 @@ import { mergeAuthHeaders } from '../core/http.js'; +import { fetchWithRetry } from './fetch-retry.js'; import { throwIfRunnerAuthFailure } from './runner-auth-error.js'; export interface FetchConfigResult { @@ -32,10 +33,9 @@ export async function fetchConfig( options.lastEtag ? { 'If-None-Match': options.lastEtag } : undefined, ); - const response = await fetch(url, { - headers, - signal: AbortSignal.timeout(30_000), - }); + // Retry transient failures (timeouts, 5xx, 429) within the boot health + // window; 304, 401/403, and other 4xx are returned for the checks below. + const response = await fetchWithRetry(url, { init: { headers } }); if (response.status === 304) { return { changed: false }; diff --git a/packages/cli/src/runtime/fetch-retry.ts b/packages/cli/src/runtime/fetch-retry.ts new file mode 100644 index 000000000..4f1535daf --- /dev/null +++ b/packages/cli/src/runtime/fetch-retry.ts @@ -0,0 +1,232 @@ +/** + * Bounded, classified retry around a single `fetch`. + * + * Cold-starting flow containers fetch their bundle/config/secrets over the + * network before the health server is up. A brief control-plane blip used to + * hard-fail the boot: one timed-out fetch and the container exited inside + * Scaleway's health window. This helper retries only TRANSIENT failures + * (timeouts, connection errors, 5xx, 429) a bounded number of times, while + * leaving deterministic failures (4xx other than 429) and successful responses + * for the caller to handle. The total time budget keeps retries inside the + * ~100s health window (failureThreshold 10 × interval 10s) so there is room + * left for archive extract, flow load, and health-server start. + */ + +/** Per-attempt request timeout. */ +const DEFAULT_PER_ATTEMPT_TIMEOUT_MS = 30_000; + +/** Total wall-clock budget across all attempts (including backoff sleeps). */ +const DEFAULT_MAX_TOTAL_MS = 60_000; + +/** Number of attempts (the first try plus retries). */ +const DEFAULT_ATTEMPTS = 3; + +/** + * Floor of remaining budget below which starting another attempt is pointless: + * a sub-millisecond AbortSignal.timeout would abort before the socket connects. + * Stopping at/below this keeps total wall-clock genuinely within maxTotalMs. + */ +const MIN_ATTEMPT_BUDGET_MS = 1_000; + +/** + * Base backoff before retry #2 and #3 respectively. With fewer base entries + * than retries, the final entry is reused for any further attempts. + */ +const BASE_BACKOFF_MS: readonly number[] = [2_000, 5_000]; + +/** Jitter band applied to each backoff: ±20%. */ +const JITTER = 0.2; + +/** libuv-style network error codes worth retrying. */ +const RETRYABLE_NETWORK_CODES: ReadonlySet = new Set([ + 'ECONNRESET', + 'ECONNREFUSED', + 'ETIMEDOUT', + 'EAI_AGAIN', + 'ENOTFOUND', +]); + +export interface FetchWithRetryOptions { + /** Total attempts (first try + retries). Default 3. */ + attempts?: number; + /** Per-attempt request timeout in ms. Default 30_000. */ + perAttemptTimeoutMs?: number; + /** Total wall-clock budget across attempts in ms. Default 60_000. */ + maxTotalMs?: number; + /** Extra fetch init merged into each attempt (signal is supplied here). */ + init?: RequestInit; +} + +/** Read an optional string `code` off an unknown error without casting. */ +function readErrorCode(value: unknown): string | undefined { + if (typeof value !== 'object' || value === null) return undefined; + const code = Reflect.get(value, 'code'); + return typeof code === 'string' ? code : undefined; +} + +/** Read the `name` of a thrown error/DOMException without casting. */ +function readErrorName(value: unknown): string | undefined { + if (typeof value !== 'object' || value === null) return undefined; + const name = Reflect.get(value, 'name'); + return typeof name === 'string' ? name : undefined; +} + +/** + * Classify a thrown fetch rejection as transient (worth retrying). + * + * - AbortSignal.timeout / abort surfaces as a TimeoutError/AbortError. + * - undici connection failures reject with a TypeError whose cause carries a + * libuv code, or expose the code directly. Treat any of the known codes + * (on the error or its cause) as transient; a bare network TypeError is also + * transient since fetch only rejects on genuine network failures, never on + * HTTP status. + */ +function isTransientThrow(error: unknown): boolean { + const name = readErrorName(error); + if (name === 'TimeoutError' || name === 'AbortError') return true; + + const directCode = readErrorCode(error); + if (directCode && RETRYABLE_NETWORK_CODES.has(directCode)) return true; + + if (typeof error === 'object' && error !== null) { + const causeCode = readErrorCode(Reflect.get(error, 'cause')); + if (causeCode && RETRYABLE_NETWORK_CODES.has(causeCode)) return true; + } + + // fetch rejects only on network failures (never on HTTP status), so a plain + // TypeError here is a transient connectivity problem. + return error instanceof TypeError; +} + +/** Classify an HTTP status as transient (worth retrying). */ +function isTransientStatus(status: number): boolean { + return status >= 500 || status === 429; +} + +/** Read an optional string `message` off an unknown value without casting. */ +function readErrorMessage(value: unknown): string | undefined { + if (typeof value !== 'object' || value === null) return undefined; + const message = Reflect.get(value, 'message'); + return typeof message === 'string' ? message : undefined; +} + +/** Short human description of the cause, for the exhaustion error message. */ +function describeReason(reason: TransientReason): string { + if (reason.kind === 'status') return `HTTP ${reason.status}`; + const name = readErrorName(reason.error); + const message = + reason.error instanceof Error ? reason.error.message : String(reason.error); + + // A bare network TypeError reads as "TypeError: fetch failed"; surface the + // underlying cause's code/message so the exhaustion error is actionable. + let detail = name ? `${name}: ${message}` : message; + if (typeof reason.error === 'object' && reason.error !== null) { + const cause = Reflect.get(reason.error, 'cause'); + const causeCode = readErrorCode(cause); + const causeMessage = readErrorMessage(cause); + const causeDetail = causeCode ?? causeMessage; + if (causeDetail) detail = `${detail} (${causeDetail})`; + } + return detail; +} + +type TransientReason = + | { kind: 'throw'; error: unknown } + | { kind: 'status'; status: number }; + +/** Backoff delay (with jitter) before the retry following attempt index `i`. */ +function backoffForAttempt(index: number): number { + const base = + BASE_BACKOFF_MS[Math.min(index, BASE_BACKOFF_MS.length - 1)] ?? 0; + const spread = base * JITTER; + return base + (Math.random() * 2 - 1) * spread; +} + +/** Promise-based sleep that fake timers can drive in tests. */ +function delay(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + +/** + * fetch with bounded, classified retry. Returns the Response on success OR on a + * non-retryable status (including 4xx other than 429) so the caller decides. + * Throws the last transient cause once attempts or the time budget are spent. + */ +export async function fetchWithRetry( + url: string, + options: FetchWithRetryOptions = {}, +): Promise { + const attempts = options.attempts ?? DEFAULT_ATTEMPTS; + const perAttemptTimeoutMs = + options.perAttemptTimeoutMs ?? DEFAULT_PER_ATTEMPT_TIMEOUT_MS; + const maxTotalMs = options.maxTotalMs ?? DEFAULT_MAX_TOTAL_MS; + const init = options.init; + + const start = Date.now(); + let lastReason: TransientReason | undefined; + let made = 0; + + for (let attempt = 0; attempt < attempts; attempt++) { + // Clamp each attempt's timeout to the budget that remains, so the total + // wall-clock is genuinely bounded by maxTotalMs (a full per-attempt timeout + // can no longer overrun the budget). If too little budget remains to make a + // meaningful attempt, stop before starting one. + const remaining = maxTotalMs - (Date.now() - start); + if (remaining <= MIN_ATTEMPT_BUDGET_MS) break; + const attemptTimeoutMs = Math.min(perAttemptTimeoutMs, remaining); + + made = attempt + 1; + let reason: TransientReason; + + try { + // Bound only the request phase. Clear the per-attempt timer the moment + // fetch resolves (headers received) or throws, so the timeout can never + // abort downstream body consumption (response.text() / stream piping), + // which runs outside this retry loop and would otherwise fail unretried. + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), attemptTimeoutMs); + let response: Response; + try { + response = await fetch(url, { + ...init, + signal: controller.signal, + }); + } finally { + clearTimeout(timeoutId); + } + + // Non-retryable status (success or deterministic 4xx) → hand back. + if (!isTransientStatus(response.status)) return response; + + // Discarded retryable response: release the socket undici holds until GC. + await response.body?.cancel(); + reason = { kind: 'status', status: response.status }; + } catch (error) { + if (!isTransientThrow(error)) throw error; + reason = { kind: 'throw', error }; + } + + lastReason = reason; + + // Stop if attempts are spent or the time budget is exhausted; otherwise + // wait out the backoff before the next try. + const isLastAttempt = attempt === attempts - 1; + const budgetSpent = Date.now() - start >= maxTotalMs; + if (isLastAttempt || budgetSpent) break; + + // Clamp the sleep to the budget that remains so total wall-clock stays + // literally within maxTotalMs; if no budget is left, stop instead of + // sleeping into an overrun. + const sleepMs = Math.min( + backoffForAttempt(attempt), + maxTotalMs - (Date.now() - start), + ); + if (sleepMs <= 0) break; + await delay(sleepMs); + } + + const cause = lastReason ? describeReason(lastReason) : 'no attempts made'; + throw new Error(`Fetch failed after ${made} attempts: ${cause}`); +} diff --git a/packages/cli/src/runtime/heartbeat.ts b/packages/cli/src/runtime/heartbeat.ts index ef4796c77..5b1125eca 100644 --- a/packages/cli/src/runtime/heartbeat.ts +++ b/packages/cli/src/runtime/heartbeat.ts @@ -1,26 +1,41 @@ import { randomBytes } from 'crypto'; import { VERSION } from '../version.js'; import { mergeAuthHeaders } from '../core/http.js'; +import { stepId } from '@walkeros/core'; import type { Collector, Logger } from '@walkeros/core'; +import type { DedupedError, RingEntry } from './log-ring.js'; +import { redactErrors, redactLogs } from './redact.js'; + +/** + * Per-destination heartbeat figures. + * + * `count`/`failed`/`duration`/`dropped` are deltas since the last reported + * snapshot (monotonic counters). `dlqSize` is a point-in-time gauge (current + * DLQ depth), reported as-is rather than delta'd so the operator sees the live + * backlog, not how it changed between two heartbeats. + */ +export interface DestinationCounter { + count: number; + failed: number; + duration: number; + /** Current DLQ depth (gauge). */ + dlqSize: number; + /** Events dropped from the destination's queue/DLQ buffers (delta). */ + dropped: number; +} export interface CounterPayload { eventsIn: number; eventsOut: number; eventsFailed: number; - destinations: Record< - string, - { count: number; failed: number; duration: number } - >; + destinations: Record; } export interface CounterSnapshot { in: number; out: number; failed: number; - destinations: Record< - string, - { count: number; failed: number; duration: number } - >; + destinations: Record; } export function computeCounterDelta( @@ -33,11 +48,17 @@ export function computeCounterDelta( count: 0, failed: 0, duration: 0, + dlqSize: 0, + dropped: 0, }; destinations[name] = { count: dest.count - prev.count, failed: dest.failed - prev.failed, duration: dest.duration - prev.duration, + // dlqSize is a current-depth gauge, not a monotonic counter: report the + // live value rather than the difference between two snapshots. + dlqSize: dest.dlqSize, + dropped: dest.dropped - prev.dropped, }; } return { @@ -49,24 +70,24 @@ export function computeCounterDelta( } /** - * Deep-copy destination status values to prevent shared references - * between snapshots from causing delta computation to always return 0. + * Build a per-destination snapshot from the collector status. Copies the + * scalar counters (preventing shared references that would zero out deltas) + * and folds in the destination's current DLQ depth (`dlqSize`) plus its total + * dropped count (`status.dropped["destination."]`, summing queue + dlq). */ function snapshotDestinations( - destinations: Record< - string, - { count: number; failed: number; duration: number } - >, -): Record { - const result: Record< - string, - { count: number; failed: number; duration: number } - > = {}; - for (const [name, dest] of Object.entries(destinations)) { + status: Collector.Status, +): Record { + const result: Record = {}; + for (const [name, dest] of Object.entries(status.destinations)) { + const drops = status.dropped[stepId('destination', name)]; + const dropped = (drops?.queue ?? 0) + (drops?.dlq ?? 0); result[name] = { count: dest.count, failed: dest.failed, duration: dest.duration, + dlqSize: dest.dlqSize, + dropped, }; } return result; @@ -87,6 +108,8 @@ export interface HeartbeatConfig { configVersion?: string; intervalMs: number; getCounters?: () => Collector.Status | undefined; + getErrors?: () => DedupedError[]; + getLogs?: () => RingEntry[]; } export interface HeartbeatHandle { @@ -122,11 +145,16 @@ export function createHeartbeat( in: status.in, out: status.out, failed: status.failed, - destinations: snapshotDestinations(status.destinations), + destinations: snapshotDestinations(status), }; counters = computeCounterDelta(current, lastReported); } + const errors = config.getErrors ? redactErrors(config.getErrors()) : []; + const logs = config.getLogs + ? redactLogs(config.getLogs().slice(-50)) + : []; + const response = await fetch( `${config.appUrl}/api/projects/${config.projectId}/runners/heartbeat`, { @@ -144,6 +172,11 @@ export function createHeartbeat( cliVersion: VERSION, uptime: Math.floor((Date.now() - startTime) / 1000), ...(counters && { counters }), + // Always send recentErrors (even []) so a heartbeat that no longer + // carries any errors clears a stale snapshot on the app side. logs + // have no clear semantics, so they stay omit-when-empty. + recentErrors: errors, + ...(logs.length && { recentLogs: logs }), }), signal: AbortSignal.timeout(10_000), }, diff --git a/packages/cli/src/runtime/index.ts b/packages/cli/src/runtime/index.ts index 8e508ab49..4a446ae2e 100644 --- a/packages/cli/src/runtime/index.ts +++ b/packages/cli/src/runtime/index.ts @@ -28,3 +28,12 @@ export { export { createPoller, type PollerConfig, type PollerHandle } from './poller'; export { createHealthServer, type HealthServer } from './health-server'; export { fetchSecrets, SecretsHttpError } from './secrets-fetcher'; +export { + LogRing, + ErrorRing, + type RingEntry, + type DedupedError, + type RecentError, + type RecentLogEntry, +} from './log-ring'; +export { redactLine, redactErrors, redactLogs } from './redact'; diff --git a/packages/cli/src/runtime/log-ring.ts b/packages/cli/src/runtime/log-ring.ts new file mode 100644 index 000000000..112bd6b27 --- /dev/null +++ b/packages/cli/src/runtime/log-ring.ts @@ -0,0 +1,75 @@ +export interface RingEntry { + time: number; + level: 'error' | 'warn' | 'info' | 'debug'; + message: string; +} + +export interface DedupedError { + message: string; + count: number; + firstSeen: number; + lastSeen: number; +} + +// Wire types (ISO timestamps) — defined here so redact.ts and heartbeat.ts +// both import from this module and avoid a circular dependency. +export interface RecentError { + message: string; + count: number; + firstSeen: string; + lastSeen: string; +} +export interface RecentLogEntry { + time: string; + level: 'error' | 'warn' | 'info' | 'debug'; + message: string; +} + +export class LogRing { + private readonly entries: RingEntry[] = []; + constructor(private readonly max: number) {} + + add(entry: RingEntry): void { + this.entries.push(entry); + if (this.entries.length > this.max) this.entries.shift(); + } + + snapshot(limit = this.max): RingEntry[] { + return this.entries.slice(Math.max(0, this.entries.length - limit)); + } +} + +export class ErrorRing { + private readonly map = new Map(); + + constructor( + private readonly maxUnique: number, + private readonly now: () => number = () => Date.now(), + ) {} + + add(message: string): void { + const ts = this.now(); + const existing = this.map.get(message); + if (existing) { + existing.count += 1; + existing.lastSeen = ts; + return; + } + if (this.map.size >= this.maxUnique) { + let oldestKey: string | undefined; + let oldest = Infinity; + for (const [k, v] of this.map) { + if (v.lastSeen < oldest) { + oldest = v.lastSeen; + oldestKey = k; + } + } + if (oldestKey !== undefined) this.map.delete(oldestKey); + } + this.map.set(message, { message, count: 1, firstSeen: ts, lastSeen: ts }); + } + + snapshot(): DedupedError[] { + return [...this.map.values()].sort((a, b) => b.lastSeen - a.lastSeen); + } +} diff --git a/packages/cli/src/runtime/poller.ts b/packages/cli/src/runtime/poller.ts index 0092c6fe8..1ffdcbb01 100644 --- a/packages/cli/src/runtime/poller.ts +++ b/packages/cli/src/runtime/poller.ts @@ -4,6 +4,13 @@ import type { Logger } from '@walkeros/core'; export interface PollerConfig { fetchOptions: Omit; intervalMs: number; + /** + * Seed value for the conditional-request etag. When set, the very first + * poll sends `If-None-Match`, so an unchanged config 304s instead of being + * re-bundled on every boot/restart. Resolved by the caller from the + * boot-time config fetch or `WALKEROS_CONFIG_ETAG`. + */ + initialEtag?: string; onUpdate: ( content: Record, version: string, @@ -21,7 +28,7 @@ export function createPoller( logger: Logger.Instance, ): PollerHandle { let timer: ReturnType | null = null; - let lastEtag: string | undefined; + let lastEtag: string | undefined = config.initialEtag; async function pollOnce(): Promise { try { @@ -47,7 +54,6 @@ export function createPoller( } function start(): void { - lastEtag = undefined; const jitter = config.intervalMs * 0.15 * (Math.random() * 2 - 1); timer = setInterval(() => pollOnce(), config.intervalMs + jitter); logger.info( diff --git a/packages/cli/src/runtime/redact.ts b/packages/cli/src/runtime/redact.ts new file mode 100644 index 000000000..6fbf5339c --- /dev/null +++ b/packages/cli/src/runtime/redact.ts @@ -0,0 +1,40 @@ +import type { + DedupedError, + RingEntry, + RecentError, + RecentLogEntry, +} from './log-ring.js'; +import { redactLine } from '../core/redact-line.js'; + +// The redactor lives in a neutral core module so the CLI logger handler +// (core/cli-logger.ts) and this heartbeat path share one set of patterns. +// `redactLine` here is the truncating wire variant (256-char heartbeat cap). +// The handler already scrubs secrets before they enter the ring, so applying +// `redactLine` to the snapshot is a cheap backstop on already-redacted text +// plus the wire-length truncation. +export { redactLine, scrubSecrets } from '../core/redact-line.js'; + +/** + * Map an array of DedupedError entries through redactLine, + * converting numeric timestamps to ISO-8601 strings. + */ +export function redactErrors(errors: DedupedError[]): RecentError[] { + return errors.map((e) => ({ + message: redactLine(e.message), + count: e.count, + firstSeen: new Date(e.firstSeen).toISOString(), + lastSeen: new Date(e.lastSeen).toISOString(), + })); +} + +/** + * Map an array of RingEntry log entries through redactLine, + * converting numeric timestamps to ISO-8601 strings. + */ +export function redactLogs(entries: RingEntry[]): RecentLogEntry[] { + return entries.map((e) => ({ + time: new Date(e.time).toISOString(), + level: e.level, + message: redactLine(e.message), + })); +} diff --git a/packages/cli/src/runtime/resolve-bundle.ts b/packages/cli/src/runtime/resolve-bundle.ts index d6741345d..44303ebe8 100644 --- a/packages/cli/src/runtime/resolve-bundle.ts +++ b/packages/cli/src/runtime/resolve-bundle.ts @@ -23,13 +23,11 @@ import { dirname, join } from 'path'; import { Readable } from 'stream'; import { x as tarExtract } from 'tar'; import { isStdinPiped, readStdin } from '../core/stdin.js'; +import { fetchWithRetry } from './fetch-retry.js'; /** Entry the runtime expects at the root of an extracted archive. */ const ARCHIVE_ENTRY = 'flow.mjs'; -/** Request timeout for the URL fetch headers (not the streaming extract). */ -const FETCH_TIMEOUT_MS = 30_000; - /** * Determine where to write fetched/stdin bundles. * In Docker: /app/flow/ exists → write there (module resolution works naturally). @@ -86,13 +84,16 @@ function writeBundleToDisk(writePath: string, content: string): void { /** * Perform the HTTP request and return the Response. - * A 30s timeout guards against silent container hangs while waiting on - * headers; it is intentionally NOT applied to body streaming downstream. + * + * fetchWithRetry bounds each attempt with a 30s timeout that is cleared once + * the response headers arrive, so it never aborts body streaming downstream, + * and retries transient failures (timeouts, connection + * errors, 5xx, 429) so a brief control-plane blip during cold start no longer + * hard-fails the boot. A non-retryable status (e.g. a 404) comes back as a + * Response and is turned into the precise error below. */ async function fetchOk(url: string): Promise { - const response = await fetch(url, { - signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), - }); + const response = await fetchWithRetry(url); if (!response.ok) { throw new Error( @@ -128,7 +129,8 @@ async function fetchTextToDisk( * extract itself is deliberately left unbounded. A multi-MB node_modules on a * cold container can take longer than 30s to unpack, and no abort signal is * wired into this pipe, so the body stream runs to completion regardless of - * the header timeout that already fired and settled. + * the per-request timeout, which fetchWithRetry clears once the response + * headers arrive. */ async function extractToDir( source: Readable, diff --git a/packages/cli/src/runtime/runner.ts b/packages/cli/src/runtime/runner.ts index 1b1cc933b..dba35c5c7 100644 --- a/packages/cli/src/runtime/runner.ts +++ b/packages/cli/src/runtime/runner.ts @@ -32,7 +32,13 @@ export interface FlowHandle { * `commands/bundle/bundler.ts`) installs each observer onto * `collector.observers` after `startFlow` returns. */ -export async function loadFlow( +/** + * Load a bundle into a `FlowHandle` WITHOUT mounting its handler onto the + * health server. This is the atomic-swap primitive: `swapFlow` loads the new + * bundle into a detached handle first, and only mounts it once the load + * succeeded, so a failed load never leaves the server handler-less. + */ +async function loadFlowHandle( file: string, config: RuntimeConfig | undefined, logger: Logger.Instance, @@ -53,11 +59,6 @@ export async function loadFlow( const result = await loadBundle(absolutePath, flowContext, logger); - // Mount flow's httpHandler onto runner's health server (opaque — no type inspection) - if (healthServer && typeof result.httpHandler === 'function') { - healthServer.setFlowHandler(result.httpHandler); - } - return { collector: { command: result.collector.command as FlowHandle['collector']['command'], @@ -69,8 +70,55 @@ export async function loadFlow( } /** - * Swap the running flow to a new bundle. Shuts down old flow FIRST to release - * the port, then loads the new bundle. Brief downtime is acceptable for Mode C. + * Signature of the bundle loader `swapFlow` uses to produce a fresh, unmounted + * handle. Defaults to `loadFlowHandle`; injectable so tests can drive the + * success/failure paths without a real on-disk bundle. + */ +export type FlowLoader = ( + file: string, + config: RuntimeConfig | undefined, + logger: Logger.Instance, + loggerConfig?: Logger.Config, + healthServer?: HealthServer, + observers?: Array, +) => Promise; + +export async function loadFlow( + file: string, + config: RuntimeConfig | undefined, + logger: Logger.Instance, + loggerConfig?: Logger.Config, + healthServer?: HealthServer, + observers?: Array, +): Promise { + const handle = await loadFlowHandle( + file, + config, + logger, + loggerConfig, + healthServer, + observers, + ); + + // Mount flow's httpHandler onto runner's health server (opaque — no type inspection) + if (healthServer && typeof handle.httpHandler === 'function') { + healthServer.setFlowHandler(handle.httpHandler); + } + + return handle; +} + +/** + * Atomically swap the running flow to a new bundle with rollback. + * + * Load-then-swap (not shut-then-load): the new bundle is loaded into a fresh, + * detached handle FIRST. Only on success is the new handler mounted and the + * OLD collector shut down. If the load fails, the OLD handler stays mounted, + * `/ready` is untouched (the old flow keeps serving), the error is logged, and + * the unchanged OLD handle is returned. This avoids the wedge where a failed + * load leaves the container handler-less with `/ready` stuck at 503. + * + * `load` is injectable for testing; it defaults to the real detached loader. */ export async function swapFlow( currentHandle: FlowHandle, @@ -80,33 +128,48 @@ export async function swapFlow( loggerConfig?: Logger.Config, healthServer?: HealthServer, observers?: Array, + load: FlowLoader = loadFlowHandle, ): Promise { - logger.info('Shutting down current flow for hot-swap...'); + logger.info('Loading new flow for hot-swap...'); + + // 1. Load the new bundle into a fresh, unmounted handle. The old flow stays + // mounted and serving throughout this step. + let newHandle: FlowHandle; + try { + newHandle = await load( + newFile, + config, + logger, + loggerConfig, + healthServer, + observers, + ); + } catch (error) { + // Rollback: keep the OLD handler mounted, keep /ready true, return the OLD + // handle unchanged. No wedge — the old flow continues serving. + logger.error( + `Hot-swap load failed, keeping current flow: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + return currentHandle; + } - // Detach old handler, health endpoints still work during swap - if (healthServer) { - healthServer.setFlowHandler(null); + // 2. Success — mount the new handler, then shut the OLD collector down. Mount + // before shutdown so the server never has a window without a handler. + if (healthServer && typeof newHandle.httpHandler === 'function') { + healthServer.setFlowHandler(newHandle.httpHandler); } - // Delegate to collector's shutdown command (destroys sources, destinations, transformers) try { if (currentHandle.collector.command) { await currentHandle.collector.command('shutdown'); } } catch (error) { + // The new flow is already live; an old-collector shutdown error is non-fatal. logger.debug(`Shutdown warning: ${error}`); } - // Load new flow, mounts new handler onto same server - const newHandle = await loadFlow( - newFile, - config, - logger, - loggerConfig, - healthServer, - observers, - ); - logger.info('Flow swapped successfully'); return newHandle; } diff --git a/packages/cli/src/runtime/secrets-fetcher.ts b/packages/cli/src/runtime/secrets-fetcher.ts index 61486910e..ef63e1dd5 100644 --- a/packages/cli/src/runtime/secrets-fetcher.ts +++ b/packages/cli/src/runtime/secrets-fetcher.ts @@ -1,4 +1,5 @@ import { mergeAuthHeaders } from '../core/http.js'; +import { fetchWithRetry } from './fetch-retry.js'; import { throwIfRunnerAuthFailure } from './runner-auth-error.js'; export interface FetchSecretsOptions { @@ -29,8 +30,16 @@ export async function fetchSecrets( const { appUrl, token, projectId, flowId } = options; const url = `${appUrl}/api/projects/${encodeURIComponent(projectId)}/flows/${encodeURIComponent(flowId)}/secrets/values`; - const res = await fetch(url, { - headers: mergeAuthHeaders(token, { 'Content-Type': 'application/json' }), + // Runs sequentially after the bundle fetch within Scaleway's ~100s container + // health window, so keep the total budget small. Retries transient failures + // (timeouts, 5xx, 429); 401/403, 404, and other 4xx are returned for the + // classification/checks below. The helper also adds the per-attempt timeout + // this fetch previously lacked. + const res = await fetchWithRetry(url, { + maxTotalMs: 20_000, + init: { + headers: mergeAuthHeaders(token, { 'Content-Type': 'application/json' }), + }, }); // Classify 401/403 with the app's error code (FORBIDDEN_FLOW, FORBIDDEN_SCOPE). diff --git a/packages/cli/src/types/api.gen.d.ts b/packages/cli/src/types/api.gen.d.ts index b4c660545..b89748216 100644 --- a/packages/cli/src/types/api.gen.d.ts +++ b/packages/cli/src/types/api.gen.d.ts @@ -183,6 +183,122 @@ export interface paths { patch?: never; trace?: never; }; + '/api/account': { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post?: never; + /** + * Delete own account + * @description Soft-delete the authenticated account, starting the 30-day grace window. Requires a confirmation of the account email in the body. Revokes all sessions, API tokens, and MCP tokens. Blocked with 409 when the caller is the sole owner of a project that still has other members. + */ + delete: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: { + content: { + 'application/json': components['schemas']['DeleteAccountRequest']; + }; + }; + responses: { + /** @description Account scheduled for deletion */ + 204: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Confirmation email does not match */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; + /** @description Unauthorized */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; + /** @description Sole owner of a shared project */ + 409: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['DeleteAccountBlocked']; + }; + }; + }; + }; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + '/api/account/export': { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Export own account data + * @description Download a portable JSON export of everything the platform holds about the authenticated account: profile, memberships, token and session metadata, MCP sessions with messages, feedback, and invitations. Metadata only; token hashes and secret values are never included. Served as a file attachment. + */ + get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Account data export (file attachment) */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['AccountExportResponse']; + }; + }; + /** @description Unauthorized */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; + }; + }; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; '/api/sessions': { parameters: { query?: never; @@ -2055,7 +2171,7 @@ export interface paths { put?: never; /** * Start deployment - * @description Start a new deployment for a flow. Returns 400 AMBIGUOUS_SETTINGS if the flow has multiple named settings — use the per-settings deploy endpoint instead. + * @description Start a new deployment for a flow. The bundle runs asynchronously on the worker. Returns 400 AMBIGUOUS_CONFIG when the flow has multiple named settings (use the per-settings deploy endpoint instead). When an Idempotency-Key replays a prior request, returns 200 with status `already_created`. */ post: { parameters: { @@ -2069,6 +2185,15 @@ export interface paths { }; requestBody?: never; responses: { + /** @description Deployment started, or idempotent replay of a prior request */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['StartDeploymentResponse']; + }; + }; /** @description Deployment started */ 201: { headers: { @@ -2096,6 +2221,15 @@ export interface paths { 'application/json': components['schemas']['ErrorResponse']; }; }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; /** @description Not found */ 404: { headers: { @@ -2114,6 +2248,15 @@ export interface paths { 'application/json': components['schemas']['ErrorResponse']; }; }; + /** @description Rate limited or concurrent deploy limit (Retry-After header) */ + 429: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; /** @description Service unavailable */ 503: { headers: { @@ -2893,25 +3036,375 @@ export interface paths { }; }; put?: never; - post?: never; + post?: never; + /** + * Delete preview + * @description Delete a preview and its S3 bundle. Requires member role. + */ + delete: { + parameters: { + query?: never; + header?: never; + path: { + projectId: string; + flowId: string; + previewId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Preview deleted */ + 204: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Unauthorized */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; + /** @description Not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; + }; + }; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + '/api/projects/{projectId}/flows/{flowId}/observe-sessions': { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Start observe session + * @description Start an Observe session for a flow. Validates the flow topology, inserts the row, and kicks off detached provisioning. Returns the row immediately as arming. + */ + post: { + parameters: { + query?: never; + header?: never; + path: { + projectId: string; + flowId: string; + }; + cookie?: never; + }; + requestBody?: { + content: { + 'application/json': components['schemas']['CreateObserveSessionRequest']; + }; + }; + responses: { + /** @description Observe session started */ + 201: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ObserveSessionResponse']; + }; + }; + /** @description Validation error */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; + /** @description Unauthorized */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; + /** @description Not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; + /** @description Flow topology not supported */ + 409: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; + /** @description Rate limit exceeded */ + 429: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; + }; + }; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + '/api/projects/{projectId}/flows/{flowId}/observe-sessions/{sessionId}': { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get observe session + * @description Get an observe session: status, error message, config snapshot, web activation info, and the live server endpoint when live. + */ + get: { + parameters: { + query?: never; + header?: never; + path: { + projectId: string; + flowId: string; + sessionId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Observe session details */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ObserveSessionResponse']; + }; + }; + /** @description Unauthorized */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; + /** @description Not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; + }; + }; + put?: never; + post?: never; + /** + * End observe session + * @description End an observe session: tear down the container, revoke credentials, delete the web preview, delete the row. Idempotent. + */ + delete: { + parameters: { + query?: never; + header?: never; + path: { + projectId: string; + flowId: string; + sessionId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Observe session ended */ + 204: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Unauthorized */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; + /** @description Not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; + }; + }; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + '/api/projects/{projectId}/flows/{flowId}/observe-sessions/{sessionId}/heartbeat': { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Heartbeat observe session + * @description Keep an observe session warm. The window posts this every 30s while open; a stale session is reaped by the janitor. + */ + post: { + parameters: { + query?: never; + header?: never; + path: { + projectId: string; + flowId: string; + sessionId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Heartbeat recorded */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ObserveSessionHeartbeatResponse']; + }; + }; + /** @description Unauthorized */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; + /** @description Not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; + }; + }; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + '/api/projects/{projectId}/flows/{flowId}/observe-sessions/{sessionId}/end': { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; /** - * Delete preview - * @description Delete a preview and its S3 bundle. Requires member role. + * End observe session (beacon) + * @description The navigator.sendBeacon end target for page unload. Mirrors the DELETE end route because sendBeacon cannot send a DELETE. Idempotent. */ - delete: { + post: { parameters: { query?: never; header?: never; path: { projectId: string; flowId: string; - previewId: string; + sessionId: string; }; cookie?: never; }; requestBody?: never; responses: { - /** @description Preview deleted */ + /** @description Observe session ended */ 204: { headers: { [name: string]: unknown; @@ -2927,15 +3420,6 @@ export interface paths { 'application/json': components['schemas']['ErrorResponse']; }; }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - 'application/json': components['schemas']['ErrorResponse']; - }; - }; /** @description Not found */ 404: { headers: { @@ -2947,6 +3431,7 @@ export interface paths { }; }; }; + delete?: never; options?: never; head?: never; patch?: never; @@ -5522,7 +6007,7 @@ export interface paths { patch?: never; trace?: never; }; - '/api/projects/{projectId}/deployments/{deploymentId}/versions': { + '/api/projects/{projectId}/deployments/{deploymentId}/stream': { parameters: { query?: never; header?: never; @@ -5530,15 +6015,12 @@ export interface paths { cookie?: never; }; /** - * List deployment versions - * @description List the version history for a deployment, paginated. Requires member role. + * Stream deployment status (SSE) + * @description Server-Sent Events (`text/event-stream`) stream of a deployment's live status. Emits named events: `status` (a snapshot payload, schema below), `done` (terminal, no body), and `timeout`. The CLI consumes this with a raw fetch while waiting for a deploy to finish. Requires member role. The schema documents the JSON `data:` of a `status` event; `errorCode`/`errorMessage` carry the persisted, redacted classification of a failed deploy. */ get: { parameters: { - query?: { - limit?: number; - offset?: number | null; - }; + query?: never; header?: never; path: { projectId: string; @@ -5548,13 +6030,13 @@ export interface paths { }; requestBody?: never; responses: { - /** @description Version history */ + /** @description SSE stream; `status` event payload shape documented here. */ 200: { headers: { [name: string]: unknown; }; content: { - 'application/json': components['schemas']['ListDeploymentVersionsResponse']; + 'text/event-stream': components['schemas']['DeploymentStreamStatusEvent']; }; }; /** @description Unauthorized */ @@ -5566,15 +6048,6 @@ export interface paths { 'application/json': components['schemas']['ErrorResponse']; }; }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - 'application/json': components['schemas']['ErrorResponse']; - }; - }; /** @description Not found */ 404: { headers: { @@ -5584,15 +6057,6 @@ export interface paths { 'application/json': components['schemas']['ErrorResponse']; }; }; - /** @description Rate limited */ - 429: { - headers: { - [name: string]: unknown; - }; - content: { - 'application/json': components['schemas']['ErrorResponse']; - }; - }; }; }; put?: never; @@ -5603,7 +6067,7 @@ export interface paths { patch?: never; trace?: never; }; - '/api/projects/{projectId}/deployments/{deploymentId}/versions/current/content': { + '/api/projects/{projectId}/deployments/{deploymentId}/versions': { parameters: { query?: never; header?: never; @@ -5611,12 +6075,15 @@ export interface paths { cookie?: never; }; /** - * Get current deployed content - * @description Get the active deployed per-setting content for a deployment, used to diff changes since deploy. Content is masked and display-only; every field is null when there is no deployed baseline. Requires member role. + * List deployment versions + * @description List the version history for a deployment, paginated. Requires member role. */ get: { parameters: { - query?: never; + query?: { + limit?: number; + offset?: number | null; + }; header?: never; path: { projectId: string; @@ -5626,13 +6093,13 @@ export interface paths { }; requestBody?: never; responses: { - /** @description Deployed content (or a null baseline) */ + /** @description Version history */ 200: { headers: { [name: string]: unknown; }; content: { - 'application/json': components['schemas']['DeployedContentResponse']; + 'application/json': components['schemas']['ListDeploymentVersionsResponse']; }; }; /** @description Unauthorized */ @@ -5681,7 +6148,7 @@ export interface paths { patch?: never; trace?: never; }; - '/api/projects/{projectId}/deployments/{deploymentId}/heartbeats': { + '/api/projects/{projectId}/deployments/{deploymentId}/versions/current/content': { parameters: { query?: never; header?: never; @@ -5689,19 +6156,12 @@ export interface paths { cookie?: never; }; /** - * List deployment heartbeats - * @description List heartbeat records for a deployment with optional from/to time-range filtering and pagination. Requires member role. + * Get current deployed content + * @description Get the active deployed per-setting content for a deployment, used to diff changes since deploy. Content is masked and display-only; every field is null when there is no deployed baseline. Requires member role. */ get: { parameters: { - query?: { - /** @description ISO start of the time range. */ - from?: string; - /** @description ISO end of the time range. */ - to?: string; - limit?: number; - offset?: number | null; - }; + query?: never; header?: never; path: { projectId: string; @@ -5711,13 +6171,13 @@ export interface paths { }; requestBody?: never; responses: { - /** @description Heartbeat history */ + /** @description Deployed content (or a null baseline) */ 200: { headers: { [name: string]: unknown; }; content: { - 'application/json': components['schemas']['ListHeartbeatsResponse']; + 'application/json': components['schemas']['DeployedContentResponse']; }; }; /** @description Unauthorized */ @@ -5747,6 +6207,15 @@ export interface paths { 'application/json': components['schemas']['ErrorResponse']; }; }; + /** @description Rate limited */ + 429: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['ErrorResponse']; + }; + }; }; }; put?: never; @@ -5757,22 +6226,27 @@ export interface paths { patch?: never; trace?: never; }; - '/api/projects/{projectId}/deployments/{deploymentId}/rotate-ingest-token': { + '/api/projects/{projectId}/deployments/{deploymentId}/heartbeats': { parameters: { query?: never; header?: never; path?: never; cookie?: never; }; - get?: never; - put?: never; /** - * Rotate ingest token - * @description Rotate the ingest token for a deployment. Owner-only. No grace window: the previous token is immediately invalidated and the new token is returned once. + * List deployment heartbeats + * @description List heartbeat records for a deployment with optional from/to time-range filtering and pagination. Requires member role. */ - post: { + get: { parameters: { - query?: never; + query?: { + /** @description ISO start of the time range. */ + from?: string; + /** @description ISO end of the time range. */ + to?: string; + limit?: number; + offset?: number | null; + }; header?: never; path: { projectId: string; @@ -5782,13 +6256,13 @@ export interface paths { }; requestBody?: never; responses: { - /** @description New ingest token */ + /** @description Heartbeat history */ 200: { headers: { [name: string]: unknown; }; content: { - 'application/json': components['schemas']['RotateIngestTokenResponse']; + 'application/json': components['schemas']['ListHeartbeatsResponse']; }; }; /** @description Unauthorized */ @@ -5820,13 +6294,15 @@ export interface paths { }; }; }; + put?: never; + post?: never; delete?: never; options?: never; head?: never; patch?: never; trace?: never; }; - '/api/projects/{projectId}/deployments/{deploymentId}/trace': { + '/api/projects/{projectId}/deployments/{deploymentId}/rotate-ingest-token': { parameters: { query?: never; header?: never; @@ -5836,8 +6312,8 @@ export interface paths { get?: never; put?: never; /** - * Toggle debug tracing - * @description Enable or disable debug tracing for a deployment. Owner-only, gated by the debugTrace feature. `minutes` of 0 disables; allowed values are 0, 15, 30, 60. An absent body defaults to 15 minutes. + * Rotate ingest token + * @description Rotate the ingest token for a deployment. Owner-only. No grace window: the previous token is immediately invalidated and the new token is returned once. */ post: { parameters: { @@ -5849,28 +6325,15 @@ export interface paths { }; cookie?: never; }; - requestBody?: { - content: { - 'application/json': components['schemas']['TraceDeploymentRequest']; - }; - }; + requestBody?: never; responses: { - /** @description Updated trace window */ + /** @description New ingest token */ 200: { headers: { [name: string]: unknown; }; content: { - 'application/json': components['schemas']['TraceDeploymentResponse']; - }; - }; - /** @description Validation error */ - 400: { - headers: { - [name: string]: unknown; - }; - content: { - 'application/json': components['schemas']['ErrorResponse']; + 'application/json': components['schemas']['RotateIngestTokenResponse']; }; }; /** @description Unauthorized */ @@ -7289,10 +7752,22 @@ export interface components { * @enum {string} */ platform: 'web' | 'server'; - deploymentStatus: string | null; + serving: components['schemas']['ServingStatus']; + latestAttempt: components['schemas']['LatestAttemptStatus']; deploymentUrl: string | null; deployedAt: string | null; }; + /** @enum {string} */ + ServingStatus: 'live' | 'none'; + /** @enum {string|null} */ + LatestAttemptStatus: + | 'idle' + | 'deploying' + | 'published' + | 'active' + | 'stopped' + | 'failed' + | null; Version: { /** @example 1 */ version: number; @@ -7350,6 +7825,106 @@ export interface components { */ createdAt: string; }; + DeleteAccountRequest: { + /** @example me@example.com */ + confirm: string; + }; + DeleteAccountBlocked: { + error: { + /** + * @example SOLE_OWNER + * @enum {string} + */ + code: 'SOLE_OWNER'; + message: string; + details: { + /** + * @example [ + * "proj_abc123" + * ] + */ + projects: string[]; + }; + }; + }; + AccountExportResponse: { + /** @example 2026-06-10T12:00:00.000Z */ + exportedAt: string; + profile: { + /** @example user_a1b2c3d4 */ + id: string; + /** @example me@example.com */ + email: string; + displayName: string | null; + createdAt: string; + lastLoginAt: string | null; + /** @example user */ + globalRole: string; + traits: string[]; + }; + memberships: { + projectId: string; + projectName: string; + role: string; + joinedAt: string; + }[]; + apiTokens: { + id: string; + name: string; + projectId: string | null; + origin: string; + createdAt: string; + lastUsedAt: string | null; + expiresAt: string | null; + revokedAt: string | null; + }[]; + sessions: { + id: string; + createdAt: string; + expiresAt: string; + lastTouchedAt: string; + }[]; + mcpTokens: { + id: string; + name: string; + createdAt: string; + lastUsedAt: string | null; + expiresAt: string; + revokedAt: string | null; + }[]; + mcpSessions: { + id: string; + projectId: string | null; + createdAt: string; + lastActiveAt: string; + expiresAt: string; + messages: { + seq: number; + role: string; + content?: unknown; + createdAt: string; + }[]; + }[]; + feedback: { + id: string; + projectId: string | null; + text: string; + source: string; + createdAt: string; + }[]; + invitations: { + id: string; + projectId: string; + invitedEmail: string; + role: string; + status: string; + createdAt: string; + expiresAt: string; + acceptedAt: string | null; + declinedAt: string | null; + cancelledAt: string | null; + }[]; + }; ApiTokenSummary: { /** @example tok_a1b2c3d4 */ id: string; @@ -7471,6 +8046,9 @@ export interface components { createdAt: string; updatedAt: string | null; } | null; + serving: components['schemas']['ServingStatus']; + latestAttempt: components['schemas']['LatestAttemptStatus']; + deployedAt: string | null; /** * Format: date-time * @example 2026-01-26T14:30:00.000Z @@ -7547,6 +8125,7 @@ export interface components { | 'active' | 'stopped' | 'failed'; + serving: components['schemas']['ServingStatus']; currentVersionNumber: number | null; url: string | null; /** @example flow_a1b2c3d4 */ @@ -7556,8 +8135,6 @@ export interface components { createdAt: string; /** Format: date-time */ updatedAt: string; - /** Format: date-time */ - traceUntil: string | null; usageSummary?: { eventsIn24h: number; healthy: boolean; @@ -7594,6 +8171,21 @@ export interface components { currentVersion: components['schemas']['DeploymentVersionDetail']; versions: components['schemas']['DeploymentVersionHistoryEntry'][]; error: components['schemas']['DeploymentError']; + recentErrors?: + | { + message: string; + count: number; + firstSeen: string; + lastSeen: string; + }[] + | null; + recentLogs?: + | { + time: string; + level: string; + message: string; + }[] + | null; url: string | null; selfHosted: { /** Format: date-time */ @@ -7603,7 +8195,7 @@ export interface components { healthy: boolean; } | null; /** Format: date-time */ - traceUntil: string | null; + lastHeartbeatAt?: string | null; /** Format: date-time */ createdAt: string; /** Format: date-time */ @@ -7673,6 +8265,7 @@ export interface components { | 'active' | 'stopped' | 'failed'; + serving: components['schemas']['ServingStatus']; currentVersionNumber: number | null; url: string | null; /** @example flow_a1b2c3d4 */ @@ -7682,13 +8275,51 @@ export interface components { createdAt: string; /** Format: date-time */ updatedAt: string; - /** Format: date-time */ - traceUntil: string | null; usageSummary?: { eventsIn24h: number; healthy: boolean; }; }; + StartDeploymentResponse: + | { + /** @example dep_a1b2c3d4 */ + deploymentId: string; + /** @example k7m2x9p4q1w8 */ + slug: string; + target: string | null; + /** + * @example web + * @enum {string} + */ + type: 'web' | 'server'; + /** @enum {string} */ + status: 'deploying'; + settingsId?: string; + versionId: string; + versionNumber: number; + } + | { + deploymentId: string; + /** @enum {string} */ + status: 'already_created'; + }; + DeploymentStreamStatusEvent: { + status: string; + substatus: string | null; + /** + * @example web + * @enum {string} + */ + type: 'web' | 'server'; + target: string | null; + containerUrl: string | null; + errorCode: string | null; + errorMessage: string | null; + /** Format: date-time */ + createdAt: string; + /** Format: date-time */ + updatedAt: string; + }; ListDeploymentsResponse: { deployments: components['schemas']['DeploymentSummary'][]; total: number; @@ -7831,6 +8462,37 @@ export interface components { deploymentVersionId: string; }; }; + ObserveSessionResponse: { + /** @example ses_abc123xyz456 */ + id: string; + projectId: string; + /** @example flow_a1b2c3d4 */ + flowId: string; + status: string; + errorMessage: string | null; + configSnapshot: { + [key: string]: unknown; + }; + serverFlowName: string | null; + serverEndpoint: string | null; + web: components['schemas']['ObserveSessionWeb']; + createdBy: string; + /** Format: date-time */ + createdAt: string; + }; + ObserveSessionWeb: { + token: string; + activationUrl: string; + /** Format: uri */ + bundleUrl: string; + } | null; + CreateObserveSessionRequest: { + settingsName: string; + }; + ObserveSessionHeartbeatResponse: { + /** @enum {boolean} */ + ok: true; + }; SecretName: string; CreateSecretRequest: { name: string; @@ -8269,14 +8931,6 @@ export interface components { RotateIngestTokenResponse: { ingestToken: string; }; - TraceDeploymentRequest: { - minutes?: number; - }; - TraceDeploymentResponse: { - /** Format: date-time */ - traceUntil: string | null; - minutes: number; - }; DeploymentUsageResponse: { totalEventsIn: number; totalEventsOut: number; @@ -8290,6 +8944,7 @@ export interface components { averageThroughputPerHour: number; period: string; buckets: components['schemas']['UsageBucket'][]; + destinations?: components['schemas']['UsageDestination'][]; }; UsageBucket: { /** Format: date-time */ @@ -8299,6 +8954,14 @@ export interface components { eventsFailed: number; instances: number; }; + UsageDestination: { + name: string; + count: number; + failed: number; + duration: number; + dlqSize: number; + dropped: number; + }; CreateCustomDomainRequest: { hostname: string; deploymentId?: string; @@ -8661,12 +9324,6 @@ export interface components { /** Format: date-time */ updatedAt: string; } | null; - StartDeploymentResponse: { - deploymentId: string; - /** @enum {string} */ - type: 'web' | 'server'; - status: string; - }; ListSettingsResponse: { settings: components['schemas']['FlowSettingsSummary'][]; }; @@ -8845,9 +9502,26 @@ export interface components { count: number; failed: number; duration: number; + dlqSize?: number; + dropped?: number; }; }; }; + recentErrors?: { + message: string; + count: number; + /** Format: date-time */ + firstSeen: string; + /** Format: date-time */ + lastSeen: string; + }[]; + recentLogs?: { + /** Format: date-time */ + time: string; + /** @enum {string} */ + level: 'error' | 'warn' | 'info' | 'debug'; + message: string; + }[]; }; }; responses: never; diff --git a/packages/collector/src/__tests__/destination-timeout.test.ts b/packages/collector/src/__tests__/destination-timeout.test.ts new file mode 100644 index 000000000..25c2c74c3 --- /dev/null +++ b/packages/collector/src/__tests__/destination-timeout.test.ts @@ -0,0 +1,217 @@ +import type { Destination } from '@walkeros/core'; +import { createEvent } from '@walkeros/core'; +import { pushToDestinations, startFlow } from '..'; + +/** + * Per-destination delivery timeout: a single slow/hanging destination must + * not wedge the whole collector push. A push that never settles within the + * configured (or default) timeout is converted into a bounded, counted + * failure routed to the SAME DLQ a thrown push uses, and other destinations + * keep delivering (error isolation preserved). + */ +describe('Destination delivery timeout', () => { + function makeDestination( + push: Destination.Instance['push'], + config: Destination.Config = {}, + ): Destination.Instance { + return { push, config }; + } + + // A push that never settles, typed to the destination push contract. + const neverResolves: Destination.Instance['push'] = () => + new Promise(() => undefined); + + function errorMessage(value: unknown): string { + return value instanceof Error ? value.message : String(value); + } + + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.runOnlyPendingTimers(); + jest.useRealTimers(); + }); + + test('hanging push times out, routes to DLQ, push resolves (not hung)', async () => { + const event = createEvent(); + const { collector } = await startFlow({ + destinations: { + slow: { code: makeDestination(neverResolves, { timeout: 5_000 }) }, + }, + }); + + const resultPromise = pushToDestinations( + collector, + event, + {}, + collector.destinations, + ); + + // Advance past the timeout to trigger the race rejection. + await jest.advanceTimersByTimeAsync(5_000); + + const result = await resultPromise; + const dest = collector.destinations['slow']; + + expect(result.ok).toBeFalsy(); + expect(result.failed).toBeDefined(); + expect(Object.keys(result.failed!)).toHaveLength(1); + expect(dest.dlq).toBeDefined(); + expect(dest.dlq).toHaveLength(1); + // DLQ pair carries the original event and a timeout-cause error. + expect(dest.dlq![0][0]).toEqual(event); + const dlqError = dest.dlq![0][1]; + expect(dlqError).toBeInstanceOf(Error); + if (dlqError instanceof Error) { + // Discriminable name so DLQ entries are distinguishable without substring match. + expect(dlqError.name).toBe('DestinationTimeoutError'); + expect(dlqError.message).toContain('timed out'); + expect(dlqError.message).toContain('5000'); + } + }); + + test('isolation: a hang on one destination does not block a healthy one', async () => { + const event = createEvent(); + const healthyPush = jest.fn(async () => undefined); + + const { collector } = await startFlow({ + destinations: { + slow: { code: makeDestination(neverResolves, { timeout: 5_000 }) }, + healthy: { code: makeDestination(healthyPush, { timeout: 5_000 }) }, + }, + }); + + const resultPromise = pushToDestinations( + collector, + event, + {}, + collector.destinations, + ); + + await jest.advanceTimersByTimeAsync(5_000); + const result = await resultPromise; + + // Healthy destination delivered exactly once, unaffected by the hang. + expect(healthyPush).toHaveBeenCalledTimes(1); + // The slow one failed to DLQ. + expect(collector.destinations['slow'].dlq).toHaveLength(1); + expect(collector.destinations['healthy'].dlq ?? []).toHaveLength(0); + // Overall result reflects one failure but did not hang. + expect(result.failed).toBeDefined(); + expect(Object.keys(result.failed!)).toHaveLength(1); + }); + + test('a push that settles before the timeout is unaffected (no false timeout)', async () => { + const event = createEvent(); + const fastPush = jest.fn(async () => 'ok'); + + const { collector } = await startFlow({ + destinations: { + fast: { code: makeDestination(fastPush, { timeout: 5_000 }) }, + }, + }); + + // Baseline timers held by collector infra (e.g. cache-store sweep). + const baselineTimers = jest.getTimerCount(); + + const result = await pushToDestinations( + collector, + event, + {}, + collector.destinations, + ); + + expect(fastPush).toHaveBeenCalledTimes(1); + expect(result.ok).toBeTruthy(); + expect(collector.destinations['fast'].dlq ?? []).toHaveLength(0); + // The healthy delivery is counted as delivered. + expect(collector.status.out).toBe(1); + + // The race added no net timer: its cleared on settle. + expect(jest.getTimerCount()).toBe(baselineTimers); + }); + + test('default timeout applies when none configured', async () => { + const event = createEvent(); + const { collector } = await startFlow({ + destinations: { + slow: { code: makeDestination(neverResolves) }, + }, + }); + + const resultPromise = pushToDestinations( + collector, + event, + {}, + collector.destinations, + ); + + // Just before default (10000ms): still pending, no DLQ yet. + await jest.advanceTimersByTimeAsync(9_999); + expect(collector.destinations['slow'].dlq ?? []).toHaveLength(0); + + // Cross the default boundary: now it times out. + await jest.advanceTimersByTimeAsync(1); + const result = await resultPromise; + const dest = collector.destinations['slow']; + + expect(dest.dlq).toHaveLength(1); + expect(errorMessage(dest.dlq![0][1])).toContain('10000'); + expect(Object.keys(result.failed!)).toHaveLength(1); + }); + + test('config timeout overrides the default', async () => { + const event = createEvent(); + const { collector } = await startFlow({ + destinations: { + slow: { code: makeDestination(neverResolves, { timeout: 2_000 }) }, + }, + }); + + const resultPromise = pushToDestinations( + collector, + event, + {}, + collector.destinations, + ); + + await jest.advanceTimersByTimeAsync(2_000); + const result = await resultPromise; + const dest = collector.destinations['slow']; + + expect(dest.dlq).toHaveLength(1); + expect(errorMessage(dest.dlq![0][1])).toContain('2000'); + expect(Object.keys(result.failed!)).toHaveLength(1); + }); + + test('regression: a thrown push still routes to DLQ (isolation unchanged)', async () => { + const event = createEvent(); + const throwingPush: Destination.Instance['push'] = () => { + throw new Error('kaputt'); + }; + + const { collector } = await startFlow({ + destinations: { + boom: { code: makeDestination(throwingPush, { timeout: 5_000 }) }, + }, + }); + + const baselineTimers = jest.getTimerCount(); + + const result = await pushToDestinations( + collector, + event, + {}, + collector.destinations, + ); + const dest = collector.destinations['boom']; + + expect(result.failed).toBeDefined(); + expect(Object.keys(result.failed!)).toHaveLength(1); + expect(dest.dlq).toContainEqual([event, new Error('kaputt')]); + // The race must not leave a timer behind on the synchronous-throw path. + expect(jest.getTimerCount()).toBe(baselineTimers); + }); +}); diff --git a/packages/collector/src/__tests__/destination.consent-init-gate.test.ts b/packages/collector/src/__tests__/destination.consent-init-gate.test.ts new file mode 100644 index 000000000..8f2930e7f --- /dev/null +++ b/packages/collector/src/__tests__/destination.consent-init-gate.test.ts @@ -0,0 +1,182 @@ +import type { On } from '@walkeros/core'; +import { startFlow } from '../flow'; +import { destinationInit } from '../destination'; + +/** + * Consent is the sole gate (Option A): a destination must perform NO observable + * action (init, on-delivery, push) until its required consent is granted. These + * tests pin that invariant across every path that can reach `destinationInit`, + * including the `queueOn`-only path that a denied consent command drives via the + * eventless `pushToDestinations` after every state command. + * + * A destination that declares an `on` handler is the trigger: a consent command + * is buffered to its `queueOn` while it is not yet initialized, and the eventless + * `pushToDestinations` would otherwise init it to flush that buffer. + */ +function gatedDestination(spies: { + init: jest.Mock; + push?: jest.Mock; + on?: jest.Mock; +}) { + return { + code: { + type: 'gtag', + config: { consent: { marketing: true } }, + init: spies.init, + push: spies.push ?? jest.fn(), + on: spies.on ?? jest.fn(), + }, + }; +} + +describe('destination consent init gate (sole-gate invariant)', () => { + test('denied consent never initializes a consent-gated destination', async () => { + const init = jest.fn(); + const { collector } = await startFlow({ + run: true, + destinations: { gtag: gatedDestination({ init }) }, + }); + + await collector.command('consent', { marketing: false }); + + expect(init).not.toHaveBeenCalled(); + expect(collector.destinations.gtag.config.init).toBeFalsy(); + }); + + test('grant self-heals: denied then granted initializes exactly once', async () => { + const init = jest.fn(); + const { collector } = await startFlow({ + run: true, + destinations: { gtag: gatedDestination({ init }) }, + }); + + await collector.command('consent', { marketing: false }); + expect(init).not.toHaveBeenCalled(); + + await collector.command('consent', { marketing: true }); + expect(init).toHaveBeenCalledTimes(1); + + // Idempotent: a repeat grant must not re-init. + await collector.command('consent', { marketing: true }); + expect(init).toHaveBeenCalledTimes(1); + }); + + test('revocation after grant does not re-init and pushes nothing', async () => { + const init = jest.fn(); + const push = jest.fn(); + const on = jest.fn(); + const { collector } = await startFlow({ + run: true, + destinations: { gtag: gatedDestination({ init, push, on }) }, + }); + + await collector.command('consent', { marketing: true }); + expect(init).toHaveBeenCalledTimes(1); + on.mockClear(); + + await collector.command('consent', { marketing: false }); + + expect(init).toHaveBeenCalledTimes(1); // no re-init + expect(on).toHaveBeenCalledWith('consent', expect.any(Object)); // revoke delivered to live dest + expect(push).not.toHaveBeenCalled(); // no event pushed + }); + + test('event-level consent override initializes under collector-denied consent', async () => { + const init = jest.fn(); + const push = jest.fn(); + const { elb } = await startFlow({ + run: true, + consent: { marketing: false }, + destinations: { gtag: gatedDestination({ init, push }) }, + }); + + // The event carries its own granted consent; per-event override is valid. + await elb({ name: 'page view', data: {}, consent: { marketing: true } }); + + expect(init).toHaveBeenCalledTimes(1); + expect(push).toHaveBeenCalledTimes(1); + }); + + test('destination without a consent requirement is unaffected', async () => { + const init = jest.fn(); + const push = jest.fn(); + const { elb } = await startFlow({ + run: true, + consent: { marketing: false }, + destinations: { + plain: { code: { type: 'plain', config: {}, init, push } }, + }, + }); + + await elb({ name: 'page view', data: {} }); + + expect(init).toHaveBeenCalledTimes(1); + expect(push).toHaveBeenCalledTimes(1); + }); + + test('eventless run barrier does not init a denied consent-gated destination', async () => { + const init = jest.fn(); + const { collector } = await startFlow({ + run: false, + destinations: { gtag: gatedDestination({ init }) }, + }); + + await collector.command('consent', { marketing: false }); + await collector.command('run'); + + expect(init).not.toHaveBeenCalled(); + }); + + test('destinationInit is fail-closed: refuses a consent-gated destination without an allow token', async () => { + const init = jest.fn(); + const { collector } = await startFlow({ + run: true, + destinations: { gtag: gatedDestination({ init }) }, + }); + const destination = collector.destinations.gtag; + + // Calling the chokepoint WITHOUT an affirmative allow decision must not init. + const result = await destinationInit(collector, destination, 'gtag'); + + expect(result).toBe(false); + expect(init).not.toHaveBeenCalled(); + expect(destination.config.init).toBeFalsy(); + }); + + test('on() deliveries stay buffered under denial and flush on grant', async () => { + const init = jest.fn(); + const on = jest.fn(); + const { collector } = await startFlow({ + run: true, + destinations: { gtag: gatedDestination({ init, on }) }, + }); + + await collector.command('consent', { marketing: false }); + // The "receive an on" half of the invariant: under denial the handler must + // not be invoked; the consent delivery stays buffered in queueOn. + expect(on).not.toHaveBeenCalled(); + + await collector.command('consent', { marketing: true }); + expect(init).toHaveBeenCalledTimes(1); + // On grant the destination inits and the buffered deliveries flush to it. + expect(on).toHaveBeenCalledWith('consent', expect.any(Object)); + }); + + test('addDestination activation does not init a consent-gated destination under denial', async () => { + const init = jest.fn(); + const { collector } = await startFlow({ run: true }); + + await collector.command('consent', { marketing: false }); + await collector.command('destination', { + code: { + type: 'gtag', + config: { consent: { marketing: true } }, + init, + push: jest.fn(), + on: jest.fn(), + }, + }); + + expect(init).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/collector/src/destination.ts b/packages/collector/src/destination.ts index 543bff2d0..5757e39dd 100644 --- a/packages/collector/src/destination.ts +++ b/packages/collector/src/destination.ts @@ -48,6 +48,56 @@ const DEFAULT_DLQ_MAX = 100; const DEFAULT_BATCH_SIZE = 1_000; /** Default upper-bound on batch age in ms. Forces flush even if debounce keeps resetting. */ const DEFAULT_BATCH_AGE = 30_000; +/** + * Default per-destination delivery timeout in ms. Applied when a destination's + * `config.timeout` is `0` or undefined. A hung delivery is converted into a + * counted DLQ failure after this window so one slow destination never wedges + * the collector push. + */ +const DEFAULT_DESTINATION_TIMEOUT_MS = 10_000; + +/** + * Resolve the effective delivery timeout for a destination. A positive number + * wins; `0` or undefined falls back to {@link DEFAULT_DESTINATION_TIMEOUT_MS}. + */ +function resolveDestinationTimeout(timeout?: number): number { + return typeof timeout === 'number' && timeout > 0 + ? timeout + : DEFAULT_DESTINATION_TIMEOUT_MS; +} + +/** + * Error thrown when a destination delivery does not settle within its timeout. + * The dedicated `name` lets DLQ consumers discriminate a timeout from a + * destination-thrown error without substring matching the message. + */ +class DestinationTimeoutError extends Error { + constructor(message: string) { + super(message); + this.name = 'DestinationTimeoutError'; + } +} + +/** + * Races a delivery promise against a per-destination timeout. If the work does + * not settle within `ms`, the returned promise rejects with a + * {@link DestinationTimeoutError}; the timer is always cleared on settle so no + * dangling timer remains. The race is constructed per call site, so each + * destination times out independently and one hang never affects another. + */ +function withTimeout( + work: Promise, + ms: number, + message: string, +): Promise { + let timer: ReturnType | undefined; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => reject(new DestinationTimeoutError(message)), ms); + }); + return Promise.race([work, timeout]).finally(() => { + if (timer) clearTimeout(timer); + }); +} /** * Sentinel returned by {@link destinationPush} when an event was enqueued @@ -370,9 +420,24 @@ export async function pushToDestinations( // the queueOn-only path. Mirrors the same pattern used below at the main // init call site so failures are never silent on either branch. if (!currentQueue.length && destination.queueOn?.length) { + // Consent gate (sole-gate invariant): this branch would init the + // destination purely to flush queued on() events, but there is no push + // event whose individual consent could apply, so collector consent is + // the complete basis. Never init a consent-gated destination while its + // required consent is denied. Self-heals: handle.ts runs + // pushToDestinations after every state command, so the grant command + // re-enters here with consent satisfied and inits then. + if (!getGrantedConsent(destination.config.consent, consent)) { + return { id, destination, skipped: true }; + } let isInitialized = false; try { - isInitialized = await destinationInit(collector, destination, id); + isInitialized = await destinationInit( + collector, + destination, + id, + true, + ); } catch (err) { collector.status.failed++; const destType = destination.type || 'unknown'; @@ -466,9 +531,11 @@ export async function pushToDestinations( // silently returned undefined when init threw, hiding real failures. The // destination itself may also log a more specific error before throwing; // this is the boundary safety net so failures are never silent. + // Allowed: at least one event cleared the per-event consent gate above, + // so initialization is authorized (true) for the chokepoint guard. let isInitialized = false; try { - isInitialized = await destinationInit(collector, destination, id); + isInitialized = await destinationInit(collector, destination, id, true); } catch (err) { collector.status.failed++; const destType = destination.type || 'unknown'; @@ -824,13 +891,39 @@ export async function pushToDestinations( * @param destId - The destination ID. * @returns Whether the destination was initialized successfully. */ +/** + * A destination declares a consent requirement when its config.consent has at + * least one key. Such a destination must never be initialized without a cleared + * consent gate (see destinationInit's `allowed` parameter). + */ +function hasConsentRequirement(destination: Destination.Instance): boolean { + const required = destination.config.consent; + return !!required && Object.keys(required).length > 0; +} + export async function destinationInit( collector: Collector.Instance, destination: Destination, destId: string, + // Fail-closed consent gate. Callers MUST pass an affirmative allow decision + // (per-event on the push path, collector-consent on the queueOn path). The + // default is false so any future call site that forgets fails closed: a + // consent-gated destination is never initialized without a cleared gate. + allowed = false, ): Promise { // Check if the destination was initialized properly or try to do so if (destination.init && !destination.config.init) { + // Defense-in-depth: refuse to init a destination that declares a consent + // requirement unless the gate was cleared. We do NOT re-derive consent from + // collector state here, because the push path's decision may rest on an + // event's individual consent this function cannot see; re-deriving would + // wrongly block a legitimate event-level override. + if (!allowed && hasConsentRequirement(destination)) { + collector.logger + .scope(destination.type || 'unknown') + .debug('init blocked: consent gate not cleared'); + return false; + } // Create scoped logger for this destination: [type:id] or [unknown:id] const destType = destination.type || 'unknown'; const destLogger = collector.logger.scope(destType); @@ -1092,13 +1185,24 @@ export async function destinationPush( // partial outcome = total minus the entries reported failed. let succeededCount = snapshot.entries.length; + const batchTimeoutMs = resolveDestinationTimeout(config.timeout); const outcome = await tryCatchAsync( - useHooks( - destination.pushBatch!, - 'DestinationPushBatch', - collector.hooks, - collector.logger, - ), + ( + batchArg: Destination.Batch, + ctxArg: Destination.PushBatchContext, + ) => + withTimeout( + Promise.resolve( + useHooks( + destination.pushBatch!, + 'DestinationPushBatch', + collector.hooks, + collector.logger, + )(batchArg, ctxArg), + ), + batchTimeoutMs, + `Destination "${destId}" batch delivery timed out after ${batchTimeoutMs}ms`, + ), (err) => { succeededCount = 0; const errFinished = Date.now(); @@ -1239,13 +1343,23 @@ export async function destinationPush( emitStep(collector, inState); try { - // It's time to go to the destination's side now - const response = await useHooks( - destination.push, - 'DestinationPush', - collector.hooks, - collector.logger, - )(processed.event, context); + // It's time to go to the destination's side now. Race the push against a + // per-destination timeout so a hung delivery becomes a thrown failure + // that flows into the SAME catch -> tryCatchAsync onError -> DLQ path a + // real throw uses. The race is per call, so it never affects siblings. + const timeoutMs = resolveDestinationTimeout(config.timeout); + const response = await withTimeout( + Promise.resolve( + useHooks( + destination.push, + 'DestinationPush', + collector.hooks, + collector.logger, + )(processed.event, context), + ), + timeoutMs, + `Destination "${destId}" delivery timed out after ${timeoutMs}ms`, + ); const pushFinished = Date.now(); const outState = buildBaseState(collector, { diff --git a/packages/core/src/__tests__/cache-types.test-d.ts b/packages/core/src/__tests__/cache-types.test-d.ts index 87553edfe..19a9c192f 100644 --- a/packages/core/src/__tests__/cache-types.test-d.ts +++ b/packages/core/src/__tests__/cache-types.test-d.ts @@ -10,6 +10,7 @@ */ import type { CacheResult } from '../cache'; import { checkCache } from '../cache'; +import type { Cache, EventCacheRule, StoreCacheRule } from '../types/cache'; type Equal = (() => T extends X ? 1 : 2) extends () => T extends Y ? 1 : 2 @@ -24,3 +25,24 @@ type _CheckCacheReturnsPromise = Expect< >; void (null as unknown as _CheckCacheReturnsPromise); + +// The discriminated cache-rule union must accept each rule shape and reject +// fields that belong only to the other variant. `update` is event-only; a +// StoreCacheRule carrying it must fail the build. +const _eventOk: Cache = { + // `update` is event-only; assert EventCacheRule accepts it so this file also + // fails the build if the field is ever dropped from EventCacheRule. + rules: [{ key: ['event.id'], ttl: 60, update: { foo: 'bar' } }], +}; +void _eventOk; + +const _storeOk: Cache = { + rules: [{ ttl: 60 }], +}; +void _storeOk; + +const _storeBad: Cache = { + // @ts-expect-error -- update is not allowed in StoreCacheRule + rules: [{ ttl: 60, update: { foo: 'bar' } }], +}; +void _storeBad; diff --git a/packages/core/src/__tests__/cache-types.test.ts b/packages/core/src/__tests__/cache-types.test.ts deleted file mode 100644 index 2c53d66db..000000000 --- a/packages/core/src/__tests__/cache-types.test.ts +++ /dev/null @@ -1,19 +0,0 @@ -import type { Cache, EventCacheRule, StoreCacheRule } from '../types/cache'; - -// Compile-time assertions: these should fail TS until the union exists. -const _eventOk: Cache = { - rules: [{ key: ['event.id'], ttl: 60 }], -}; - -const _storeOk: Cache = { - rules: [{ ttl: 60 }], -}; - -const _storeBad: Cache = { - // @ts-expect-error -- update is not allowed in StoreCacheRule - rules: [{ ttl: 60, update: { foo: 'bar' } }], -}; - -it('compiles', () => { - expect(true).toBe(true); -}); diff --git a/packages/core/src/__tests__/types/contract.test.ts b/packages/core/src/__tests__/types/contract.test.ts deleted file mode 100644 index a82f3e8a3..000000000 --- a/packages/core/src/__tests__/types/contract.test.ts +++ /dev/null @@ -1,74 +0,0 @@ -import type { Flow } from '../../types'; - -describe('Contract types', () => { - it('should accept a valid named contract at root config level', () => { - const setup: Flow.Json = { - version: 4, - contract: { - default: { - description: 'Base contract', - schema: { - type: 'object', - properties: { - globals: { - type: 'object', - required: ['country'], - properties: { - country: { type: 'string' }, - }, - }, - }, - }, - events: { - product: { - '*': { - properties: { - data: { - type: 'object', - required: ['id'], - properties: { - id: { type: 'string', description: 'Product SKU' }, - }, - }, - }, - }, - }, - }, - }, - web: { - extend: 'default', - schema: { - type: 'object', - properties: { - consent: { - type: 'object', - required: ['analytics'], - }, - }, - }, - }, - }, - flows: { - default: { config: { platform: 'web' } }, - }, - }; - expect(setup.contract).toBeDefined(); - }); - - it('should accept contract with extend chain', () => { - const contract: Flow.Contract = { - default: { description: 'base' }, - web: { extend: 'default', events: { product: { view: {} } } }, - web_loggedin: { - extend: 'web', - schema: { - type: 'object', - properties: { - user: { type: 'object', required: ['id'] }, - }, - }, - }, - }; - expect(Object.keys(contract)).toHaveLength(3); - }); -}); diff --git a/packages/core/src/__tests__/types/observer.test.ts b/packages/core/src/__tests__/types/observer.test.ts deleted file mode 100644 index ccb89e2f1..000000000 --- a/packages/core/src/__tests__/types/observer.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Type-only test for ObserverFn. Compiles-or-fails-at-typecheck. - */ -import type { Collector, FlowState, ObserverFn } from '../..'; - -describe('ObserverFn type', () => { - test('accepts a FlowState consumer', () => { - const fn: ObserverFn = (state: FlowState) => { - // touching a known field proves the structural import works - void state.flowId; - }; - expect(typeof fn).toBe('function'); - }); -}); - -test('Collector.Instance carries an observers Set', () => { - const observers = new Set(); - // If observers is missing from Instance, this assignment fails typecheck. - const surface: Pick = { observers }; - expect(surface.observers.size).toBe(0); -}); diff --git a/packages/core/src/__tests__/types/store.test.ts b/packages/core/src/__tests__/types/store.test.ts deleted file mode 100644 index a0291442b..000000000 --- a/packages/core/src/__tests__/types/store.test.ts +++ /dev/null @@ -1,136 +0,0 @@ -import type { Store } from '../../types'; - -describe('Store types', () => { - describe('Types bundle', () => { - it('defaults are assignable', () => { - const types: Store.Types = { - settings: undefined, - initSettings: undefined, - env: {}, - setup: undefined, - credentials: undefined, - }; - expect(types).toBeDefined(); - }); - - it('accepts custom generics', () => { - interface MySettings { - host: string; - port: number; - } - interface MyEnv extends Store.BaseEnv { - redis: unknown; - } - interface MyInit { - host?: string; - port?: number; - } - - const types: Store.Types = { - settings: { host: 'localhost', port: 6379 }, - initSettings: { host: 'localhost' }, - env: { redis: {} }, - setup: undefined, - credentials: undefined, - }; - expect(types.settings.host).toBe('localhost'); - }); - }); - - describe('Type extractors', () => { - it('extracts Settings from Types', () => { - type T = Store.Types<{ host: string }>; - type S = Store.Settings; - const s: S = { host: 'localhost' }; - expect(s.host).toBe('localhost'); - }); - - it('extracts Env from Types', () => { - type T = Store.Types; - type E = Store.Env; - const e: E = { custom: true }; - expect(e.custom).toBe(true); - }); - }); - - describe('Instance', () => { - it('has get, set, delete (no push)', () => { - const instance: Store.Instance = { - type: 'memory', - config: {}, - get: (key: string) => undefined, - set: (key: string, value: unknown, ttl?: number) => {}, - delete: (key: string) => {}, - }; - expect(instance.type).toBe('memory'); - expect('push' in instance).toBe(false); - }); - - it('allows async get/set/delete', () => { - const instance: Store.Instance = { - type: 'redis', - config: {}, - get: async (key: string) => undefined, - set: async (key: string, value: unknown) => {}, - delete: async (key: string) => {}, - }; - expect(instance.type).toBe('redis'); - }); - - it('allows optional destroy', () => { - const instance: Store.Instance = { - type: 'redis', - config: {}, - get: (key: string) => undefined, - set: (key: string, value: unknown) => {}, - delete: (key: string) => {}, - destroy: async ({ config }) => {}, - }; - expect(instance.destroy).toBeDefined(); - }); - }); - - describe('GetFn / SetFn / DeleteFn', () => { - it('GetFn supports sync and async return', () => { - const syncGet: Store.GetFn = (key) => 'value'; - const asyncGet: Store.GetFn = async (key) => 'value'; - expect(typeof syncGet).toBe('function'); - expect(typeof asyncGet).toBe('function'); - }); - - it('SetFn accepts ttl parameter', () => { - const set: Store.SetFn = (key, value, ttl) => {}; - expect(typeof set).toBe('function'); - }); - }); - - describe('InitStores / Stores', () => { - it('InitStores maps IDs to InitStore', () => { - const stores: Store.InitStores = { - cache: { - code: async (context) => ({ - type: 'memory', - config: {}, - get: (key) => undefined, - set: (key, value) => {}, - delete: (key) => {}, - }), - }, - }; - expect(stores['cache']).toBeDefined(); - }); - - it('Stores maps IDs to Instance', () => { - const stores: Store.Stores = { - cache: { - type: 'memory', - config: {}, - get: (key) => undefined, - set: (key, value) => {}, - delete: (key) => {}, - }, - }; - expect(stores['cache'].type).toBe('memory'); - }); - }); -}); diff --git a/packages/core/src/__tests__/types/telemetry.test.ts b/packages/core/src/__tests__/types/telemetry.test.ts deleted file mode 100644 index d62651156..000000000 --- a/packages/core/src/__tests__/types/telemetry.test.ts +++ /dev/null @@ -1,86 +0,0 @@ -import type { - FlowState, - FlowStatePhase, - FlowStepType, -} from '../../types/telemetry'; - -describe('FlowState types', () => { - test('compiles a literal with every optional field', () => { - const state: FlowState = { - flowId: 'default', - platform: 'web', - stepId: 'destination.gtag', - stepType: 'destination', - phase: 'out', - eventId: '1234567890abcdef', - timestamp: '2026-05-26T10:00:00.000Z', - elapsedMs: 42, - durationMs: 5, - inEvent: { name: 'page view' }, - outEvent: { event: 'page_view' }, - error: { name: 'Error', message: 'boom' }, - mappingKey: 'page view', - contractRule: 'page view', - consent: { marketing: true }, - consentApplied: { marketing: true }, - branchId: '0123456789abcdef', - batch: { size: 10, index: 3 }, - skipReason: 'consent', - meta: { extra: 'value' }, - }; - expect(state.flowId).toBe('default'); - expect(state.platform).toBe('web'); - expect(state.stepType).toBe('destination'); - }); - - test('FlowStatePhase narrows to known phases only', () => { - // Exhaustive list, accepted by the compiler. - const phases: FlowStatePhase[] = [ - 'init', - 'in', - 'out', - 'error', - 'skip', - 'flush', - ]; - expect(phases).toHaveLength(6); - - // Type-level narrowing assertion: assigning the union to a tagged tuple - // of literal types proves the union has not widened to `string`. - const isValidPhase = (value: FlowStatePhase): boolean => { - switch (value) { - case 'init': - case 'in': - case 'out': - case 'error': - case 'skip': - case 'flush': - return true; - } - }; - expect(isValidPhase('init')).toBe(true); - }); - - test('FlowStepType narrows to known step kinds only', () => { - const kinds: FlowStepType[] = [ - 'source', - 'transformer', - 'collector', - 'destination', - 'store', - ]; - expect(kinds).toHaveLength(5); - - const isValidKind = (value: FlowStepType): boolean => { - switch (value) { - case 'source': - case 'transformer': - case 'collector': - case 'destination': - case 'store': - return true; - } - }; - expect(isValidKind('source')).toBe(true); - }); -}); diff --git a/packages/core/src/schemas/destination.ts b/packages/core/src/schemas/destination.ts index b8aa8c7f1..da049cab9 100644 --- a/packages/core/src/schemas/destination.ts +++ b/packages/core/src/schemas/destination.ts @@ -170,6 +170,12 @@ export const ConfigSchema = z .describe( 'Maximum failed-push entries retained in dlq for this destination. FIFO drop on overflow. Default 100.', ), + timeout: z + .number() + .optional() + .describe( + 'Per-destination delivery timeout in ms (default 10000); a delivery that does not settle in this window is routed to the DLQ like a thrown push.', + ), batch: z .union([ z.number(), diff --git a/packages/core/src/types/destination.ts b/packages/core/src/types/destination.ts index 52ca5b510..ca948d9ee 100644 --- a/packages/core/src/types/destination.ts +++ b/packages/core/src/types/destination.ts @@ -137,6 +137,11 @@ export interface Config { state?: import('./state').State | import('./state').State[]; /** Completely skip this destination — no init, no push, no queuing. */ disabled?: boolean; + /** + * Per-destination delivery timeout in ms (default 10000); a delivery that + * does not settle within this window is routed to the DLQ like a thrown push. + */ + timeout?: number; /** Return this value instead of calling push(). Uses !== undefined check to support falsy values. */ mock?: unknown; /** diff --git a/packages/core/src/types/simulation.ts b/packages/core/src/types/simulation.ts index fd30f67c6..6135117d8 100644 --- a/packages/core/src/types/simulation.ts +++ b/packages/core/src/types/simulation.ts @@ -34,6 +34,15 @@ export interface Result { calls: Call[]; /** Execution time in ms */ duration: number; + /** + * Entity-action key of the matched mapping rule, e.g. "product add" or + * "product *". Populated for destination simulations when a mapping rule + * matched; absent when no rule matched or the step has no mapping. + * Also absent when the matched rule uses batched delivery (batching + * reports no per-event key) and on error results, where the key is not + * available. + */ + mappingKey?: string; /** Error if the step threw */ error?: Error; } diff --git a/packages/server/destinations/amplitude/src/__tests__/stepExamples.test.ts b/packages/server/destinations/amplitude/src/__tests__/stepExamples.test.ts index e4068e015..0335af8fe 100644 --- a/packages/server/destinations/amplitude/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/amplitude/src/__tests__/stepExamples.test.ts @@ -6,6 +6,7 @@ import type { WalkerOS, Mapping as WalkerOSMapping } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { clone } from '@walkeros/core'; import { examples } from '../dev'; +import type { AmplitudeStepExample } from '../examples/step'; import type { Env, IdentifyInstance, @@ -166,57 +167,55 @@ function spyEnv(env: Env): { env: Env; collected: () => CallRecord[] } { } describe('amplitude server destination -- step examples', () => { - it.each(Object.entries(examples.step))('%s', async (name, rawExample) => { - const example = rawExample as { - in?: unknown; - mapping?: unknown; - out?: unknown; - command?: 'consent' | 'user' | 'config' | 'run'; - settings?: Partial; - configInclude?: string[]; - }; + it.each<[string, AmplitudeStepExample]>(Object.entries(examples.step))( + '%s', + async (name, example) => { + const env = clone(examples.env.push) as Env; + const { env: spiedEnv, collected } = spyEnv(env); - const env = clone(examples.env.push) as Env; - const { env: spiedEnv, collected } = spyEnv(env); + const dest = jest.requireActual('../').default; + const { elb } = await startFlow(); - const dest = jest.requireActual('../').default; - const { elb } = await startFlow(); + const baseSettings: Partial & { apiKey: string } = { + apiKey: 'test-project', + ...(example.settings || {}), + }; - const baseSettings: Partial & { apiKey: string } = { - apiKey: 'test-project', - ...(example.settings || {}), - }; + if (example.command === 'consent') { + await elb('walker destination', { + code: { ...dest, env: spiedEnv }, + config: { + consent: { analytics: true }, + include: example.configInclude, + settings: baseSettings, + }, + }); + // Grant first when the example declares it, so the gated destination is + // loaded before the consent under test (it never loads under denial). + // Both the grant and the consent-under-test effects are asserted. + if (example.before) await elb('walker consent', example.before); + await elb('walker consent', example.in as WalkerOS.Consent); + } else { + const event = example.in as WalkerOS.Event; + const mapping = example.mapping as WalkerOSMapping.Rule | undefined; + const mappingConfig = mapping + ? { [event.entity]: { [event.action]: mapping } } + : undefined; - if (example.command === 'consent') { - await elb('walker destination', { - code: { ...dest, env: spiedEnv }, - config: { - consent: { analytics: true }, - include: example.configInclude, - settings: baseSettings, - }, - }); - await elb('walker consent', example.in as WalkerOS.Consent); - } else { - const event = example.in as WalkerOS.Event; - const mapping = example.mapping as WalkerOSMapping.Rule | undefined; - const mappingConfig = mapping - ? { [event.entity]: { [event.action]: mapping } } - : undefined; + await elb('walker destination', { + code: { ...dest, env: spiedEnv }, + config: { + include: example.configInclude, + settings: baseSettings, + mapping: mappingConfig, + }, + }); + await elb(event); + } - await elb('walker destination', { - code: { ...dest, env: spiedEnv }, - config: { - include: example.configInclude, - settings: baseSettings, - mapping: mappingConfig, - }, - }); - await elb(event); - } + const actual = collected().filter(([path]) => path !== 'amplitude.init'); - const actual = collected().filter(([path]) => path !== 'amplitude.init'); - - expect(actual).toEqual(example.out); - }); + expect(actual).toEqual(example.out); + }, + ); }); diff --git a/packages/server/destinations/amplitude/src/examples/step.ts b/packages/server/destinations/amplitude/src/examples/step.ts index 772d9f28d..1452e679d 100644 --- a/packages/server/destinations/amplitude/src/examples/step.ts +++ b/packages/server/destinations/amplitude/src/examples/step.ts @@ -10,6 +10,8 @@ import type { Settings } from '../types'; export type AmplitudeStepExample = Flow.StepExample & { settings?: Partial; configInclude?: string[]; + /** Consent granted before `in` so a gated destination is loaded first. */ + before?: WalkerOS.Consent; }; /** @@ -389,9 +391,10 @@ export const eventOptionsTimeInsertId: AmplitudeStepExample = { }; /** - * Consent revoked -> amplitude.setOptOut(true). The destination checks - * the consent keys declared in config.consent and toggles optOut - * accordingly (strict: all required keys must be granted). + * Consent revoked -> amplitude.setOptOut(true). After analytics consent is + * granted (the destination loads), revoking it toggles optOut via the + * on('consent') handler. The destination is never loaded under denied consent, + * so the opt-out is a real revocation of an already-granted destination. * * Uses the canonical StepExample.command='consent' pattern: the test * runner dispatches via elb('walker consent', in) instead of pushing @@ -400,11 +403,15 @@ export const eventOptionsTimeInsertId: AmplitudeStepExample = { export const consentRevokeOptOut: AmplitudeStepExample = { title: 'Consent revoked', description: - 'A walker consent command with analytics denied opts out of Amplitude tracking via setOptOut(true).', + 'After analytics consent is granted (Amplitude loads), revoking it opts out of tracking via setOptOut(true).', command: 'consent', + before: { analytics: true }, in: { analytics: false } as WalkerOS.Consent, settings: {} as Partial, - out: [['amplitude.setOptOut', true]], + out: [ + ['amplitude.setOptOut', false], + ['amplitude.setOptOut', true], + ], }; /** diff --git a/packages/server/destinations/gcp/src/bigquery/__mocks__/@google-cloud/bigquery-storage.ts b/packages/server/destinations/gcp/src/bigquery/__mocks__/@google-cloud/bigquery-storage.ts index 85c78f648..6fb70eeb4 100644 --- a/packages/server/destinations/gcp/src/bigquery/__mocks__/@google-cloud/bigquery-storage.ts +++ b/packages/server/destinations/gcp/src/bigquery/__mocks__/@google-cloud/bigquery-storage.ts @@ -8,6 +8,7 @@ interface MockRowError { let nextAppendRowErrors: MockRowError[] | null = null; let nextAppendThrow: unknown = null; +let nextGetResultReject: unknown = null; let nextCreateStreamConnectionError: unknown = null; class MockJSONWriter { @@ -23,12 +24,19 @@ class MockJSONWriter { } const queuedErrors = nextAppendRowErrors; nextAppendRowErrors = null; + const queuedReject = nextGetResultReject; + nextGetResultReject = null; return { - getResult: async () => ({ - appendResult: { offset: { value: '0' } }, - rowErrors: queuedErrors ?? [], - writeStream: 'projects/p/datasets/d/tables/t/streams/_default', - }), + getResult: async () => { + // Models a gRPC deadline-exceeded (or any stream error): the append is + // accepted but getResult() rejects when the stream's deadline fires. + if (queuedReject !== null) throw queuedReject; + return { + appendResult: { offset: { value: '0' } }, + rowErrors: queuedErrors ?? [], + writeStream: 'projects/p/datasets/d/tables/t/streams/_default', + }; + }, }; } close(): void { @@ -45,10 +53,13 @@ class MockWriterClient { }) { calls.push({ method: 'WriterClient.ctor', args: [args] }); } - async createStreamConnection(args: { - destinationTable: string; - streamId: string; - }) { + async createStreamConnection( + args: { + destinationTable: string; + streamId: string; + }, + options?: unknown, + ) { // Match the real SDK: callers must pass streamId=DefaultStream (not streamType) // for default-stream use. Passing streamType triggers a CreateWriteStream // call with type='DEFAULT', which BQ rejects as TYPE_UNSPECIFIED. @@ -57,7 +68,9 @@ class MockWriterClient { `mock createStreamConnection: expected streamId='DEFAULT', got ${JSON.stringify(args)}`, ); } - calls.push({ method: 'createStreamConnection', args: [args] }); + // args[1] captures the gax CallOptions (e.g. { timeout }) so tests can + // assert the request deadline is forwarded to the appendRows stream. + calls.push({ method: 'createStreamConnection', args: [args, options] }); const queuedError = nextCreateStreamConnectionError; if (queuedError !== null) { nextCreateStreamConnectionError = null; @@ -67,8 +80,13 @@ class MockWriterClient { getStreamId: () => `${args.destinationTable}/streams/_default`, }; } - async getWriteStream(args: { streamId: string; view?: number }) { - calls.push({ method: 'getWriteStream', args: [args] }); + async getWriteStream( + args: { streamId: string; view?: number }, + options?: unknown, + ) { + // args[1] captures the gax CallOptions so tests can assert the deadline + // is forwarded to the unary schema fetch. + calls.push({ method: 'getWriteStream', args: [args, options] }); // Minimal tableSchema shape consumed by adapt.convertStorageSchemaToProto2Descriptor return { tableSchema: { fields: [] }, @@ -122,6 +140,7 @@ function __resetMockCalls() { calls.length = 0; nextAppendRowErrors = null; nextAppendThrow = null; + nextGetResultReject = null; nextCreateStreamConnectionError = null; } @@ -141,6 +160,15 @@ function __setNextAppendThrow(err: unknown): void { nextAppendThrow = err; } +/** + * Test-only: make the next appendRows().getResult() reject, simulating a gRPC + * deadline-exceeded (or any stream error) surfacing through the pending write. + * Auto-resets after one use. + */ +function __setNextGetResultReject(err: unknown): void { + nextGetResultReject = err; +} + /** * Test-only: queue an error for the next createStreamConnection() call so * tests can simulate openWriter failures (NotFound, INVALID_ARGUMENT, etc.). @@ -158,5 +186,6 @@ export { __resetMockCalls, __setNextAppendRowErrors, __setNextAppendThrow, + __setNextGetResultReject, __setNextOpenWriterError, }; diff --git a/packages/server/destinations/gcp/src/bigquery/__tests__/bigquery-storage-mock.d.ts b/packages/server/destinations/gcp/src/bigquery/__tests__/bigquery-storage-mock.d.ts index cd35b9fa5..efa2e05b4 100644 --- a/packages/server/destinations/gcp/src/bigquery/__tests__/bigquery-storage-mock.d.ts +++ b/packages/server/destinations/gcp/src/bigquery/__tests__/bigquery-storage-mock.d.ts @@ -5,6 +5,7 @@ declare module '@google-cloud/bigquery-storage' { errors: Array<{ index: number; code: number; message: string }>, ): void; export function __setNextAppendThrow(err: unknown): void; + export function __setNextGetResultReject(err: unknown): void; export function __setNextOpenWriterError(err: unknown): void; } diff --git a/packages/server/destinations/gcp/src/bigquery/__tests__/index.test.ts b/packages/server/destinations/gcp/src/bigquery/__tests__/index.test.ts index bb9e42915..b80ff2b3f 100644 --- a/packages/server/destinations/gcp/src/bigquery/__tests__/index.test.ts +++ b/packages/server/destinations/gcp/src/bigquery/__tests__/index.test.ts @@ -12,6 +12,7 @@ import { __resetMockCalls, __setNextAppendRowErrors, __setNextAppendThrow, + __setNextGetResultReject, __setNextOpenWriterError, } from '@google-cloud/bigquery-storage'; import { @@ -41,10 +42,17 @@ describe('Server Destination BigQuery', () => { const mockCollector = {} as Collector.Instance; let testEnv: Env; - async function callInit(initSettings: InitSettings, logger?: MockLogger) { + async function callInit( + initSettings: InitSettings, + logger?: MockLogger, + timeout?: number, + ) { if (!destination.init) throw new Error('destination.init undefined'); return destination.init({ - config: { settings: initSettings }, + config: + timeout === undefined + ? { settings: initSettings } + : { settings: initSettings, timeout }, collector: mockCollector, env: testEnv, logger: logger ?? createMockLogger(), @@ -155,6 +163,43 @@ describe('Server Destination BigQuery', () => { expect(errorContext.error).toBe('TYPE_UNSPECIFIED: bad write stream type'); }); + test('init logs and throws an informative, DLQ-routable error on a non-NotFound failure', async () => { + // The destination no longer scrubs error messages itself: secret redaction + // is standardized at the CLI logger handler (covers stderr + heartbeat ring) + // and the thrown error is scrubbed on output there. Here the destination's + // job is to log the failure and rethrow the RAW error so it stays + // informative and DLQ-routable, keeping its routing `code`. + const underlyingError: Error & { code?: number } = Object.assign( + new Error('streams/_default" contains illegal characters'), + { code: 3 }, // INVALID_ARGUMENT — not NotFound, so it hits the catch-all + ); + __setNextOpenWriterError(underlyingError); + + const logger = createMockLogger(); + + let thrown: unknown; + try { + await callInit({ projectId, datasetId, tableId }, logger); + } catch (err) { + thrown = err; + } + + // The thrown error is the raw error: informative and DLQ-routable, with its + // routing code intact (read without a cast). + expect(thrown).toBe(underlyingError); + expect(thrown).toBeInstanceOf(Error); + if (!(thrown instanceof Error)) return; + expect(thrown.message).toContain('contains illegal characters'); + expect('code' in thrown).toBe(true); + if (!('code' in thrown)) return; + const withCode: { code?: unknown } = thrown; + expect(withCode.code).toBe(3); + + // Still informative for the operator. + const serialized = JSON.stringify(logger.error.mock.calls); + expect(serialized).toContain('BigQuery init failed'); + }); + test('push appends one row through JSONWriter', async () => { const logger = createMockLogger(); const { writer, writeClient } = await openWriter( @@ -480,4 +525,203 @@ describe('Server Destination BigQuery', () => { ).rejects.toThrow('writer is missing'); }); }); + + describe('gRPC deadline (config.timeout)', () => { + // The Storage Write API appendRows runs on the long-lived bidi stream opened + // by createStreamConnection. The deadline is therefore applied at the + // stream-connection level (gax CallOptions.timeout), which governs every + // appendRows/getResult on that stream, and at the unary getWriteStream call. + // The deadline derives from the standard per-step config.timeout, the same + // value the collector uses to race the push, not a destination-custom knob. + + test('init forwards the standard config.timeout as the gax deadline on the appendRows stream and schema fetch', async () => { + await callInit({ projectId, datasetId, tableId }, undefined, 5000); + + const streamCall = __getMockCalls().find( + (c) => c.method === 'createStreamConnection', + ); + expect(streamCall?.args[1]).toEqual({ timeout: 5000 }); + + const schemaCall = __getMockCalls().find( + (c) => c.method === 'getWriteStream', + ); + expect(schemaCall?.args[1]).toEqual({ timeout: 5000 }); + }); + + test('init applies the default deadline when config.timeout is unset', async () => { + await callInit({ projectId, datasetId, tableId }); + + const streamCall = __getMockCalls().find( + (c) => c.method === 'createStreamConnection', + ); + expect(streamCall?.args[1]).toEqual({ timeout: 10000 }); + + const schemaCall = __getMockCalls().find( + (c) => c.method === 'getWriteStream', + ); + expect(schemaCall?.args[1]).toEqual({ timeout: 10000 }); + }); + + test('init treats config.timeout: 0 as "use the default" (no zero-ms deadline)', async () => { + // A zero-ms deadline would expire immediately; 0 is not a "disabled" + // sentinel. Resolution falls back to the default so writer.ts always gets + // a positive deadline, mirroring the collector's resolveDestinationTimeout. + await callInit({ projectId, datasetId, tableId }, undefined, 0); + + const streamCall = __getMockCalls().find( + (c) => c.method === 'createStreamConnection', + ); + expect(streamCall?.args[1]).toEqual({ timeout: 10000 }); + + const schemaCall = __getMockCalls().find( + (c) => c.method === 'getWriteStream', + ); + expect(schemaCall?.args[1]).toEqual({ timeout: 10000 }); + }); + + test('a deadline-exceeded getResult surfaces as a rejection from push (so the collector can DLQ it)', async () => { + const logger = createMockLogger(); + const { writer, writeClient } = await openWriter( + { projectId, datasetId, tableId, timeout: 1000 }, + logger, + ); + __resetMockCalls(); + + const settings: Settings = { + client: new BigQuery({ projectId }), + projectId, + datasetId, + tableId, + location: 'EU', + writer, + writeClient, + }; + const config: Config = { settings }; + + const deadlineError: Error & { code?: number } = Object.assign( + new Error('Deadline exceeded'), + { code: 4 }, // gRPC DEADLINE_EXCEEDED + ); + __setNextGetResultReject(deadlineError); + + await expect( + destination.push( + event, + createMockContext({ + config, + rule: undefined, + data: undefined, + env: testEnv, + id: 'test-bq', + }), + ), + ).rejects.toThrow('Deadline exceeded'); + }); + + test('a getResult rejection surfaces the raw, informative, DLQ-routable error from push', async () => { + // Single-event path (batching off): a getResult() rejection must surface + // to the collector/DLQ as the RAW error, keeping its message and routing + // `code`. Secret redaction is standardized at the CLI logger handler, so + // the destination no longer scrubs here. + const logger = createMockLogger(); + const { writer, writeClient } = await openWriter( + { projectId, datasetId, tableId }, + logger, + ); + __resetMockCalls(); + + const settings: Settings = { + client: new BigQuery({ projectId }), + projectId, + datasetId, + tableId, + location: 'EU', + writer, + writeClient, + }; + const config: Config = { settings }; + + const appendError: Error & { code?: number } = Object.assign( + new Error('streams/_default" contains illegal characters'), + { code: 3 }, + ); + __setNextGetResultReject(appendError); + + let thrown: unknown; + try { + await destination.push( + event, + createMockContext({ + config, + rule: undefined, + data: undefined, + env: testEnv, + logger, + id: 'test-bq', + }), + ); + } catch (err) { + thrown = err; + } + + // The raw error surfaces: an Error, informative, with the routing code + // intact (read without a cast). + expect(thrown).toBe(appendError); + expect(thrown).toBeInstanceOf(Error); + if (!(thrown instanceof Error)) return; + expect(thrown.message).toContain('contains illegal characters'); + expect('code' in thrown).toBe(true); + if (!('code' in thrown)) return; + const withCode: { code?: unknown } = thrown; + expect(withCode.code).toBe(3); + + // The destination still logs the failure for the operator. + const serialized = JSON.stringify(logger.error.mock.calls); + expect(serialized).toContain('BigQuery row append threw'); + }); + + test('a deadline-exceeded getResult rejects pushBatch (whole batch DLQ)', async () => { + if (!destination.pushBatch) throw new Error('pushBatch missing'); + + const logger = createMockLogger(); + const { writer, writeClient } = await openWriter( + { projectId, datasetId, tableId, timeout: 1000 }, + logger, + ); + __resetMockCalls(); + + const settings: Settings = { + client: new BigQuery({ projectId }), + projectId, + datasetId, + tableId, + location: 'EU', + writer, + writeClient, + }; + const config: Config = { settings }; + + const deadlineError: Error & { code?: number } = Object.assign( + new Error('Deadline exceeded'), + { code: 4 }, + ); + __setNextGetResultReject(deadlineError); + + const events = [createEvent(), createEvent()]; + const data: Array = events.map(() => undefined); + const entries = events.map((e) => ({ event: e })); + + await expect( + destination.pushBatch!( + { key: 'k', events, data, entries }, + createMockContext({ + config, + env: testEnv, + logger, + id: 'test-bq', + }), + ), + ).rejects.toThrow('Deadline exceeded'); + }); + }); }); diff --git a/packages/server/destinations/gcp/src/bigquery/__tests__/writer.test.ts b/packages/server/destinations/gcp/src/bigquery/__tests__/writer.test.ts index 41fae218a..3c695f89d 100644 --- a/packages/server/destinations/gcp/src/bigquery/__tests__/writer.test.ts +++ b/packages/server/destinations/gcp/src/bigquery/__tests__/writer.test.ts @@ -55,6 +55,31 @@ describe('openWriter', () => { }); }); + test('forwards timeout as the gax deadline to createStreamConnection and getWriteStream', async () => { + const logger = createMockLogger(); + await openWriter( + { projectId: 'p', datasetId: 'd', tableId: 't', timeout: 7500 }, + logger, + ); + const streamCall = __getMockCalls().find( + (c) => c.method === 'createStreamConnection', + ); + expect(streamCall?.args[1]).toEqual({ timeout: 7500 }); + const schemaCall = __getMockCalls().find( + (c) => c.method === 'getWriteStream', + ); + expect(schemaCall?.args[1]).toEqual({ timeout: 7500 }); + }); + + test('omits the deadline when timeout is unset', async () => { + const logger = createMockLogger(); + await openWriter({ projectId: 'p', datasetId: 'd', tableId: 't' }, logger); + const streamCall = __getMockCalls().find( + (c) => c.method === 'createStreamConnection', + ); + expect(streamCall?.args[1]).toBeUndefined(); + }); + test('closeWriter calls close on writer and writeClient', async () => { const logger = createMockLogger(); const { writer, writeClient } = await openWriter( diff --git a/packages/server/destinations/gcp/src/bigquery/index.ts b/packages/server/destinations/gcp/src/bigquery/index.ts index 634aa11f5..03463a822 100644 --- a/packages/server/destinations/gcp/src/bigquery/index.ts +++ b/packages/server/destinations/gcp/src/bigquery/index.ts @@ -8,6 +8,12 @@ import { openWriter, closeWriter } from './writer'; // Types export * as DestinationBigQuery from './types'; +// Default gRPC deadline (ms) when the standard per-step `config.timeout` is +// unset or <= 0. Mirrors the collector's DEFAULT_DESTINATION_TIMEOUT_MS and its +// `> 0 ? value : default` rule (packages/collector/src/destination.ts), so the +// gax deadline matches the window the collector uses to race the push. +const DEFAULT_TIMEOUT_MS = 10_000; + export const destinationBigQuery: Destination = { type: 'gcp-bigquery', @@ -18,6 +24,14 @@ export const destinationBigQuery: Destination = { async init({ config: partialConfig, env, logger, id }) { const config = getConfig(partialConfig, env, logger); + // The gax deadline derives from the standard per-step config.timeout (the + // same value the collector uses to race the push), not a destination-custom + // knob. A positive number wins; 0/unset falls back to the default. + const timeout = + config.timeout && config.timeout > 0 + ? config.timeout + : DEFAULT_TIMEOUT_MS; + // Open the long-lived JSONWriter on the _default stream. // Hard-fail when the dataset/table is missing. try { @@ -27,12 +41,19 @@ export const destinationBigQuery: Destination = { datasetId: config.settings.datasetId, tableId: config.settings.tableId, bigquery: config.settings.bigquery, + timeout, }, logger, ); config.settings.writer = writer; config.settings.writeClient = writeClient; } catch (err) { + // Log the failure and rethrow the raw error. Secret redaction is + // standardized at the CLI logger handler, which scrubs every line before + // both stderr and the heartbeat ring, so the destination logs the message + // as-is. The raw error keeps its `code` naturally, so it stays + // NotFound-classifiable and DLQ-routable. + const message = err instanceof Error ? err.message : String(err); if (isNotFound(err)) { const target = `${config.settings.datasetId}.${config.settings.tableId}`; const project = config.settings.projectId; @@ -42,7 +63,7 @@ export const destinationBigQuery: Destination = { { project, target, - error: err instanceof Error ? err.message : String(err), + error: message, }, ); } else { @@ -51,7 +72,7 @@ export const destinationBigQuery: Destination = { // resolve stream, build proto descriptor), so it logs here before // re-throwing. logger.error('BigQuery init failed', { - error: err instanceof Error ? err.message : String(err), + error: message, }); } throw err; diff --git a/packages/server/destinations/gcp/src/bigquery/push.ts b/packages/server/destinations/gcp/src/bigquery/push.ts index feff28585..0efce57f0 100644 --- a/packages/server/destinations/gcp/src/bigquery/push.ts +++ b/packages/server/destinations/gcp/src/bigquery/push.ts @@ -23,8 +23,19 @@ export const push: PushFn = async function ( rowCount: rows.length, }); - const pending = writer.appendRows(rows); - const result = await pending.getResult(); + let result; + try { + const pending = writer.appendRows(rows); + result = await pending.getResult(); + } catch (err) { + // Log the failure and rethrow the raw error to the collector/DLQ. Secret + // redaction is standardized at the CLI logger handler; the raw error keeps + // its `code` so the row stays DLQ-routable. + logger.error('BigQuery row append threw', { + error: err instanceof Error ? err.message : String(err), + }); + throw err; + } if (result.rowErrors && result.rowErrors.length > 0) { // Single-event push path: throw with row context so the caller sees the failure. diff --git a/packages/server/destinations/gcp/src/bigquery/pushBatch.ts b/packages/server/destinations/gcp/src/bigquery/pushBatch.ts index f1dc2e10a..1831e0801 100644 --- a/packages/server/destinations/gcp/src/bigquery/pushBatch.ts +++ b/packages/server/destinations/gcp/src/bigquery/pushBatch.ts @@ -76,6 +76,8 @@ export const pushBatch: PushBatchFn = async (batch, { config, logger }) => { offset: result.appendResult?.offset?.value, }); } catch (err) { + // Log the failure and rethrow the raw error so the whole batch routes to the + // DLQ. Secret redaction is standardized at the CLI logger handler. logger.error('BigQuery batch append threw', { error: err instanceof Error ? err.message : String(err), }); diff --git a/packages/server/destinations/gcp/src/bigquery/writer.ts b/packages/server/destinations/gcp/src/bigquery/writer.ts index 73b65a187..7abbfeeee 100644 --- a/packages/server/destinations/gcp/src/bigquery/writer.ts +++ b/packages/server/destinations/gcp/src/bigquery/writer.ts @@ -2,6 +2,13 @@ import type { Logger } from '@walkeros/core'; import type { BigQueryOptions } from '@google-cloud/bigquery'; import { managedwriter, adapt, protos } from '@google-cloud/bigquery-storage'; +// gax CallOptions, derived from the SDK's own method signature so we don't take +// a direct dependency on `google-gax` (a transitive dep of bigquery-storage). +// `getWriteStream(request, options?: CallOptions)` -> the optional 2nd param. +type CallOptions = NonNullable< + Parameters[1] +>; + export interface OpenWriterArgs { projectId: string; datasetId: string; @@ -9,6 +16,14 @@ export interface OpenWriterArgs { // Auth forwarded from settings.bigquery so the data-plane WriterClient // authenticates like the control plane instead of falling back to ADC. bigquery?: BigQueryOptions; + /** + * gRPC deadline in milliseconds, derived from the standard per-step + * `config.timeout`. Applied as the gax `CallOptions.timeout` on the appendRows + * bidi stream (via createStreamConnection) and the unary getWriteStream schema + * fetch, so a hanging call is cancelled by gRPC and rejects instead of running + * detached. + */ + timeout?: number; } export interface WriterHandles { @@ -31,13 +46,21 @@ export async function openWriter( args: OpenWriterArgs, logger: Logger.Instance, ): Promise { - const { projectId, datasetId, tableId, bigquery } = args; + const { projectId, datasetId, tableId, bigquery, timeout } = args; const destinationTable = `projects/${projectId}/datasets/${datasetId}/tables/${tableId}`; logger.debug('Opening BigQuery Storage Write API writer', { destinationTable, }); + // gax call options carrying the per-request deadline. The StreamConnection + // stores these and applies them to the underlying appendRows bidi stream, so + // every appendRows/getResult on this writer inherits the deadline. When the + // deadline fires, gRPC cancels the call and getResult() rejects (no detached + // promise). Left undefined when no timeout is configured. + const callOptions: CallOptions | undefined = + timeout === undefined ? undefined : { timeout }; + const writeClient = new managedwriter.WriterClient({ projectId, ...bigquery, @@ -47,15 +70,21 @@ export async function openWriter( // implicit `_default` stream without calling CreateWriteStream. Passing // managedwriter.DefaultStream as streamType triggers a CreateWriteStream // call with type='DEFAULT', which BQ rejects as TYPE_UNSPECIFIED. - const connection = await writeClient.createStreamConnection({ - destinationTable, - streamId: managedwriter.DefaultStream, - }); + const connection = await writeClient.createStreamConnection( + { + destinationTable, + streamId: managedwriter.DefaultStream, + }, + callOptions, + ); const streamId = connection.getStreamId(); - const writeStream = await writeClient.getWriteStream({ - streamId, - view: protos.google.cloud.bigquery.storage.v1.WriteStreamView.FULL, - }); + const writeStream = await writeClient.getWriteStream( + { + streamId, + view: protos.google.cloud.bigquery.storage.v1.WriteStreamView.FULL, + }, + callOptions, + ); if (!writeStream.tableSchema) { throw new Error( `BigQuery write stream ${streamId} returned no tableSchema; cannot build proto descriptor`, diff --git a/packages/server/destinations/posthog/src/__tests__/stepExamples.test.ts b/packages/server/destinations/posthog/src/__tests__/stepExamples.test.ts index 34d798fbe..3aaf86d89 100644 --- a/packages/server/destinations/posthog/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/posthog/src/__tests__/stepExamples.test.ts @@ -7,6 +7,7 @@ jest.mock('posthog-node', () => ({ import type { WalkerOS, Mapping as WalkerOSMapping } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { examples } from '../dev'; +import type { PostHogStepExample } from '../examples/step'; import type { Env, Settings } from '../types'; type CallRecord = [string, ...unknown[]]; @@ -63,54 +64,52 @@ function spyEnv(): { env: Env; collected: () => CallRecord[] } { } describe('posthog server destination — step examples', () => { - it.each(Object.entries(examples.step))('%s', async (_name, rawExample) => { - const example = rawExample as { - in?: unknown; - mapping?: unknown; - out?: unknown; - command?: 'consent' | 'user' | 'config' | 'run'; - settings?: Partial; - configInclude?: string[]; - }; + it.each<[string, PostHogStepExample]>(Object.entries(examples.step))( + '%s', + async (name, example) => { + const { env, collected } = spyEnv(); - const { env, collected } = spyEnv(); + const dest = jest.requireActual('../').default; + const { elb } = await startFlow(); - const dest = jest.requireActual('../').default; - const { elb } = await startFlow(); + const baseSettings: Partial & { apiKey: string } = { + apiKey: 'phc_test', + ...(example.settings || {}), + }; - const baseSettings: Partial & { apiKey: string } = { - apiKey: 'phc_test', - ...(example.settings || {}), - }; + if (example.command === 'consent') { + await elb('walker destination', { + code: { ...dest, env }, + config: { + consent: { analytics: true }, + include: example.configInclude, + settings: baseSettings, + }, + }); + // Grant first when the example declares it, so the gated destination is + // loaded before the consent under test (it never loads under denial). + // Both the grant and the consent-under-test effects are asserted. + if (example.before) await elb('walker consent', example.before); + await elb('walker consent', example.in as WalkerOS.Consent); + } else { + const event = example.in as WalkerOS.Event; + const mapping = example.mapping as WalkerOSMapping.Rule | undefined; + const mappingConfig = mapping + ? { [event.entity]: { [event.action]: mapping } } + : undefined; - if (example.command === 'consent') { - await elb('walker destination', { - code: { ...dest, env }, - config: { - consent: { analytics: true }, - include: example.configInclude, - settings: baseSettings, - }, - }); - await elb('walker consent', example.in as WalkerOS.Consent); - } else { - const event = example.in as WalkerOS.Event; - const mapping = example.mapping as WalkerOSMapping.Rule | undefined; - const mappingConfig = mapping - ? { [event.entity]: { [event.action]: mapping } } - : undefined; - - await elb('walker destination', { - code: { ...dest, env }, - config: { - include: example.configInclude, - settings: baseSettings, - mapping: mappingConfig, - }, - }); - await elb(event); - } + await elb('walker destination', { + code: { ...dest, env }, + config: { + include: example.configInclude, + settings: baseSettings, + mapping: mappingConfig, + }, + }); + await elb(event); + } - expect(collected()).toEqual(example.out); - }); + expect(collected()).toEqual(example.out); + }, + ); }); diff --git a/packages/server/destinations/posthog/src/examples/step.ts b/packages/server/destinations/posthog/src/examples/step.ts index 75821ce06..a90b0f227 100644 --- a/packages/server/destinations/posthog/src/examples/step.ts +++ b/packages/server/destinations/posthog/src/examples/step.ts @@ -14,6 +14,8 @@ import type { Settings } from '../types'; export type PostHogStepExample = Flow.StepExample & { settings?: Partial; configInclude?: string[]; + /** Consent granted before `in` so a gated destination is loaded first. */ + before?: WalkerOS.Consent; }; /** @@ -205,16 +207,20 @@ export const captureWithGroupContext: PostHogStepExample = { }; /** - * Consent revoked - client.disable() is called. + * Consent revoked - after analytics consent is granted (PostHog loads and is + * enabled), revoking it calls client.disable() via the on('consent') handler. + * The destination is never loaded under denied consent, so the disable is a + * real revocation of an already-granted destination. */ export const consentRevoke: PostHogStepExample = { title: 'Consent revoked', description: 'A walker consent command with analytics denied calls client.disable on the PostHog client.', command: 'consent', + before: { analytics: true }, in: { analytics: false } as WalkerOS.Consent, settings: {} as Partial, - out: [['client.disable']], + out: [['client.enable'], ['client.disable']], }; /** diff --git a/packages/server/sources/express/src/__tests__/cache-roundtrip.test.ts b/packages/server/sources/express/src/__tests__/cache-roundtrip.test.ts index df81b88b7..43e406679 100644 --- a/packages/server/sources/express/src/__tests__/cache-roundtrip.test.ts +++ b/packages/server/sources/express/src/__tests__/cache-roundtrip.test.ts @@ -56,8 +56,12 @@ describe('Express source cache round-trip', () => { express: { code: sourceExpress, // No port: drive the handler in-process via the source's `push`. + // `async: false` makes the GET handler await the push so the + // cache/asset destination can respond with real bytes before the + // GIF fallback applies (the default respond-first pixel mode would + // win the race with the GIF). config: { - settings: { paths: ['/walker.js'] }, + settings: { paths: ['/walker.js'], async: false }, ingest: { map: { method: { key: 'method' }, @@ -195,7 +199,7 @@ describe('Express source cache round-trip', () => { express: { code: sourceExpress, config: { - settings: { paths: ['/walker.js'] }, + settings: { paths: ['/walker.js'], async: false }, ingest: { map: { method: { key: 'method' }, @@ -324,7 +328,7 @@ describe('Express source cache round-trip', () => { express: { code: sourceExpress, config: { - settings: { paths: ['/asset'] }, + settings: { paths: ['/asset'], async: false }, ingest: { map: { method: { key: 'method' }, path: { key: 'url' } }, }, diff --git a/packages/server/sources/express/src/__tests__/index.test.ts b/packages/server/sources/express/src/__tests__/index.test.ts index 5adb48b1c..26993a790 100644 --- a/packages/server/sources/express/src/__tests__/index.test.ts +++ b/packages/server/sources/express/src/__tests__/index.test.ts @@ -120,6 +120,7 @@ describe('sourceExpress', () => { expect(source.config.settings).toEqual({ paths: ['/collect'], cors: true, + async: true, }); expect(typeof source.push).toBe('function'); expect(source.app).toBeDefined(); @@ -147,6 +148,7 @@ describe('sourceExpress', () => { expect(source.config.settings).toEqual({ paths: ['/events'], cors: false, + async: true, }); }); @@ -384,14 +386,14 @@ describe('sourceExpress', () => { }); }); - it('should handle collector errors', async () => { + it('should handle collector errors when async is disabled', async () => { const errorPush = jest .fn() .mockRejectedValue(new Error('Collector error')); const source = await sourceExpress( createSourceContext( - {}, + { settings: { async: false } }, { push: errorPush as never, command: mockCommand as never, @@ -711,4 +713,196 @@ describe('sourceExpress', () => { expect(source.config.settings?.paths).toEqual(['/events']); }); }); + + describe('respond-first async ack', () => { + // A push that returns a promise we can resolve/reject on demand, to prove + // the HTTP response is produced without waiting for delivery to complete. + function createDeferredPush(): { + push: jest.MockedFunction<(...args: unknown[]) => Promise>; + resolve: (value?: unknown) => void; + reject: (reason?: unknown) => void; + } { + let resolve!: (value?: unknown) => void; + let reject!: (reason?: unknown) => void; + const pending = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + const push = jest.fn(() => pending) as jest.MockedFunction< + (...args: unknown[]) => Promise + >; + return { push, resolve, reject }; + } + + it('GET returns the GIF even if push never resolves', async () => { + const { push } = createDeferredPush(); + const source = await sourceExpress( + createSourceContext( + {}, + { + push: push as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: createMockLogger(), + }, + ), + ); + + const req = createMockRequest({ + method: 'GET', + url: '/collect.gif?event=page%20view', + }); + const res = createMockResponse(); + + // Resolves immediately despite the never-resolving push. + await source.push(req, res); + + expect(res.responseHeaders?.['Content-Type']).toBe('image/gif'); + expect(Buffer.isBuffer(res.responseBody)).toBe(true); + expect(push).toHaveBeenCalled(); + // Push is still pending — we did not block on it. + }); + + it('GET logs a rejected push and does not throw out of the handler', async () => { + const { push, reject } = createDeferredPush(); + const logger = createMockLogger(); + const source = await sourceExpress( + createSourceContext( + {}, + { + push: push as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger, + }, + ), + ); + + const req = createMockRequest({ + method: 'GET', + url: '/collect.gif?event=page%20view', + }); + const res = createMockResponse(); + + await source.push(req, res); + expect(res.responseHeaders?.['Content-Type']).toBe('image/gif'); + + const error = new Error('delivery failed'); + reject(error); + // Flush microtasks so the .catch handler runs. + await Promise.resolve(); + await Promise.resolve(); + + expect(logger.error).toHaveBeenCalledWith(error); + }); + + it('POST async (default) responds before push resolves', async () => { + const { push, resolve } = createDeferredPush(); + const source = await sourceExpress( + createSourceContext( + {}, + { + push: push as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: createMockLogger(), + }, + ), + ); + + const req = createMockRequest({ + method: 'POST', + body: { event: 'page view' }, + }); + const res = createMockResponse(); + + await source.push(req, res); + + // Responded already, before the push promise settles. + expect(res.statusCode).toBe(200); + expect(res.responseBody).toMatchObject({ success: true }); + expect(push).toHaveBeenCalledWith({ event: 'page view' }); + + resolve(); + }); + + it('POST async (default) catches a rejected push without changing the 2xx response', async () => { + const { push, reject } = createDeferredPush(); + const logger = createMockLogger(); + const source = await sourceExpress( + createSourceContext( + {}, + { + push: push as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger, + }, + ), + ); + + const req = createMockRequest({ + method: 'POST', + body: { event: 'page view' }, + }); + const res = createMockResponse(); + + await source.push(req, res); + + expect(res.statusCode).toBe(200); + expect(res.responseBody).toMatchObject({ success: true }); + + const error = new Error('delivery failed'); + reject(error); + await Promise.resolve(); + await Promise.resolve(); + + // Rejection was caught/logged, not thrown, and response unchanged. + expect(logger.error).toHaveBeenCalledWith(error); + expect(res.statusCode).toBe(200); + expect(res.responseBody).toMatchObject({ success: true }); + }); + + it('POST async:false awaits push, then responds (regression)', async () => { + const order: string[] = []; + const slowPush = jest.fn(async () => { + await Promise.resolve(); + order.push('push'); + return { ok: true }; + }); + const source = await sourceExpress( + createSourceContext( + { settings: { async: false } }, + { + push: slowPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: createMockLogger(), + }, + ), + ); + + const res = createMockResponse(); + const sendSpy = res.send as jest.Mock; + const jsonSpy = res.json as jest.Mock; + jsonSpy.mockImplementation((body: unknown) => { + order.push('respond'); + (res as { responseBody?: unknown }).responseBody = body; + return res; + }); + + const req = createMockRequest({ + method: 'POST', + body: { event: 'page view' }, + }); + + await source.push(req, res); + + // Push completed before the response was sent. + expect(order).toEqual(['push', 'respond']); + expect(res.statusCode).toBe(200); + expect(res.responseBody).toMatchObject({ success: true }); + expect(sendSpy).not.toHaveBeenCalled(); + }); + }); }); diff --git a/packages/server/sources/express/src/index.ts b/packages/server/sources/express/src/index.ts index c11dfbe2b..ab92a1634 100644 --- a/packages/server/sources/express/src/index.ts +++ b/packages/server/sources/express/src/index.ts @@ -5,6 +5,14 @@ import type { Source } from '@walkeros/core'; import type { ExpressSource, Types, EventRequest } from './types'; import { setCorsHeaders, TRANSPARENT_GIF } from './utils'; +/** + * Normalize an unknown rejection reason into an Error for the logger. + * A fire-and-forget push can reject with any value; the logger accepts + * `string | Error`, so non-Error reasons are wrapped. + */ +const toError = (value: unknown): Error => + value instanceof Error ? value : new Error(String(value)); + /** * Express source initialization * @@ -30,6 +38,8 @@ export const sourceExpress = async ( const settings = { ...userSettings, cors: userSettings.cors ?? true, + // Respond-first by default: a 2xx means "accepted", not "delivered". + async: userSettings.async ?? true, paths: userSettings.paths ?? (userSettings.path ? [userSettings.path] : ['/collect']), @@ -97,19 +107,38 @@ export const sourceExpress = async ( // Parse query parameters to event data using requestToData const parsedData = requestToData(req.url); - // Send to collector + // Default GIF body (idempotent fallback; skipped if a step already + // called respond, e.g. a cache/asset destination serving real bytes). + const respondGif = () => + respond({ + body: TRANSPARENT_GIF, + headers: { + 'Content-Type': 'image/gif', + 'Cache-Control': 'no-cache, no-store, must-revalidate', + }, + }); + if (parsedData && typeof parsedData === 'object') { - await env.push(parsedData); + if (settings.async) { + // Respond-first: the tracking pixel must return instantly and + // never block on backend delivery. Fire the push without + // awaiting; a rejected push is logged (destination errors are + // DLQ'd inside the collector). A 2xx means "accepted", not + // "delivered". + respondGif(); + env.push(parsedData).catch((err: unknown) => { + env.logger.error(toError(err)); + }); + } else { + // Synchronous: await the push so a step (e.g. a cache/asset + // destination) can respond with real content before the GIF + // fallback applies. + await env.push(parsedData); + respondGif(); + } + } else { + respondGif(); } - - // Default: 1x1 GIF (skipped if a step already called respond) - respond({ - body: TRANSPARENT_GIF, - headers: { - 'Content-Type': 'image/gif', - 'Cache-Control': 'no-cache, no-store, must-revalidate', - }, - }); return; } @@ -118,9 +147,19 @@ export const sourceExpress = async ( const eventData = req.body && typeof req.body === 'object' ? req.body : {}; - await env.push(eventData); - - respond({ body: { success: true, timestamp: Date.now() } }); + if (settings.async) { + // Respond-first ("accepted"), then deliver asynchronously. A + // rejected push is logged, not surfaced to the client and not left + // unhandled (destination errors are DLQ'd inside the collector). + respond({ body: { success: true, timestamp: Date.now() } }); + env.push(eventData).catch((err: unknown) => { + env.logger.error(toError(err)); + }); + } else { + // Synchronous ack: wait for delivery to settle before responding. + await env.push(eventData); + respond({ body: { success: true, timestamp: Date.now() } }); + } return; } diff --git a/packages/server/sources/express/src/schemas/settings.ts b/packages/server/sources/express/src/schemas/settings.ts index 7842a0d6f..c73747af3 100644 --- a/packages/server/sources/express/src/schemas/settings.ts +++ b/packages/server/sources/express/src/schemas/settings.ts @@ -32,6 +32,13 @@ export const SettingsSchema = z.object({ 'CORS configuration: false = disabled, true = allow all origins (default), object = custom configuration', ) .default(true), + + async: z + .boolean() + .describe( + 'Respond-first delivery (default true). When true, POST responds 2xx ("accepted") immediately and pushes to the collector without blocking the response; a rejected push is logged (destination errors are DLQ\'d inside the collector). When false, the response waits for the push to settle. The GET pixel always responds first regardless of this flag. A 2xx means accepted, not delivered.', + ) + .default(true), }); export type Settings = z.infer; diff --git a/packages/server/sources/express/src/types.ts b/packages/server/sources/express/src/types.ts index d76db9ac5..59a597b5f 100644 --- a/packages/server/sources/express/src/types.ts +++ b/packages/server/sources/express/src/types.ts @@ -27,6 +27,13 @@ export interface Mapping { } // Express-specific push type (uses Express Request/Response types) +// +// Ack contract: a 2xx response means the event was *accepted*, not that it was +// *delivered*. With `settings.async` (the default, `true`) the handler responds +// first and pushes to the collector without blocking the response; rejected +// pushes are logged and destination errors are DLQ'd inside the collector. The +// GET tracking pixel always responds first regardless of the flag. Set +// `settings.async: false` to make the response wait for delivery to settle. export type Push = (req: Request, res: Response) => Promise; export interface Env extends CoreSource.Env { diff --git a/packages/web/destinations/amplitude/src/__tests__/stepExamples.test.ts b/packages/web/destinations/amplitude/src/__tests__/stepExamples.test.ts index 5aefbea80..17b391743 100644 --- a/packages/web/destinations/amplitude/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/amplitude/src/__tests__/stepExamples.test.ts @@ -6,6 +6,7 @@ import type { WalkerOS, Mapping as WalkerOSMapping } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { clone } from '@walkeros/core'; import { examples } from '../dev'; +import type { AmplitudeStepExample } from '../examples/step'; import type { Env, IdentifyInstance, @@ -162,61 +163,61 @@ function spyEnv(env: Env): { env: Env; collected: () => CallRecord[] } { } describe('amplitude destination — step examples', () => { - it.each(Object.entries(examples.step))('%s', async (name, rawExample) => { - const example = rawExample as { - in?: unknown; - mapping?: unknown; - out?: ReadonlyArray; - command?: 'consent' | 'user' | 'config' | 'run'; - settings?: Partial; - configInclude?: string[]; - }; - - const env = clone(examples.env.push) as Env; - const { env: spiedEnv, collected } = spyEnv(env); + it.each<[string, AmplitudeStepExample]>(Object.entries(examples.step))( + '%s', + async (name, example) => { + const env = clone(examples.env.push) as Env; + const { env: spiedEnv, collected } = spyEnv(env); - const dest = jest.requireActual('../').default; - const { elb } = await startFlow(); + const dest = jest.requireActual('../').default; + const { elb } = await startFlow(); - const baseSettings: Partial & { apiKey: string } = { - apiKey: 'test-project', - ...(example.settings || {}), - }; + const baseSettings: Partial & { apiKey: string } = { + apiKey: 'test-project', + ...(example.settings || {}), + }; - if (example.command === 'consent') { - // Consent examples need config.consent declared so the destination's - // on() handler knows which walkerOS consent key to check. - await elb('walker destination', { - code: { ...dest, env: spiedEnv }, - config: { - consent: { analytics: true }, - include: example.configInclude, - settings: baseSettings, - }, - }); - await elb('walker consent', example.in as WalkerOS.Consent); - } else { - const event = example.in as WalkerOS.Event; - const mapping = example.mapping as WalkerOSMapping.Rule | undefined; - const mappingConfig = mapping - ? { [event.entity]: { [event.action]: mapping } } - : undefined; + if (example.command === 'consent') { + // Consent examples need config.consent declared so the destination's + // on() handler knows which walkerOS consent key to check. + await elb('walker destination', { + code: { ...dest, env: spiedEnv }, + config: { + consent: { analytics: true }, + include: example.configInclude, + settings: baseSettings, + }, + }); + // Grant first when the example declares it, so the gated destination is + // loaded before the consent under test (it never loads under denial). + // Both the grant and the consent-under-test effects are asserted. + if (example.before) await elb('walker consent', example.before); + await elb('walker consent', example.in as WalkerOS.Consent); + } else { + const event = example.in as WalkerOS.Event; + const mapping = example.mapping as WalkerOSMapping.Rule | undefined; + const mappingConfig = mapping + ? { [event.entity]: { [event.action]: mapping } } + : undefined; - await elb('walker destination', { - code: { ...dest, env: spiedEnv }, - config: { - include: example.configInclude, - settings: baseSettings, - mapping: mappingConfig, - }, - }); - await elb(event); - } + await elb('walker destination', { + code: { ...dest, env: spiedEnv }, + config: { + include: example.configInclude, + settings: baseSettings, + mapping: mappingConfig, + }, + }); + await elb(event); + } - // Drop init — every example triggers init once; it's not part of `out`. - const expected = (example.out ?? []) as ReadonlyArray; - const actual = collected().filter(([path]) => path !== 'amplitude.initAll'); + // Drop init — every example triggers init once; it's not part of `out`. + const expected = (example.out ?? []) as ReadonlyArray; + const actual = collected().filter( + ([path]) => path !== 'amplitude.initAll', + ); - expect(actual).toEqual(expected); - }); + expect(actual).toEqual(expected); + }, + ); }); diff --git a/packages/web/destinations/amplitude/src/examples/step.ts b/packages/web/destinations/amplitude/src/examples/step.ts index 5970f6480..e227a244c 100644 --- a/packages/web/destinations/amplitude/src/examples/step.ts +++ b/packages/web/destinations/amplitude/src/examples/step.ts @@ -12,6 +12,8 @@ import type { Settings } from '../types'; export type AmplitudeStepExample = Flow.StepExample & { settings?: Partial; configInclude?: string[]; + /** Consent granted before `in` so a gated destination is loaded first. */ + before?: WalkerOS.Consent; }; /** @@ -409,9 +411,13 @@ export const consentRevokeOptOut: AmplitudeStepExample = { description: 'A walker consent command with analytics denied opts out of Amplitude tracking via setOptOut(true).', command: 'consent', + before: { analytics: true }, in: { analytics: false } as WalkerOS.Consent, settings: {} as Partial, - out: [['amplitude.setOptOut', true]], + out: [ + ['amplitude.setOptOut', false], + ['amplitude.setOptOut', true], + ], }; /** diff --git a/packages/web/destinations/clarity/src/__tests__/stepExamples.test.ts b/packages/web/destinations/clarity/src/__tests__/stepExamples.test.ts index 23575eb23..ec84421a1 100644 --- a/packages/web/destinations/clarity/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/clarity/src/__tests__/stepExamples.test.ts @@ -11,6 +11,7 @@ import type { WalkerOS, Mapping as WalkerOSMapping } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { clone } from '@walkeros/core'; import { examples } from '../dev'; +import type { ClarityStepExample } from '../examples/step'; import type { Env, Settings } from '../types'; type CallRecord = [string, ...unknown[]]; @@ -42,62 +43,62 @@ function spyEnv(env: Env): { } describe('clarity destination — step examples', () => { - it.each(Object.entries(examples.step))('%s', async (name, rawExample) => { - const example = rawExample as { - in?: unknown; - mapping?: unknown; - out?: ReadonlyArray; - command?: 'consent' | 'user' | 'config' | 'run'; - settings?: Partial; - configInclude?: string[]; - }; - - const env = clone(examples.env.push) as Env; - const { env: spiedEnv, collected } = spyEnv(env); + it.each<[string, ClarityStepExample]>(Object.entries(examples.step))( + '%s', + async (name, example) => { + const env = clone(examples.env.push) as Env; + const { env: spiedEnv, collected } = spyEnv(env); - const dest = jest.requireActual('../').default; - const { elb } = await startFlow(); + const dest = jest.requireActual('../').default; + const { elb } = await startFlow(); - const baseSettings: Partial & { apiKey: string } = { - apiKey: 'test-project', - ...(example.settings || {}), - }; + const baseSettings: Partial & { apiKey: string } = { + apiKey: 'test-project', + ...(example.settings || {}), + }; - if (example.command === 'consent') { - // Command examples: route `in` through elb('walker ', in) - // rather than pushing it as an event. - elb('walker destination', { - code: { ...dest, env: spiedEnv }, - config: { - include: example.configInclude, - settings: baseSettings, - }, - }); - await elb('walker consent', example.in as WalkerOS.Consent); - } else { - // Standard event example. - const event = example.in as WalkerOS.Event; - const mapping = example.mapping as WalkerOSMapping.Rule | undefined; - const mappingConfig = mapping - ? { [event.entity]: { [event.action]: mapping } } - : undefined; + if (example.command === 'consent') { + // Command examples: route `in` through elb('walker ', in) + // rather than pushing it as an event. Declare the required consent on + // the destination so the collector gates init until consent is granted. + await elb('walker destination', { + code: { ...dest, env: spiedEnv }, + config: { + consent: { analytics: true, marketing: true }, + include: example.configInclude, + settings: baseSettings, + }, + }); + // Load the gated destination under a prior grant first when the example + // declares one, so a revoke acts on an already-granted destination. Both + // the grant and the consent-under-test effects are asserted. + if (example.before) await elb('walker consent', example.before); + await elb('walker consent', example.in as WalkerOS.Consent); + } else { + // Standard event example. + const event = example.in as WalkerOS.Event; + const mapping = example.mapping as WalkerOSMapping.Rule | undefined; + const mappingConfig = mapping + ? { [event.entity]: { [event.action]: mapping } } + : undefined; - elb('walker destination', { - code: { ...dest, env: spiedEnv }, - config: { - include: example.configInclude, - settings: baseSettings, - mapping: mappingConfig, - }, - }); - await elb(event); - } + elb('walker destination', { + code: { ...dest, env: spiedEnv }, + config: { + include: example.configInclude, + settings: baseSettings, + mapping: mappingConfig, + }, + }); + await elb(event); + } - // Drop the init call — every example triggers init once, it is not part - // of the declared `out`. - const expected = (example.out ?? []) as ReadonlyArray; - const actual = collected().filter(([path]) => path !== 'clarity.init'); + // Drop the init call — every example triggers init once, it is not part + // of the declared `out`. + const expected = (example.out ?? []) as ReadonlyArray; + const actual = collected().filter(([path]) => path !== 'clarity.init'); - expect(actual).toEqual(expected); - }); + expect(actual).toEqual(expected); + }, + ); }); diff --git a/packages/web/destinations/clarity/src/examples/step.ts b/packages/web/destinations/clarity/src/examples/step.ts index bf65c7267..7f438ad66 100644 --- a/packages/web/destinations/clarity/src/examples/step.ts +++ b/packages/web/destinations/clarity/src/examples/step.ts @@ -11,6 +11,8 @@ import type { Settings } from '../types'; export type ClarityStepExample = Flow.StepExample & { settings?: Partial; configInclude?: string[]; + /** Consent granted before `in` so a gated destination is loaded first. */ + before?: WalkerOS.Consent; }; /** @@ -281,15 +283,19 @@ export const consentGrantBoth: ClarityStepExample = { }; /** - * Consent revocation - denied categories call Clarity.consentV2(...) with - * denied flags. The destination does NOT call the legacy `Clarity.consent(false)` - * API at all - consentV2 is the single source of truth for consent state. + * Consent revocation - after analytics and marketing are granted (Clarity + * loads), revoking them calls Clarity.consentV2(...) with denied flags. The + * destination does NOT call the legacy `Clarity.consent(false)` API at all - + * consentV2 is the single source of truth for consent state. The destination + * is never loaded under denied consent, so the revoke acts on an + * already-granted destination. */ export const consentRevoke: ClarityStepExample = { title: 'Consent revoked', description: 'A walker consent command with analytics and marketing denied calls Clarity.consentV2 with denied flags.', command: 'consent', + before: { analytics: true, marketing: true }, in: { analytics: false, marketing: false } as WalkerOS.Consent, settings: { consent: { @@ -298,6 +304,10 @@ export const consentRevoke: ClarityStepExample = { }, }, out: [ + [ + 'clarity.consentV2', + { analytics_Storage: 'granted', ad_Storage: 'granted' }, + ], [ 'clarity.consentV2', { analytics_Storage: 'denied', ad_Storage: 'denied' }, diff --git a/packages/web/destinations/fullstory/src/__tests__/stepExamples.test.ts b/packages/web/destinations/fullstory/src/__tests__/stepExamples.test.ts index f8eca3b6e..556169aac 100644 --- a/packages/web/destinations/fullstory/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/fullstory/src/__tests__/stepExamples.test.ts @@ -10,6 +10,7 @@ import type { WalkerOS, Mapping as WalkerOSMapping } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { clone } from '@walkeros/core'; import { examples } from '../dev'; +import type { FullStoryStepExample } from '../examples/step'; import type { Env, Settings } from '../types'; type CallRecord = [string, ...unknown[]]; @@ -47,80 +48,66 @@ function spyEnv(env: Env): { } describe('fullstory destination -- step examples', () => { - it.each(Object.entries(examples.step))('%s', async (name, rawExample) => { - const example = rawExample as { - in?: unknown; - mapping?: unknown; - out?: ReadonlyArray; - command?: 'consent' | 'user' | 'config' | 'run'; - settings?: Partial; - }; - - const env = clone(examples.env.push) as Env; - const { env: spiedEnv, collected } = spyEnv(env); + it.each<[string, FullStoryStepExample]>(Object.entries(examples.step))( + '%s', + async (name, example) => { + const env = clone(examples.env.push) as Env; + const { env: spiedEnv, collected } = spyEnv(env); - const dest = jest.requireActual('../').default; - const { elb } = await startFlow(); - - const baseSettings: Partial & { orgId: string } = { - orgId: 'o-TEST-na1', - ...(example.settings || {}), - }; + const dest = jest.requireActual('../').default; + const { elb } = await startFlow(); - if (example.command === 'consent') { - elb('walker destination', { - code: { ...dest, env: spiedEnv }, - config: { - settings: baseSettings, - }, - }); - // Trigger init by pushing a priming event before consent so the - // destination's env is wired and on('consent') sees the spy env. - await elb({ - entity: 'ping', - action: 'init', - } as unknown as WalkerOS.PartialEvent); - await elb('walker consent', example.in as WalkerOS.Consent); - } else { - const event = example.in as WalkerOS.Event; - const mapping = example.mapping as WalkerOSMapping.Rule | undefined; - const mappingConfig = mapping - ? { [event.entity]: { [event.action]: mapping } } - : undefined; + const baseSettings: Partial & { orgId: string } = { + orgId: 'o-TEST-na1', + ...(example.settings || {}), + }; - elb('walker destination', { - code: { ...dest, env: spiedEnv }, - config: { - settings: baseSettings, - mapping: mappingConfig, - }, - }); - await elb(event); - } + if (example.command === 'consent') { + // Declare the example's consent key(s) on the destination so the + // collector gates its load: under consent gating a destination is + // never initialized while its required consent is denied, and + // on('consent') knows what to check. + const requiredConsent: WalkerOS.Consent = {}; + for (const key of Object.keys({ + ...(example.before ?? {}), + ...((example.in as WalkerOS.Consent) ?? {}), + })) { + requiredConsent[key] = true; + } + await elb('walker destination', { + code: { ...dest, env: spiedEnv }, + config: { + consent: requiredConsent, + settings: baseSettings, + }, + }); + // Grant first when the example declares it, so the gated destination is + // loaded before the consent under test (it never loads under denial). + // Both the grant and the consent-under-test effects are asserted. + if (example.before) await elb('walker consent', example.before); + await elb('walker consent', example.in as WalkerOS.Consent); + } else { + const event = example.in as WalkerOS.Event; + const mapping = example.mapping as WalkerOSMapping.Rule | undefined; + const mappingConfig = mapping + ? { [event.entity]: { [event.action]: mapping } } + : undefined; - // Drop init + priming trackEvent calls -- every example triggers init - // once, and consent examples push a priming event that fires trackEvent. - const expected = (example.out ?? []) as ReadonlyArray; - const actual = collected().filter(([path, arg]) => { - if (path === 'fullstory.init') return false; - if ( - example.command === 'consent' && - path === 'fullstory.trackEvent' && - isPrimingEvent(arg) - ) { - return false; + await elb('walker destination', { + code: { ...dest, env: spiedEnv }, + config: { + settings: baseSettings, + mapping: mappingConfig, + }, + }); + await elb(event); } - return true; - }); - expect(actual).toEqual(expected); - }); -}); + // Drop the init-time call -- every example triggers init once. + const expected = (example.out ?? []) as ReadonlyArray; + const actual = collected().filter(([path]) => path !== 'fullstory.init'); -function isPrimingEvent(arg: unknown): boolean { - return ( - typeof arg === 'object' && - arg !== null && - (arg as { name?: unknown }).name === 'ping init' + expect(actual).toEqual(expected); + }, ); -} +}); diff --git a/packages/web/destinations/fullstory/src/examples/step.ts b/packages/web/destinations/fullstory/src/examples/step.ts index ed5a6f865..2308427a1 100644 --- a/packages/web/destinations/fullstory/src/examples/step.ts +++ b/packages/web/destinations/fullstory/src/examples/step.ts @@ -7,6 +7,8 @@ import type { Settings } from '../types'; */ export type FullStoryStepExample = Flow.StepExample & { settings?: Partial; + /** Consent granted before `in` so a gated destination is loaded first. */ + before?: WalkerOS.Consent; }; /** @@ -250,20 +252,24 @@ export const consentGrantCapture: FullStoryStepExample = { }; /** - * Consent revoke -- revoking consent calls FullStory('shutdown'). + * Consent revoke -- after analytics consent is granted (FullStory starts + * capture), revoking it calls FullStory('shutdown'). The destination is never + * loaded under denied consent, so the shutdown is a real revocation of an + * already-granted destination. */ export const consentRevokeCapture: FullStoryStepExample = { title: 'Shutdown capture', description: 'A walker consent revoke for analytics calls FullStory shutdown to stop session recording.', command: 'consent', + before: { analytics: true }, in: { analytics: false } as WalkerOS.Consent, settings: { consent: { analytics: 'capture', }, }, - out: [['fullstory.shutdown']], + out: [['fullstory.start'], ['fullstory.shutdown']], }; /** @@ -285,18 +291,25 @@ export const consentGrantFlag: FullStoryStepExample = { }; /** - * Consent flag revoke -- calls setIdentity({ consent: false }). + * Consent flag revoke -- after marketing consent is granted (the destination + * loads), revoking it calls setIdentity({ consent: false }). The destination is + * never loaded under denied consent, so the revoke acts on an already-granted + * destination. */ export const consentRevokeFlag: FullStoryStepExample = { title: 'Consent flag revoked', description: 'A walker consent revoke with action consent sets the FullStory identity consent flag to false.', command: 'consent', + before: { marketing: true }, in: { marketing: false } as WalkerOS.Consent, settings: { consent: { marketing: 'consent', }, }, - out: [['fullstory.setIdentity', { consent: false }]], + out: [ + ['fullstory.setIdentity', { consent: true }], + ['fullstory.setIdentity', { consent: false }], + ], }; diff --git a/packages/web/destinations/heap/src/__tests__/stepExamples.test.ts b/packages/web/destinations/heap/src/__tests__/stepExamples.test.ts index b5666d291..4e079ff4f 100644 --- a/packages/web/destinations/heap/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/heap/src/__tests__/stepExamples.test.ts @@ -2,6 +2,7 @@ import type { WalkerOS, Mapping as WalkerOSMapping } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { clone } from '@walkeros/core'; import { examples } from '../dev'; +import type { HeapStepExample } from '../examples/step'; import type { Env, HeapSDK, Settings } from '../types'; type CallRecord = [string, ...unknown[]]; @@ -40,57 +41,56 @@ function spyEnv(env: Env): { env: Env; collected: () => CallRecord[] } { } describe('heap destination — step examples', () => { - it.each(Object.entries(examples.step))('%s', async (name, rawExample) => { - const example = rawExample as { - in?: unknown; - mapping?: unknown; - out?: ReadonlyArray; - command?: 'consent' | 'user' | 'config' | 'run'; - settings?: Partial; - }; - - const env = clone(examples.env.push) as Env; - const { env: spiedEnv, collected } = spyEnv(env); + it.each<[string, HeapStepExample]>(Object.entries(examples.step))( + '%s', + async (name, example) => { + const env = clone(examples.env.push) as Env; + const { env: spiedEnv, collected } = spyEnv(env); - const dest = jest.requireActual('../').default; - const { elb } = await startFlow(); + const dest = jest.requireActual('../').default; + const { elb } = await startFlow(); - const baseSettings: Partial & { appId: string } = { - appId: 'test-app-id', - ...(example.settings || {}), - }; + const baseSettings: Partial & { appId: string } = { + appId: 'test-app-id', + ...(example.settings || {}), + }; - if (example.command === 'consent') { - // Consent examples: declare the consent key on the destination so - // on() knows what to check, then fire walker consent. - await elb('walker destination', { - code: { ...dest, env: spiedEnv }, - config: { - consent: { analytics: true }, - settings: baseSettings, - }, - }); - await elb('walker consent', example.in as WalkerOS.Consent); - } else { - const event = example.in as WalkerOS.Event; - const mapping = example.mapping as WalkerOSMapping.Rule | undefined; - const mappingConfig = mapping - ? { [event.entity]: { [event.action]: mapping } } - : undefined; + if (example.command === 'consent') { + // Consent examples: declare the consent key on the destination so + // on() knows what to check, then fire walker consent. + await elb('walker destination', { + code: { ...dest, env: spiedEnv }, + config: { + consent: { analytics: true }, + settings: baseSettings, + }, + }); + // Grant first when the example declares it, so the gated destination is + // loaded before the consent under test (it never loads under denial). + // Both the grant and the consent-under-test effects are asserted. + if (example.before) await elb('walker consent', example.before); + await elb('walker consent', example.in as WalkerOS.Consent); + } else { + const event = example.in as WalkerOS.Event; + const mapping = example.mapping as WalkerOSMapping.Rule | undefined; + const mappingConfig = mapping + ? { [event.entity]: { [event.action]: mapping } } + : undefined; - await elb('walker destination', { - code: { ...dest, env: spiedEnv }, - config: { - settings: baseSettings, - mapping: mappingConfig, - }, - }); - await elb(event); - } + await elb('walker destination', { + code: { ...dest, env: spiedEnv }, + config: { + settings: baseSettings, + mapping: mappingConfig, + }, + }); + await elb(event); + } - const expected = (example.out ?? []) as ReadonlyArray; - // Filter init-time load() call. - const actual = collected().filter(([path]) => path !== 'heap.load'); - expect(actual).toEqual(expected); - }); + const expected = (example.out ?? []) as ReadonlyArray; + // Filter init-time load() call. + const actual = collected().filter(([path]) => path !== 'heap.load'); + expect(actual).toEqual(expected); + }, + ); }); diff --git a/packages/web/destinations/heap/src/examples/step.ts b/packages/web/destinations/heap/src/examples/step.ts index 56e7af5d8..16a8d31d4 100644 --- a/packages/web/destinations/heap/src/examples/step.ts +++ b/packages/web/destinations/heap/src/examples/step.ts @@ -1,4 +1,4 @@ -import type { Flow } from '@walkeros/core'; +import type { Flow, WalkerOS } from '@walkeros/core'; import { getEvent } from '@walkeros/core'; import type { Settings } from '../types'; @@ -9,6 +9,8 @@ import type { Settings } from '../types'; */ export type HeapStepExample = Flow.StepExample & { settings?: Partial; + /** Consent granted before `in` so a gated destination is loaded first. */ + before?: WalkerOS.Consent; }; /** @@ -181,16 +183,19 @@ export const globalEventProperties: HeapStepExample = { }; /** - * Consent revoked - on('consent') handler calls heap.stopTracking() - * when a required consent key is false. + * Consent granted then revoked: the destination loads under the grant + * (heap.startTracking), and the later revoke calls heap.stopTracking via the + * on('consent') handler. A gated destination never loads under denial, so the + * stop is a real revocation of an already-granted destination. */ export const consentRevokeStopTracking: HeapStepExample = { title: 'Consent revoked', description: - 'A walker consent revoke for analytics calls heap.stopTracking to pause event capture.', + 'After analytics consent is granted (Heap loads and starts tracking), revoking it calls heap.stopTracking to pause event capture.', command: 'consent', + before: { analytics: true }, in: { analytics: false }, - out: [['heap.stopTracking']], + out: [['heap.startTracking'], ['heap.stopTracking']], }; /** diff --git a/packages/web/destinations/mixpanel/src/__tests__/stepExamples.test.ts b/packages/web/destinations/mixpanel/src/__tests__/stepExamples.test.ts index 137f1968f..9b47a3065 100644 --- a/packages/web/destinations/mixpanel/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/mixpanel/src/__tests__/stepExamples.test.ts @@ -11,6 +11,7 @@ import type { WalkerOS, Mapping as WalkerOSMapping } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { clone } from '@walkeros/core'; import { examples } from '../dev'; +import type { MixpanelStepExample } from '../examples/step'; import type { Env, MixpanelGroup, Settings } from '../types'; type CallRecord = [string, ...unknown[]]; @@ -114,68 +115,67 @@ function spyEnv(env: Env): { env: Env; collected: () => CallRecord[] } { } describe('mixpanel destination — step examples', () => { - it.each(Object.entries(examples.step))('%s', async (name, rawExample) => { - const example = rawExample as { - in?: unknown; - mapping?: unknown; - out?: ReadonlyArray; - command?: 'consent' | 'user' | 'config' | 'run'; - settings?: Partial; - configInclude?: string[]; - }; + it.each<[string, MixpanelStepExample]>(Object.entries(examples.step))( + '%s', + async (name, example) => { + const env = clone(examples.env.push) as Env; + const { env: spiedEnv, collected } = spyEnv(env); - const env = clone(examples.env.push) as Env; - const { env: spiedEnv, collected } = spyEnv(env); + const dest = jest.requireActual('../').default; + const { elb } = await startFlow(); - const dest = jest.requireActual('../').default; - const { elb } = await startFlow(); + const baseSettings: Partial & { apiKey: string } = { + apiKey: 'test-project', + ...(example.settings || {}), + }; - const baseSettings: Partial & { apiKey: string } = { - apiKey: 'test-project', - ...(example.settings || {}), - }; + if (example.command === 'consent') { + await elb('walker destination', { + code: { ...dest, env: spiedEnv }, + config: { + consent: { analytics: true }, + include: example.configInclude, + settings: baseSettings, + }, + }); + // Grant first when the example declares it, so the gated destination is + // loaded before the consent under test (it never loads under denial). + // Both the grant and the consent-under-test effects are asserted. + if (example.before) await elb('walker consent', example.before); + await elb('walker consent', example.in as WalkerOS.Consent); + } else { + const event = example.in as WalkerOS.Event; + const mapping = example.mapping as WalkerOSMapping.Rule | undefined; + const mappingConfig = mapping + ? { [event.entity]: { [event.action]: mapping } } + : undefined; - if (example.command === 'consent') { - await elb('walker destination', { - code: { ...dest, env: spiedEnv }, - config: { - consent: { analytics: true }, - include: example.configInclude, - settings: baseSettings, - }, - }); - await elb('walker consent', example.in as WalkerOS.Consent); - } else { - const event = example.in as WalkerOS.Event; - const mapping = example.mapping as WalkerOSMapping.Rule | undefined; - const mappingConfig = mapping - ? { [event.entity]: { [event.action]: mapping } } - : undefined; + await elb('walker destination', { + code: { ...dest, env: spiedEnv }, + config: { + include: example.configInclude, + settings: baseSettings, + mapping: mappingConfig, + }, + }); + await elb(event); + } - await elb('walker destination', { - code: { ...dest, env: spiedEnv }, - config: { - include: example.configInclude, - settings: baseSettings, - mapping: mappingConfig, - }, + // Drop init + bookkeeping — every example triggers init once; the + // synthetic 'mixpanel.get_group' marker is also dropped unless an + // example explicitly expects it. + const expected = (example.out ?? []) as ReadonlyArray; + const expectsGetGroupMarker = expected.some( + ([path]) => path === 'mixpanel.get_group', + ); + const actual = collected().filter(([path]) => { + if (path === 'mixpanel.init') return false; + if (path === 'mixpanel.get_group' && !expectsGetGroupMarker) + return false; + return true; }); - await elb(event); - } - - // Drop init + bookkeeping — every example triggers init once; the - // synthetic 'mixpanel.get_group' marker is also dropped unless an - // example explicitly expects it. - const expected = (example.out ?? []) as ReadonlyArray; - const expectsGetGroupMarker = expected.some( - ([path]) => path === 'mixpanel.get_group', - ); - const actual = collected().filter(([path]) => { - if (path === 'mixpanel.init') return false; - if (path === 'mixpanel.get_group' && !expectsGetGroupMarker) return false; - return true; - }); - expect(actual).toEqual(expected); - }); + expect(actual).toEqual(expected); + }, + ); }); diff --git a/packages/web/destinations/mixpanel/src/examples/step.ts b/packages/web/destinations/mixpanel/src/examples/step.ts index 4be31c552..f092284cb 100644 --- a/packages/web/destinations/mixpanel/src/examples/step.ts +++ b/packages/web/destinations/mixpanel/src/examples/step.ts @@ -10,6 +10,8 @@ import type { Settings } from '../types'; export type MixpanelStepExample = Flow.StepExample & { settings?: Partial; configInclude?: string[]; + /** Consent granted before `in` so a gated destination is loaded first. */ + before?: WalkerOS.Consent; }; /** @@ -385,17 +387,21 @@ export const companyUpdateGroupProfile: MixpanelStepExample = { }; /** - * Consent revoked → mixpanel.opt_out_tracking(). The destination checks - * the consent keys declared in config.consent and toggles opt_out/opt_in. + * Consent revoked → mixpanel.opt_out_tracking(). After analytics consent is + * granted (Mixpanel loads and opts in), revoking it calls + * mixpanel.opt_out_tracking() via the consent handler. The destination is + * never loaded under denied consent, so the opt-out is a real revocation of an + * already-granted destination. */ export const consentRevokeOptOut: MixpanelStepExample = { title: 'Consent revoked', description: 'A walker consent command with analytics denied calls mixpanel.opt_out_tracking to stop event capture.', command: 'consent', + before: { analytics: true }, in: { analytics: false } as WalkerOS.Consent, settings: {} as Partial, - out: [['mixpanel.opt_out_tracking']], + out: [['mixpanel.opt_in_tracking'], ['mixpanel.opt_out_tracking']], }; /** diff --git a/packages/web/destinations/optimizely/src/__tests__/stepExamples.test.ts b/packages/web/destinations/optimizely/src/__tests__/stepExamples.test.ts index 6fcf0a0f4..09496191d 100644 --- a/packages/web/destinations/optimizely/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/optimizely/src/__tests__/stepExamples.test.ts @@ -5,6 +5,7 @@ jest.mock('@optimizely/optimizely-sdk', () => ({ import type { WalkerOS, Mapping as WalkerOSMapping } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { examples } from '../dev'; +import type { OptimizelyStepExample } from '../examples/step'; import type { Env, OptimizelyUserContext, Settings } from '../types'; type CallRecord = [string, ...unknown[]]; @@ -42,64 +43,52 @@ function spyEnv(): { env: Env; collected: () => CallRecord[] } { } describe('optimizely destination -- step examples', () => { - it.each(Object.entries(examples.step))('%s', async (_name, rawExample) => { - const example = rawExample as { - in?: unknown; - mapping?: unknown; - out?: ReadonlyArray; - command?: 'consent' | 'user' | 'config' | 'run'; - settings?: Partial; - }; + it.each<[string, OptimizelyStepExample]>(Object.entries(examples.step))( + '%s', + async (name, example) => { + const { env, collected } = spyEnv(); + const dest = jest.requireActual('../').default; + const { elb } = await startFlow(); - const { env, collected } = spyEnv(); - const dest = jest.requireActual('../').default; - const { elb } = await startFlow(); + const baseSettings: Partial & { sdkKey: string } = { + sdkKey: 'test-sdk-key', + userId: 'user.id', + ...(example.settings || {}), + }; - const baseSettings: Partial & { sdkKey: string } = { - sdkKey: 'test-sdk-key', - userId: 'user.id', - ...(example.settings || {}), - }; + if (example.command === 'consent') { + await elb('walker destination', { + code: { ...dest, env }, + config: { + consent: { analytics: true }, + settings: baseSettings, + }, + }); + // Load the gated destination under a prior grant first when the example + // declares one, so a revoke acts on an already-granted destination. + if (example.before) await elb('walker consent', example.before); + await elb('walker consent', example.in as WalkerOS.Consent); + } else { + const event = example.in as WalkerOS.Event; + const mapping = example.mapping as WalkerOSMapping.Rule | undefined; + const mappingConfig = mapping + ? { [event.entity]: { [event.action]: mapping } } + : undefined; - if (example.command === 'consent') { - await elb('walker destination', { - code: { ...dest, env }, - config: { - consent: { analytics: true }, - settings: baseSettings, - }, - }); - // Prime the destination so init runs and state.client exists before - // consent revocation fires on('consent'). - await elb({ - entity: 'ping', - action: 'init', - data: { id: 'prime' }, - } as unknown as WalkerOS.PartialEvent); - await elb('walker consent', { analytics: true } as WalkerOS.Consent); - // Reset the call log so only the post-consent behavior is asserted. - collected().length = 0; - await elb('walker consent', example.in as WalkerOS.Consent); - } else { - const event = example.in as WalkerOS.Event; - const mapping = example.mapping as WalkerOSMapping.Rule | undefined; - const mappingConfig = mapping - ? { [event.entity]: { [event.action]: mapping } } - : undefined; - - await elb('walker destination', { - code: { ...dest, env }, - config: { - settings: baseSettings, - mapping: mappingConfig, - }, - }); - await elb(event); - } + await elb('walker destination', { + code: { ...dest, env }, + config: { + settings: baseSettings, + mapping: mappingConfig, + }, + }); + await elb(event); + } - const expected = (example.out ?? []) as ReadonlyArray; - const actual = collected(); + const expected = (example.out ?? []) as ReadonlyArray; + const actual = collected(); - expect(actual).toEqual(expected); - }); + expect(actual).toEqual(expected); + }, + ); }); diff --git a/packages/web/destinations/optimizely/src/examples/step.ts b/packages/web/destinations/optimizely/src/examples/step.ts index 6eeb9812a..d9c677700 100644 --- a/packages/web/destinations/optimizely/src/examples/step.ts +++ b/packages/web/destinations/optimizely/src/examples/step.ts @@ -7,6 +7,8 @@ import type { Settings } from '../types'; */ export type OptimizelyStepExample = Flow.StepExample & { settings?: Partial; + /** Consent granted before `in` so a gated destination is loaded first. */ + before?: WalkerOS.Consent; }; /** @@ -161,6 +163,7 @@ export const consentRevoked: OptimizelyStepExample = { description: 'A walker consent command with analytics denied closes the Optimizely client and flushes queued events.', command: 'consent', + before: { analytics: true }, in: { analytics: false } as WalkerOS.Consent, settings: {} as Partial, out: [['optimizely.close']], diff --git a/packages/web/destinations/posthog/src/__tests__/stepExamples.test.ts b/packages/web/destinations/posthog/src/__tests__/stepExamples.test.ts index 3304acfd3..f7ad8f0de 100644 --- a/packages/web/destinations/posthog/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/posthog/src/__tests__/stepExamples.test.ts @@ -11,6 +11,7 @@ import type { WalkerOS, Mapping as WalkerOSMapping } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { clone } from '@walkeros/core'; import { examples } from '../dev'; +import type { PostHogStepExample } from '../examples/step'; import type { Env, PostHogSDK, Settings } from '../types'; type CallRecord = [string, ...unknown[]]; @@ -70,61 +71,59 @@ function spyEnv(env: Env): { env: Env; collected: () => CallRecord[] } { } describe('posthog destination — step examples', () => { - it.each(Object.entries(examples.step))('%s', async (name, rawExample) => { - const example = rawExample as { - in?: unknown; - mapping?: unknown; - out?: ReadonlyArray; - command?: 'consent' | 'user' | 'config' | 'run'; - settings?: Partial; - configInclude?: string[]; - }; + it.each<[string, PostHogStepExample]>(Object.entries(examples.step))( + '%s', + async (name, example) => { + const env = clone(examples.env.push) as Env; + const { env: spiedEnv, collected } = spyEnv(env); - const env = clone(examples.env.push) as Env; - const { env: spiedEnv, collected } = spyEnv(env); + const dest = jest.requireActual('../').default; + const { elb } = await startFlow(); - const dest = jest.requireActual('../').default; - const { elb } = await startFlow(); + const baseSettings: Partial & { apiKey: string } = { + apiKey: 'phc_test', + ...(example.settings || {}), + }; - const baseSettings: Partial & { apiKey: string } = { - apiKey: 'phc_test', - ...(example.settings || {}), - }; + if (example.command === 'consent') { + // Consent examples need config.consent declared so the destination's + // on() handler knows which walkerOS consent key to check. + elb('walker destination', { + code: { ...dest, env: spiedEnv }, + config: { + consent: { analytics: true }, + include: example.configInclude, + settings: baseSettings, + }, + }); + // Load the gated destination under a prior grant first when the example + // declares one, so a revoke acts on an already-granted destination. The + // pre-grant's calls are dropped so only the example's consent is asserted. + if (example.before) await elb('walker consent', example.before); + await elb('walker consent', example.in as WalkerOS.Consent); + } else { + const event = example.in as WalkerOS.Event; + const mapping = example.mapping as WalkerOSMapping.Rule | undefined; + const mappingConfig = mapping + ? { [event.entity]: { [event.action]: mapping } } + : undefined; - if (example.command === 'consent') { - // Consent examples need config.consent declared so the destination's - // on() handler knows which walkerOS consent key to check. - elb('walker destination', { - code: { ...dest, env: spiedEnv }, - config: { - consent: { analytics: true }, - include: example.configInclude, - settings: baseSettings, - }, - }); - await elb('walker consent', example.in as WalkerOS.Consent); - } else { - const event = example.in as WalkerOS.Event; - const mapping = example.mapping as WalkerOSMapping.Rule | undefined; - const mappingConfig = mapping - ? { [event.entity]: { [event.action]: mapping } } - : undefined; + elb('walker destination', { + code: { ...dest, env: spiedEnv }, + config: { + include: example.configInclude, + settings: baseSettings, + mapping: mappingConfig, + }, + }); + await elb(event); + } - elb('walker destination', { - code: { ...dest, env: spiedEnv }, - config: { - include: example.configInclude, - settings: baseSettings, - mapping: mappingConfig, - }, - }); - await elb(event); - } + // Drop init — every example triggers init once; it's not part of `out`. + const expected = (example.out ?? []) as ReadonlyArray; + const actual = collected().filter(([path]) => path !== 'posthog.init'); - // Drop init — every example triggers init once; it's not part of `out`. - const expected = (example.out ?? []) as ReadonlyArray; - const actual = collected().filter(([path]) => path !== 'posthog.init'); - - expect(actual).toEqual(expected); - }); + expect(actual).toEqual(expected); + }, + ); }); diff --git a/packages/web/destinations/posthog/src/examples/step.ts b/packages/web/destinations/posthog/src/examples/step.ts index daecd1fe1..53e87a91a 100644 --- a/packages/web/destinations/posthog/src/examples/step.ts +++ b/packages/web/destinations/posthog/src/examples/step.ts @@ -10,6 +10,8 @@ import type { Settings } from '../types'; export type PostHogStepExample = Flow.StepExample & { settings?: Partial; configInclude?: string[]; + /** Consent granted before `in` so a gated destination is loaded first. */ + before?: WalkerOS.Consent; }; /** @@ -340,9 +342,10 @@ export const consentRevokeOptOut: PostHogStepExample = { description: 'A walker consent command with analytics denied calls posthog.opt_out_capturing to stop capture and replay.', command: 'consent', + before: { analytics: true }, in: { analytics: false } as WalkerOS.Consent, settings: {} as Partial, - out: [['posthog.opt_out_capturing']], + out: [['posthog.opt_in_capturing'], ['posthog.opt_out_capturing']], }; /** diff --git a/packages/web/destinations/tiktok/src/__tests__/stepExamples.test.ts b/packages/web/destinations/tiktok/src/__tests__/stepExamples.test.ts index 620f0cd2a..f5e9b9542 100644 --- a/packages/web/destinations/tiktok/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/tiktok/src/__tests__/stepExamples.test.ts @@ -2,6 +2,7 @@ import type { WalkerOS, Mapping as WalkerOSMapping } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { clone } from '@walkeros/core'; import { examples } from '../dev'; +import type { TikTokStepExample } from '../examples/step'; import type { Env, Settings, TTQ } from '../types'; type CallRecord = [string, ...unknown[]]; @@ -35,60 +36,57 @@ function spyTtq(env: Env): { } describe('tiktok destination — step examples', () => { - it.each(Object.entries(examples.step))('%s', async (_name, rawExample) => { - const example = rawExample as { - in?: unknown; - mapping?: unknown; - out?: ReadonlyArray; - command?: 'consent' | 'user' | 'config' | 'run'; - settings?: Partial; - configInclude?: string[]; - }; - - // Fresh clone per test so mutations don't bleed across cases. - const env = clone(examples.env.push) as Env; - const { env: spiedEnv, collected } = spyTtq(env); + it.each<[string, TikTokStepExample]>(Object.entries(examples.step))( + '%s', + async (name, example) => { + // Fresh clone per test so mutations don't bleed across cases. + const env = clone(examples.env.push) as Env; + const { env: spiedEnv, collected } = spyTtq(env); - const dest = jest.requireActual('../').default; - const { elb } = await startFlow(); + const dest = jest.requireActual('../').default; + const { elb } = await startFlow(); - const baseSettings: Partial & { apiKey: string } = { - apiKey: 'test-pixel-id', - ...(example.settings || {}), - }; + const baseSettings: Partial & { apiKey: string } = { + apiKey: 'test-pixel-id', + ...(example.settings || {}), + }; - if (example.command === 'consent') { - await elb('walker destination', { - code: { ...dest, env: spiedEnv }, - config: { - consent: { marketing: true }, - include: example.configInclude, - settings: baseSettings, - }, - }); - await elb('walker consent', example.in as WalkerOS.Consent); - } else { - const event = example.in as WalkerOS.Event; - const mapping = example.mapping as WalkerOSMapping.Rule | undefined; - const mappingConfig = mapping - ? { [event.entity]: { [event.action]: mapping } } - : undefined; + if (example.command === 'consent') { + await elb('walker destination', { + code: { ...dest, env: spiedEnv }, + config: { + consent: { marketing: true }, + include: example.configInclude, + settings: baseSettings, + }, + }); + // Load the gated destination under a prior grant first when the example + // declares one, so a revoke acts on an already-granted destination. + if (example.before) await elb('walker consent', example.before); + await elb('walker consent', example.in as WalkerOS.Consent); + } else { + const event = example.in as WalkerOS.Event; + const mapping = example.mapping as WalkerOSMapping.Rule | undefined; + const mappingConfig = mapping + ? { [event.entity]: { [event.action]: mapping } } + : undefined; - await elb('walker destination', { - code: { ...dest, env: spiedEnv }, - config: { - include: example.configInclude, - settings: baseSettings, - mapping: mappingConfig, - }, - }); - await elb(event); - } + await elb('walker destination', { + code: { ...dest, env: spiedEnv }, + config: { + include: example.configInclude, + settings: baseSettings, + mapping: mappingConfig, + }, + }); + await elb(event); + } - // Drop ttq.load — it's an init-time call, not part of per-push `out`. - const expected = (example.out ?? []) as ReadonlyArray; - const actual = collected().filter(([path]) => path !== 'ttq.load'); + // Drop ttq.load — it's an init-time call, not part of per-push `out`. + const expected = (example.out ?? []) as ReadonlyArray; + const actual = collected().filter(([path]) => path !== 'ttq.load'); - expect(actual).toEqual(expected); - }); + expect(actual).toEqual(expected); + }, + ); }); diff --git a/packages/web/destinations/tiktok/src/examples/step.ts b/packages/web/destinations/tiktok/src/examples/step.ts index 436af162a..5c474487b 100644 --- a/packages/web/destinations/tiktok/src/examples/step.ts +++ b/packages/web/destinations/tiktok/src/examples/step.ts @@ -10,6 +10,8 @@ import type { Settings } from '../types'; export type TikTokStepExample = Flow.StepExample & { settings?: Partial; configInclude?: string[]; + /** Consent granted before `in` so a gated destination is loaded first. */ + before?: WalkerOS.Consent; }; /** @@ -382,7 +384,8 @@ export const consentRevokeDisableCookie: TikTokStepExample = { description: 'A walker consent revoke for marketing calls ttq.disableCookie so TikTok stops using its first-party cookie.', command: 'consent', + before: { marketing: true }, in: { marketing: false } as WalkerOS.Consent, settings: {} as Partial, - out: [['ttq.disableCookie']], + out: [['ttq.enableCookie'], ['ttq.disableCookie']], }; diff --git a/packages/web/sources/browser/src/__tests__/command-coverage.test.ts b/packages/web/sources/browser/src/__tests__/command-coverage.test.ts deleted file mode 100644 index 6e731b8ae..000000000 --- a/packages/web/sources/browser/src/__tests__/command-coverage.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import * as fs from 'fs'; -import * as path from 'path'; - -/** - * Coverage regression: every walker command declared in the public elb - * interface (BrowserPush + Elb.WalkerCommands) must have at least one - * occurrence of the literal "walker " in a test file inside this - * package. This catches a future refactor that silently drops a handler - * along with the only test that exercised it. - */ -const WALKER_COMMANDS = [ - 'init', - 'run', - 'destination', - 'hook', - 'consent', - 'config', - 'on', - 'user', -] as const; - -const TEST_DIR = __dirname; - -function readAllTestFiles(): string { - const files = fs - .readdirSync(TEST_DIR) - .filter((name) => name.endsWith('.test.ts')); - return files - .map((name) => fs.readFileSync(path.join(TEST_DIR, name), 'utf8')) - .join('\n'); -} - -describe('walker command coverage', () => { - const combined = readAllTestFiles(); - - for (const command of WALKER_COMMANDS) { - test(`"walker ${command}" appears in at least one test file`, () => { - expect(combined).toContain(`walker ${command}`); - }); - } -}); diff --git a/packages/web/sources/cmps/usercentrics/README.md b/packages/web/sources/cmps/usercentrics/README.md index 8377cf4cd..2016eac7e 100644 --- a/packages/web/sources/cmps/usercentrics/README.md +++ b/packages/web/sources/cmps/usercentrics/README.md @@ -6,9 +6,9 @@ # @walkeros/web-source-cmp-usercentrics -Integrates Usercentrics consent management with walkerOS by listening for a -configured window event and mapping category or service consent state to -walkerOS consent groups. +Integrates Usercentrics consent management with walkerOS using the official +Usercentrics events and consent getters, mapping category or service consent +state to walkerOS consent groups. [Documentation](https://www.walkeros.io/docs/sources/web/cmps/usercentrics) • @@ -37,6 +37,39 @@ await startFlow({ }); ``` +## How it works + +The source uses Usercentrics' official integration surface, no custom data-layer +event is required: + +1. Already initialized: if the CMP has loaded before the source, consent is read + statically through the official getters (V2 `UC_UI.getServicesBaseInfo()`, V3 + `__ucCmp.getConsentDetails()`). +2. CMP loads after the source: the source listens for `UC_UI_INITIALIZED` and + reads the current consent state once the CMP signals it is ready. +3. User decisions: the source listens for `UC_UI_CMP_EVENT` (consent actions + `ACCEPT_ALL`, `DENY_ALL`, and `SAVE`) and republishes the updated state. + +Consent is then mapped via `categoryMap` and published with +`elb('walker consent', state)`. + +## Timing considerations + +A returning visitor's prior choice is applied on page load, either from the +static getter read (CMP already initialized) or on `UC_UI_INITIALIZED` (CMP +loads later). First-visit defaults are suppressed under the default +`explicitOnly: true`: only a state the user has actively decided is published +(V3 `consent.type === EXPLICIT`; V2 an `EXPLICIT` entry in the service consent +history). Set `explicitOnly: false` to publish any consent snapshot, including +implicit first-visit defaults. + +## Settings + +`explicitOnly` (default `true`) is the cross-version gate for "the user has +actively decided," applied identically for V2 and V3. There is no configurable +data-layer event setting: the `eventName` setting has been removed, and the +source listens to the always-emitted official events. + ## Documentation Full configuration, mapping, and examples live in the docs: diff --git a/packages/web/sources/cmps/usercentrics/src/__tests__/index.test.ts b/packages/web/sources/cmps/usercentrics/src/__tests__/index.test.ts index 307804593..ca9bd8297 100644 --- a/packages/web/sources/cmps/usercentrics/src/__tests__/index.test.ts +++ b/packages/web/sources/cmps/usercentrics/src/__tests__/index.test.ts @@ -1,7 +1,5 @@ import { sourceUsercentrics } from '../index'; import { createMockLogger } from '@walkeros/core'; -import * as inputs from '../examples/inputs'; -import * as outputs from '../examples/outputs'; import type { UsercentricsV2Api, UsercentricsV2Service, @@ -15,6 +13,8 @@ import { createMockElb, createMockWindow, createUsercentricsSource, + makeUcUi, + makeV2Service, ConsentCall, MockWindow, } from './test-utils'; @@ -48,8 +48,9 @@ describe('Usercentrics Source', () => { const mockWindow = createMockWindow(); const source = await createUsercentricsSource(mockWindow, mockElb); - expect(source.config.settings?.eventName).toBe('ucEvent'); expect(source.config.settings?.explicitOnly).toBe(true); + expect(source.config.settings?.apiVersion).toBe('auto'); + expect(source.config.settings?.v3EventName).toBe('UC_UI_CMP_EVENT'); expect(source.config.settings?.categoryMap).toEqual({}); }); @@ -57,119 +58,134 @@ describe('Usercentrics Source', () => { const mockWindow = createMockWindow(); const source = await createUsercentricsSource(mockWindow, mockElb, { settings: { - eventName: 'myConsentEvent', explicitOnly: false, categoryMap: { essential: 'functional' }, }, }); - expect(source.config.settings?.eventName).toBe('myConsentEvent'); expect(source.config.settings?.explicitOnly).toBe(false); expect(source.config.settings?.categoryMap).toEqual({ essential: 'functional', }); + // Untouched defaults remain. + expect(source.config.settings?.apiVersion).toBe('auto'); + expect(source.config.settings?.v3EventName).toBe('UC_UI_CMP_EVENT'); }); - test('registers event listener on configured event name', async () => { + test('registers official V2 listeners when only UC_UI is present', async () => { const mockWindow = createMockWindow(); - await createUsercentricsSource(mockWindow, mockElb); - - expect(mockWindow.addEventListener).toHaveBeenCalledWith( - 'ucEvent', - expect.any(Function), + mockWindow.__setUcUi( + makeUcUi([makeV2Service('essential', true, 'explicit')]), ); - }); - test('registers listener on custom event name', async () => { - const mockWindow = createMockWindow(); await createUsercentricsSource(mockWindow, mockElb, { - settings: { eventName: 'UC_SDK_EVENT' }, + settings: { apiVersion: 'v2' }, }); expect(mockWindow.addEventListener).toHaveBeenCalledWith( - 'UC_SDK_EVENT', + 'UC_UI_INITIALIZED', + expect.any(Function), + ); + expect(mockWindow.addEventListener).toHaveBeenCalledWith( + 'UC_UI_CMP_EVENT', expect.any(Function), ); }); }); describe('explicit consent filtering', () => { - test('processes explicit consent events', async () => { + test('publishes a returning visitor whose history is explicit (default explicitOnly)', async () => { const mockWindow = createMockWindow(); - await createUsercentricsSource(mockWindow, mockElb); - - mockWindow.__dispatchEvent('ucEvent', inputs.fullConsent); - - expect(consentCalls).toHaveLength(1); - }); - - test('processes explicit consent with uppercase type', async () => { - const mockWindow = createMockWindow(); - await createUsercentricsSource(mockWindow, mockElb); - - mockWindow.__dispatchEvent('ucEvent', inputs.fullConsentUpperCase); - - expect(consentCalls).toHaveLength(1); - expect(consentCalls[0].consent).toEqual(outputs.fullConsentMapped); - }); - - test('ignores implicit consent when explicitOnly=true', async () => { - const mockWindow = createMockWindow(); - await createUsercentricsSource(mockWindow, mockElb); - - mockWindow.__dispatchEvent('ucEvent', inputs.implicitConsent); - - expect(consentCalls).toHaveLength(0); - }); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'explicit'), + makeV2Service('marketing', true, 'explicit'), + ]), + ); - test('processes implicit consent when explicitOnly=false', async () => { - const mockWindow = createMockWindow(); await createUsercentricsSource(mockWindow, mockElb, { - settings: { explicitOnly: false }, + settings: { apiVersion: 'v2' }, }); - mockWindow.__dispatchEvent('ucEvent', inputs.implicitConsent); - expect(consentCalls).toHaveLength(1); - expect(consentCalls[0].consent).toEqual(outputs.minimalConsentMapped); + expect(consentCalls[0].consent).toEqual({ + essential: true, + marketing: true, + }); }); - }); - describe('non-consent event filtering', () => { - test('ignores non-consent_status events', async () => { + test('suppresses a first-visit implicit-only snapshot under default explicitOnly', async () => { const mockWindow = createMockWindow(); - await createUsercentricsSource(mockWindow, mockElb); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'implicit'), + makeV2Service('marketing', false, 'implicit'), + ]), + ); - mockWindow.__dispatchEvent('ucEvent', inputs.nonConsentEvent); + await createUsercentricsSource(mockWindow, mockElb, { + settings: { apiVersion: 'v2' }, + }); expect(consentCalls).toHaveLength(0); }); - test('ignores events without detail', async () => { + test('publishes an implicit-only snapshot when explicitOnly=false', async () => { const mockWindow = createMockWindow(); - await createUsercentricsSource(mockWindow, mockElb); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'implicit'), + makeV2Service('marketing', false, 'implicit'), + ]), + ); - // Dispatch event with no detail - mockWindow.__dispatchEvent('ucEvent'); + await createUsercentricsSource(mockWindow, mockElb, { + settings: { apiVersion: 'v2', explicitOnly: false }, + }); - expect(consentCalls).toHaveLength(0); + expect(consentCalls).toHaveLength(1); + expect(consentCalls[0].consent).toEqual({ + essential: true, + marketing: false, + }); }); }); describe('group-level consent (ucCategory)', () => { - test('maps partial consent correctly', async () => { - const mockWindow = createMockWindow(); - await createUsercentricsSource(mockWindow, mockElb); + test('maps partial consent via strict-AND aggregation', async () => { + const mockWindow = createMockWindow(); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'explicit'), + makeV2Service('functional', true, 'explicit'), + makeV2Service('marketing', false, 'explicit'), + ]), + ); - mockWindow.__dispatchEvent('ucEvent', inputs.partialConsent); + await createUsercentricsSource(mockWindow, mockElb, { + settings: { apiVersion: 'v2' }, + }); - expect(consentCalls[0].consent).toEqual(outputs.partialConsentMapped); + expect(consentCalls[0].consent).toEqual({ + essential: true, + functional: true, + marketing: false, + }); }); test('applies custom category mapping', async () => { const mockWindow = createMockWindow(); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'explicit'), + makeV2Service('functional', true, 'explicit'), + makeV2Service('marketing', true, 'explicit'), + ]), + ); + await createUsercentricsSource(mockWindow, mockElb, { settings: { + apiVersion: 'v2', categoryMap: { essential: 'functional', functional: 'functional', @@ -178,22 +194,23 @@ describe('Usercentrics Source', () => { }, }); - mockWindow.__dispatchEvent('ucEvent', inputs.fullConsent); - - expect(consentCalls[0].consent).toEqual(outputs.fullConsentCustomMapped); + expect(consentCalls[0].consent).toEqual({ + functional: true, + marketing: true, + }); }); test('passes through unmapped categories', async () => { const mockWindow = createMockWindow(); - await createUsercentricsSource(mockWindow, mockElb); - - mockWindow.__dispatchEvent('ucEvent', { - event: 'consent_status', - type: 'explicit', - ucCategory: { - essential: true, - custom_group: true, - }, + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'explicit'), + makeV2Service('custom_group', true, 'explicit'), + ]), + ); + + await createUsercentricsSource(mockWindow, mockElb, { + settings: { apiVersion: 'v2' }, }); expect(consentCalls[0].consent).toEqual({ @@ -202,10 +219,19 @@ describe('Usercentrics Source', () => { }); }); - test('uses strict AND logic when multiple categories map to same group', async () => { + test('uses strict AND when multiple categories map to the same group', async () => { const mockWindow = createMockWindow(); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'explicit'), + makeV2Service('functional', false, 'explicit'), + makeV2Service('marketing', false, 'explicit'), + ]), + ); + await createUsercentricsSource(mockWindow, mockElb, { settings: { + apiVersion: 'v2', categoryMap: { essential: 'functional', functional: 'functional', @@ -214,17 +240,7 @@ describe('Usercentrics Source', () => { }); // essential=true, functional=false both map to 'functional'. - // Strict AND: any deny signal denies → functional = false. - mockWindow.__dispatchEvent('ucEvent', { - event: 'consent_status', - type: 'explicit', - ucCategory: { - essential: true, - functional: false, - marketing: false, - }, - }); - + // Strict AND: any deny denies → functional = false. expect(consentCalls[0].consent).toEqual({ functional: false, marketing: false, @@ -233,8 +249,17 @@ describe('Usercentrics Source', () => { test('strict AND: all contributing sources true → target true', async () => { const mockWindow = createMockWindow(); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'explicit'), + makeV2Service('functional', true, 'explicit'), + makeV2Service('marketing', false, 'explicit'), + ]), + ); + await createUsercentricsSource(mockWindow, mockElb, { settings: { + apiVersion: 'v2', categoryMap: { essential: 'functional', functional: 'functional', @@ -242,16 +267,6 @@ describe('Usercentrics Source', () => { }, }); - mockWindow.__dispatchEvent('ucEvent', { - event: 'consent_status', - type: 'explicit', - ucCategory: { - essential: true, - functional: true, - marketing: false, - }, - }); - expect(consentCalls[0].consent).toEqual({ functional: true, marketing: false, @@ -259,141 +274,162 @@ describe('Usercentrics Source', () => { }); }); - describe('service-level consent', () => { - test('extracts individual services when ucCategory has non-boolean values', async () => { - const mockWindow = createMockWindow(); - await createUsercentricsSource(mockWindow, mockElb); - - mockWindow.__dispatchEvent('ucEvent', inputs.serviceLevelConsent); + describe('category-level aggregation across services', () => { + // Through the real V2 adapter, buildDetailFromServices always produces a + // boolean ucCategory, so parseConsent takes the group-level branch and the + // per-service name keys are not consumed. These tests exercise that + // end-to-end V2 path (multiple services per category, strict-AND). - expect(consentCalls[0].consent).toEqual(outputs.serviceLevelMapped); - }); - - test('normalizes service names to lowercase with underscores', async () => { + test('aggregates multiple services within a category (strict AND)', async () => { const mockWindow = createMockWindow(); - await createUsercentricsSource(mockWindow, mockElb); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('marketing', true, 'explicit', 'Google Analytics'), + makeV2Service('marketing', false, 'explicit', 'Meta Pixel'), + ]), + ); - mockWindow.__dispatchEvent('ucEvent', { - event: 'consent_status', - type: 'explicit', - ucCategory: { essential: 'partial' }, - 'My Custom Service': true, + await createUsercentricsSource(mockWindow, mockElb, { + settings: { apiVersion: 'v2' }, }); - expect(consentCalls[0].consent).toHaveProperty('my_custom_service', true); + // One denied service in 'marketing' denies the whole category. + expect(consentCalls[0].consent).toEqual({ marketing: false }); }); - test('merges boolean ucCategory entries with service keys', async () => { + test('a single accepted service yields its category as true', async () => { const mockWindow = createMockWindow(); - await createUsercentricsSource(mockWindow, mockElb); - - // ucCategory has mix of boolean and non-boolean - // Boolean entries from ucCategory should be included - mockWindow.__dispatchEvent('ucEvent', { - event: 'consent_status', - type: 'explicit', - ucCategory: { - essential: true, // boolean - include - marketing: 'partial', // non-boolean - skip (use services) - }, - 'Facebook Pixel': true, - }); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'explicit', 'My Custom Service'), + ]), + ); - expect(consentCalls[0].consent).toEqual({ - essential: true, - facebook_pixel: true, + await createUsercentricsSource(mockWindow, mockElb, { + settings: { apiVersion: 'v2' }, }); + + expect(consentCalls[0].consent).toEqual({ essential: true }); }); - test('applies categoryMap to boolean ucCategory entries in service-level mode', async () => { + test('applies categoryMap to aggregated categories', async () => { const mockWindow = createMockWindow(); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'explicit'), + makeV2Service('marketing', true, 'explicit', 'Facebook Pixel'), + ]), + ); + await createUsercentricsSource(mockWindow, mockElb, { settings: { + apiVersion: 'v2', categoryMap: { essential: 'functional' }, }, }); - mockWindow.__dispatchEvent('ucEvent', { - event: 'consent_status', - type: 'explicit', - ucCategory: { - essential: true, // boolean - mapped to 'functional' - marketing: 'partial', // non-boolean - skipped - }, - 'Facebook Pixel': true, - }); - expect(consentCalls[0].consent).toEqual({ functional: true, - facebook_pixel: true, + marketing: true, }); }); }); describe('event handling', () => { - test('handles consent change events', async () => { + test('re-reads and publishes on a UC_UI_CMP_EVENT decision', async () => { const mockWindow = createMockWindow(); - await createUsercentricsSource(mockWindow, mockElb); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'explicit'), + makeV2Service('marketing', false, 'explicit'), + ]), + ); - // First consent - mockWindow.__dispatchEvent('ucEvent', inputs.minimalConsent); + await createUsercentricsSource(mockWindow, mockElb, { + settings: { apiVersion: 'v2' }, + }); + + // Static read at init published once. expect(consentCalls).toHaveLength(1); - expect(consentCalls[0].consent).toEqual(outputs.minimalConsentMapped); + expect(consentCalls[0].consent).toEqual({ + essential: true, + marketing: false, + }); + + // User accepts marketing; the getter now returns the updated services. + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'explicit'), + makeV2Service('marketing', true, 'explicit'), + ]), + ); + mockWindow.__dispatchCmpEvent({ source: 'button', type: 'ACCEPT_ALL' }); - // User updates consent - mockWindow.__dispatchEvent('ucEvent', inputs.fullConsent); expect(consentCalls).toHaveLength(2); - expect(consentCalls[1].consent).toEqual(outputs.fullConsentMapped); + expect(consentCalls[1].consent).toEqual({ + essential: true, + marketing: true, + }); }); test('handles consent withdrawal (revocation)', async () => { const mockWindow = createMockWindow(); - await createUsercentricsSource(mockWindow, mockElb); - - // User initially accepts all - mockWindow.__dispatchEvent('ucEvent', inputs.fullConsent); - expect(consentCalls[0].consent).toEqual(outputs.fullConsentMapped); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'explicit'), + makeV2Service('marketing', true, 'explicit'), + ]), + ); - // User revokes marketing - mockWindow.__dispatchEvent('ucEvent', inputs.partialConsent); - expect(consentCalls[1].consent).toEqual(outputs.partialConsentMapped); - }); + await createUsercentricsSource(mockWindow, mockElb, { + settings: { apiVersion: 'v2' }, + }); - test('handles multiple consent changes', async () => { - const mockWindow = createMockWindow(); - await createUsercentricsSource(mockWindow, mockElb); + expect(consentCalls[0].consent).toEqual({ + essential: true, + marketing: true, + }); - mockWindow.__dispatchEvent('ucEvent', inputs.minimalConsent); - mockWindow.__dispatchEvent('ucEvent', inputs.partialConsent); - mockWindow.__dispatchEvent('ucEvent', inputs.fullConsent); + // User revokes marketing. + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'explicit'), + makeV2Service('marketing', false, 'explicit'), + ]), + ); + mockWindow.__dispatchCmpEvent({ source: 'button', type: 'SAVE' }); - expect(consentCalls).toHaveLength(3); - expect(consentCalls[2].consent).toEqual(outputs.fullConsentMapped); + expect(consentCalls[1].consent).toEqual({ + essential: true, + marketing: false, + }); }); - }); - describe('cleanup', () => { - test('destroy removes event listener', async () => { + test('ignores non-decision CMP events', async () => { const mockWindow = createMockWindow(); - const source = await createUsercentricsSource(mockWindow, mockElb); + mockWindow.__setUcUi( + makeUcUi([makeV2Service('marketing', true, 'explicit')]), + ); - await source.destroy?.({ - id: 'test', - config: source.config, - env: {} as never, - logger: createMockLogger(), + await createUsercentricsSource(mockWindow, mockElb, { + settings: { apiVersion: 'v2' }, }); + const before = consentCalls.length; - expect(mockWindow.removeEventListener).toHaveBeenCalledWith( - 'ucEvent', - expect.any(Function), - ); + mockWindow.__dispatchCmpEvent({ source: 'first', type: 'CMP_SHOWN' }); + + expect(consentCalls).toHaveLength(before); }); + }); - test('destroy removes listener for custom event name', async () => { + describe('cleanup', () => { + test('destroy removes both official V2 listeners', async () => { const mockWindow = createMockWindow(); + mockWindow.__setUcUi( + makeUcUi([makeV2Service('marketing', true, 'explicit')]), + ); const source = await createUsercentricsSource(mockWindow, mockElb, { - settings: { eventName: 'myConsentEvent' }, + settings: { apiVersion: 'v2' }, }); await source.destroy?.({ @@ -404,7 +440,11 @@ describe('Usercentrics Source', () => { }); expect(mockWindow.removeEventListener).toHaveBeenCalledWith( - 'myConsentEvent', + 'UC_UI_INITIALIZED', + expect.any(Function), + ); + expect(mockWindow.removeEventListener).toHaveBeenCalledWith( + 'UC_UI_CMP_EVENT', expect.any(Function), ); }); @@ -445,28 +485,6 @@ describe('Usercentrics Source', () => { (mockWindow as unknown as { __ucCmp: UsercentricsV3Api }).__ucCmp = ucCmp; } - function withUcUi(mockWindow: MockWindow, ucUi: UsercentricsV2Api): void { - (mockWindow as unknown as { UC_UI: UsercentricsV2Api }).UC_UI = ucUi; - } - - /** - * Dispatch a V3 event (detail shape differs from V2). - */ - function dispatchV3Event( - mockWindow: MockWindow, - eventName: string, - detail: { source: string; type: string }, - ): void { - ( - mockWindow as unknown as { - __dispatchEvent: ( - event: string, - detail: { source: string; type: string }, - ) => void; - } - ).__dispatchEvent(eventName, detail); - } - /** * Drain the microtask queue so awaited V3 calls settle. Fake timers are * on globally, so a setTimeout flush would hang. @@ -493,14 +511,12 @@ describe('Usercentrics Source', () => { }); // V2 also present, but should be ignored because __ucCmp wins in auto mode. - const v2Services: UsercentricsV2Service[] = [ - { categorySlug: 'essential', consent: { status: true } }, - { categorySlug: 'marketing', consent: { status: true } }, // would differ from V3 - ]; - withUcUi(mockWindow, { - isInitialized: () => true, - getServicesBaseInfo: () => v2Services, - }); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'explicit'), + makeV2Service('marketing', true, 'explicit'), + ]), + ); await createUsercentricsSource(mockWindow, mockElb, { settings: { apiVersion: 'auto' }, @@ -515,21 +531,17 @@ describe('Usercentrics Source', () => { }); }); - test('auto + only V2 present: V2 fires via event (static read suppressed by explicitOnly)', async () => { + test('auto + only V2 present (explicit history): static read publishes the returning visitor', async () => { const mockWindow = createMockWindow(); - const v2Services: UsercentricsV2Service[] = [ - { categorySlug: 'essential', consent: { status: true } }, - { categorySlug: 'marketing', consent: { status: false } }, - ]; - withUcUi(mockWindow, { - isInitialized: () => true, - getServicesBaseInfo: () => v2Services, - }); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'explicit'), + makeV2Service('marketing', false, 'explicit'), + ]), + ); await createUsercentricsSource(mockWindow, mockElb, { - // Default explicitOnly=true would drop the implicit static read. - // Use false here to prove the V2 static read still flows end-to-end. - settings: { apiVersion: 'auto', explicitOnly: false }, + settings: { apiVersion: 'auto' }, }); await flushPromises(); @@ -540,44 +552,53 @@ describe('Usercentrics Source', () => { }); }); - test('auto + neither present: both listeners registered', async () => { + test('auto + only V2 present (implicit-only history): first visit does NOT publish', async () => { const mockWindow = createMockWindow(); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'implicit'), + makeV2Service('marketing', false, 'implicit'), + ]), + ); await createUsercentricsSource(mockWindow, mockElb, { settings: { apiVersion: 'auto' }, }); await flushPromises(); - // Nothing fired yet — no static state on either API. expect(consentCalls).toHaveLength(0); + }); - // V2 event fires. - mockWindow.__dispatchEvent('ucEvent', { - event: 'consent_status', - type: 'explicit', - ucCategory: { essential: true, marketing: false }, - }); - expect(consentCalls).toHaveLength(1); + test('auto + neither present: both listeners registered', async () => { + const mockWindow = createMockWindow(); - // Now V3 API becomes available and dispatches its event. - const v3Details: UsercentricsV3ConsentDetails = { - consent: buildConsentData({ type: 'EXPLICIT' }), - categories: { - essential: buildCategory('ALL_ACCEPTED', 'Essential'), - }, - }; - withUcCmp(mockWindow, { - isInitialized: jest.fn().mockResolvedValue(true), - getConsentDetails: jest.fn().mockResolvedValue(v3Details), - }); - dispatchV3Event(mockWindow, 'UC_UI_CMP_EVENT', { - source: 'CMP', - type: 'ACCEPT_ALL', + await createUsercentricsSource(mockWindow, mockElb, { + settings: { apiVersion: 'auto' }, }); await flushPromises(); - expect(consentCalls).toHaveLength(2); - expect(consentCalls[1].consent).toEqual({ essential: true }); + // Nothing fired yet — no static state on either API. + expect(consentCalls).toHaveLength(0); + + // Both adapter event buses are wired. + expect(mockWindow.addEventListener).toHaveBeenCalledWith( + 'UC_UI_INITIALIZED', + expect.any(Function), + ); + expect(mockWindow.addEventListener).toHaveBeenCalledWith( + 'UC_UI_CMP_EVENT', + expect.any(Function), + ); + + // V2 API now becomes available with an explicit decision; the + // UC_UI_INITIALIZED lifecycle event drives the V2 static read. + mockWindow.__setUcUi( + makeUcUi([makeV2Service('essential', true, 'explicit')]), + ); + mockWindow.__dispatchInitialized(); + + expect(consentCalls).toHaveLength(1); + expect(consentCalls[0].consent).toEqual({ essential: true }); }); test('apiVersion=v2 + only V3 present: V3 is ignored, V2 listener still works', async () => { @@ -597,12 +618,15 @@ describe('Usercentrics Source', () => { expect(getConsentDetails).not.toHaveBeenCalled(); expect(consentCalls).toHaveLength(0); - // V2 listener is active — dispatching ucEvent reaches it. - mockWindow.__dispatchEvent('ucEvent', { - event: 'consent_status', - type: 'explicit', - ucCategory: { essential: true, marketing: false }, - }); + // V2 API becomes available; a decision event drives the V2 read. + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'explicit'), + makeV2Service('marketing', false, 'explicit'), + ]), + ); + mockWindow.__dispatchCmpEvent({ source: 'button', type: 'ACCEPT_ALL' }); + expect(consentCalls).toHaveLength(1); expect(consentCalls[0].consent).toEqual({ essential: true, @@ -613,12 +637,13 @@ describe('Usercentrics Source', () => { test('apiVersion=v3 + only V2 present: V2 is ignored, V3 listener still works', async () => { const mockWindow = createMockWindow(); const getServicesBaseInfo = jest.fn((): UsercentricsV2Service[] => [ - { categorySlug: 'essential', consent: { status: true } }, + makeV2Service('essential', true, 'explicit'), ]); - withUcUi(mockWindow, { + const ucUi: UsercentricsV2Api = { isInitialized: () => true, getServicesBaseInfo, - }); + }; + mockWindow.__setUcUi(ucUi); await createUsercentricsSource(mockWindow, mockElb, { settings: { apiVersion: 'v3' }, @@ -640,10 +665,7 @@ describe('Usercentrics Source', () => { isInitialized: jest.fn().mockResolvedValue(true), getConsentDetails: jest.fn().mockResolvedValue(v3Details), }); - dispatchV3Event(mockWindow, 'UC_UI_CMP_EVENT', { - source: 'CMP', - type: 'ACCEPT_ALL', - }); + mockWindow.__dispatchCmpEvent({ source: 'button', type: 'ACCEPT_ALL' }); await flushPromises(); expect(consentCalls).toHaveLength(1); @@ -654,17 +676,15 @@ describe('Usercentrics Source', () => { describe('factory side-effect-free (init hygiene)', () => { test('factory attaches no listener and emits no consent until init() runs', async () => { const mockWindow = createMockWindow(); - // V2 already initialized: a static read WOULD emit if the factory did it. - (mockWindow as unknown as { UC_UI: UsercentricsV2Api }).UC_UI = { - isInitialized: () => true, - getServicesBaseInfo: (): UsercentricsV2Service[] => [ - { categorySlug: 'essential', consent: { status: true } }, - ], - }; + // V2 already initialized with an explicit decision: a static read WOULD + // emit if the factory did it. + mockWindow.__setUcUi( + makeUcUi([makeV2Service('essential', true, 'explicit')]), + ); const source = await sourceUsercentrics({ collector: {} as never, - config: { settings: { apiVersion: 'v2', explicitOnly: false } }, + config: { settings: { apiVersion: 'v2' } }, env: { push: mockElb, command: mockElb, @@ -681,11 +701,11 @@ describe('Usercentrics Source', () => { expect(mockWindow.addEventListener).not.toHaveBeenCalled(); expect(consentCalls).toHaveLength(0); - // init() (Pass 2) performs the adapter setup: listener + static read emit. + // init() (Pass 2) performs the adapter setup: listeners + static read emit. await source.init?.(); expect(mockWindow.addEventListener).toHaveBeenCalledWith( - 'ucEvent', + 'UC_UI_INITIALIZED', expect.any(Function), ); expect(consentCalls).toHaveLength(1); @@ -702,34 +722,10 @@ describe('Usercentrics Source', () => { command: mockElb, elb: mockElb, window: undefined, - logger: { - error: () => {}, - warn: () => {}, - info: () => {}, - debug: () => {}, - json: () => {}, - throw: (m: string | Error) => { - throw typeof m === 'string' ? new Error(m) : m; - }, - scope: function () { - return this; - }, - }, + logger: createMockLogger(), }, id: 'test-usercentrics', - logger: { - error: () => {}, - warn: () => {}, - info: () => {}, - debug: () => {}, - json: () => {}, - throw: (m: string | Error) => { - throw typeof m === 'string' ? new Error(m) : m; - }, - scope: function () { - return this; - }, - }, + logger: createMockLogger(), withScope: async (_r, _resp, body) => body({} as never), }); diff --git a/packages/web/sources/cmps/usercentrics/src/__tests__/stepExamples.test.ts b/packages/web/sources/cmps/usercentrics/src/__tests__/stepExamples.test.ts index 837f44d12..8e7591f6a 100644 --- a/packages/web/sources/cmps/usercentrics/src/__tests__/stepExamples.test.ts +++ b/packages/web/sources/cmps/usercentrics/src/__tests__/stepExamples.test.ts @@ -2,11 +2,34 @@ import type { Collector, Elb } from '@walkeros/core'; import { createMockLogger } from '@walkeros/core'; import { sourceUsercentrics } from '../index'; import { examples } from '../dev'; +import type { UsercentricsV2Api, UsercentricsV2Service } from '../types'; + +/** Typed access to the `UC_UI` global on the jsdom window. */ +interface UcWindow { + UC_UI?: UsercentricsV2Api; +} + +function ucWindow(): UcWindow { + return window as unknown as UcWindow; +} describe('Step Examples', () => { + beforeEach(() => { + ucWindow().UC_UI = undefined; + }); + + afterEach(() => { + ucWindow().UC_UI = undefined; + }); + it.each(Object.entries(examples.step))('%s', async (_name, example) => { - const content = example.in as Record; - const mapping = example.mapping as Record | undefined; + const services = example.in as UsercentricsV2Service[]; + const mapping = example.mapping as + | { settings?: Record } + | undefined; + const dispatch = + (example.trigger?.options as { dispatch?: string } | undefined) + ?.dispatch ?? 'init'; const mockElb = jest.fn(async () => ({ ok: true, @@ -19,16 +42,21 @@ describe('Step Examples', () => { allowed: true, } as unknown as Collector.Instance; + const ucUi: UsercentricsV2Api = { + isInitialized: () => true, + getServicesBaseInfo: () => services, + }; + + // 'init': UC_UI is present when the source runs, so the static read at + // init emits the snapshot. 'cmp': attach UC_UI only after init so the + // static read is a no-op, then drive the consent-change path. + if (dispatch === 'init') ucWindow().UC_UI = ucUi; + const source = await sourceUsercentrics({ collector: collectorStub, config: { settings: { - ...(mapping?.eventName - ? { eventName: mapping.eventName as string } - : {}), - ...(mapping?.categoryMap - ? { categoryMap: mapping.categoryMap as Record } - : {}), + ...(mapping?.settings || {}), }, }, env: { @@ -43,14 +71,19 @@ describe('Step Examples', () => { withScope: async (_r, _resp, body) => body({} as never), }); - // Adapter setup (listener attach) happens in init(), not the factory. + // Adapter setup (listener attach + static read) happens in init(). await source.init?.(); - // Dispatch CMP event — source listener catches it - const eventName = (mapping?.eventName as string) || 'ucEvent'; - window.dispatchEvent(new CustomEvent(eventName, { detail: content })); + if (dispatch === 'cmp') { + ucWindow().UC_UI = ucUi; + window.dispatchEvent( + new CustomEvent('UC_UI_CMP_EVENT', { + detail: { source: 'button', type: 'ACCEPT_ALL' }, + }), + ); + } - // Source pushes via detached elb chain — yield for it + // Source pushes via detached elb chain — yield for it. for (let i = 0; i < 10 && mockElb.mock.calls.length === 0; i++) { await Promise.resolve(); } diff --git a/packages/web/sources/cmps/usercentrics/src/__tests__/test-utils.ts b/packages/web/sources/cmps/usercentrics/src/__tests__/test-utils.ts index 641bfde9c..2509cb100 100644 --- a/packages/web/sources/cmps/usercentrics/src/__tests__/test-utils.ts +++ b/packages/web/sources/cmps/usercentrics/src/__tests__/test-utils.ts @@ -1,7 +1,13 @@ import type { WalkerOS, Elb, Collector, Source } from '@walkeros/core'; import { createMockLogger } from '@walkeros/core'; import { sourceUsercentrics } from '../index'; -import type { Types, UsercentricsEventDetail } from '../types'; +import type { + Types, + UsercentricsConsentType, + UsercentricsV2Api, + UsercentricsV2Service, + UsercentricsV3CmpEventDetail, +} from '../types'; /** * Track consent commands called via elb @@ -11,10 +17,19 @@ export interface ConsentCall { } /** - * Mock window with test helpers for dispatching events + * Mock window with test helpers for dispatching the official Usercentrics V2 + * events (`UC_UI_INITIALIZED`, `UC_UI_CMP_EVENT`) and for attaching a typed + * `UC_UI` API mock. No scattered casts: the mock is built against a typed + * interface and cast once at the `env.window` boundary inside the source + * factory helper. */ export interface MockWindow extends Window { - __dispatchEvent: (event: string, detail?: UsercentricsEventDetail) => void; + /** Dispatch the synchronous `UC_UI_INITIALIZED` lifecycle event. */ + __dispatchInitialized: () => void; + /** Dispatch a `UC_UI_CMP_EVENT` with the given V3 CMP event detail. */ + __dispatchCmpEvent: (detail: UsercentricsV3CmpEventDetail) => void; + /** Attach (or replace) the `window.UC_UI` V2 API mock. */ + __setUcUi: (ucUi: UsercentricsV2Api) => void; } /** @@ -34,28 +49,83 @@ export function createMockElb(consentCalls: ConsentCall[]) { } /** - * Create a mock window that supports addEventListener/removeEventListener - * and can dispatch Usercentrics events for testing. + * Build a single V2 service entry. When `historyType` is provided the service + * carries a one-entry consent history with that type, which the explicit-gate + * inspects. `name` is the service display name surfaced as a service-level key. + */ +export function makeV2Service( + categorySlug: string, + status: boolean, + historyType?: UsercentricsConsentType, + name?: string, +): UsercentricsV2Service { + return { + categorySlug, + ...(name !== undefined ? { name } : {}), + consent: { + status, + ...(historyType !== undefined + ? { history: [{ type: historyType, status }] } + : {}), + }, + }; +} + +/** + * Build a UC_UI V2 API mock from a fixed service list. Defaults to + * `isInitialized() === true`; override per-case for the not-yet-initialized + * path. + */ +export function makeUcUi( + services: UsercentricsV2Service[], + overrides: Partial = {}, +): UsercentricsV2Api { + return { + isInitialized: () => true, + getServicesBaseInfo: () => services, + ...overrides, + }; +} + +/** + * Create a mock window that supports addEventListener/removeEventListener, + * carries a typed optional `UC_UI`, and can dispatch the official Usercentrics + * events for testing. */ export function createMockWindow(): MockWindow { const listeners: Record void>> = {}; + let ucUi: UsercentricsV2Api | undefined; - const mockWindow = { + const mockWindow: Pick & { + UC_UI?: UsercentricsV2Api; + __dispatchInitialized: MockWindow['__dispatchInitialized']; + __dispatchCmpEvent: MockWindow['__dispatchCmpEvent']; + __setUcUi: MockWindow['__setUcUi']; + } = { + get UC_UI() { + return ucUi; + }, addEventListener: jest.fn((event: string, handler: (e: Event) => void) => { if (!listeners[event]) listeners[event] = []; listeners[event].push(handler); - }), + }) as unknown as Window['addEventListener'], removeEventListener: jest.fn( (event: string, handler: (e: Event) => void) => { if (listeners[event]) { listeners[event] = listeners[event].filter((h) => h !== handler); } }, - ), - // Helper to dispatch events in tests - __dispatchEvent: (event: string, detail?: UsercentricsEventDetail) => { - const e = detail ? new CustomEvent(event, { detail }) : new Event(event); - listeners[event]?.forEach((handler) => handler(e)); + ) as unknown as Window['removeEventListener'], + __setUcUi: (next: UsercentricsV2Api) => { + ucUi = next; + }, + __dispatchInitialized: () => { + const e = new Event('UC_UI_INITIALIZED'); + listeners['UC_UI_INITIALIZED']?.forEach((handler) => handler(e)); + }, + __dispatchCmpEvent: (detail: UsercentricsV3CmpEventDetail) => { + const e = new CustomEvent('UC_UI_CMP_EVENT', { detail }); + listeners['UC_UI_CMP_EVENT']?.forEach((handler) => handler(e)); }, }; diff --git a/packages/web/sources/cmps/usercentrics/src/__tests__/v2.test.ts b/packages/web/sources/cmps/usercentrics/src/__tests__/v2.test.ts index 0ed6b34f0..ac462b2d7 100644 --- a/packages/web/sources/cmps/usercentrics/src/__tests__/v2.test.ts +++ b/packages/web/sources/cmps/usercentrics/src/__tests__/v2.test.ts @@ -1,14 +1,17 @@ import { createMockLogger } from '@walkeros/core'; -import { setupV2Adapter } from '../lib/v2'; -import type { - Settings, - UsercentricsV2Api, - UsercentricsV2Service, -} from '../types'; +import { + buildDetailFromServices, + hasExplicitDecision, + setupV2Adapter, +} from '../lib/v2'; +import { parseConsent } from '../lib/parseConsent'; +import type { Settings, UsercentricsV2Service } from '../types'; import { ConsentCall, createMockElb, createMockWindow, + makeUcUi, + makeV2Service, MockWindow, } from './test-utils'; @@ -18,7 +21,6 @@ import { */ function buildSettings(overrides: Partial = {}): Settings { return { - eventName: 'ucEvent', explicitOnly: true, categoryMap: {}, apiVersion: 'auto', @@ -27,15 +29,85 @@ function buildSettings(overrides: Partial = {}): Settings { }; } -/** - * Attach a UC_UI mock onto the MockWindow. `Window.UC_UI` is now optional and - * typed as the local minimal `UsercentricsV2Api` — no external-types cast. - */ -function withUcUi(mockWindow: MockWindow, ucUi: UsercentricsV2Api): void { - (mockWindow as unknown as { UC_UI: UsercentricsV2Api }).UC_UI = ucUi; -} +describe('hasExplicitDecision', () => { + test('returns true when any service has an explicit history entry', () => { + const services: UsercentricsV2Service[] = [ + makeV2Service('essential', true, 'implicit'), + makeV2Service('marketing', true, 'explicit'), + ]; + expect(hasExplicitDecision(services)).toBe(true); + }); + + test('returns false when every history entry is implicit', () => { + const services: UsercentricsV2Service[] = [ + makeV2Service('essential', true, 'implicit'), + makeV2Service('marketing', false, 'implicit'), + ]; + expect(hasExplicitDecision(services)).toBe(false); + }); + + test('returns false when history is missing or empty', () => { + const services: UsercentricsV2Service[] = [ + { categorySlug: 'essential', consent: { status: true } }, + { categorySlug: 'marketing', consent: { status: false, history: [] } }, + ]; + expect(hasExplicitDecision(services)).toBe(false); + }); + + test('is case-insensitive on the type value', () => { + // The live SDK can report the consent type in upper case ('EXPLICIT'). + // hasExplicitDecision lowercases before comparing, so an uppercase entry + // still counts as an explicit decision. JSON-parse a fixture so the + // uppercase value reaches the helper as a UsercentricsV2Service without a + // literal-narrowing cast. + const services: UsercentricsV2Service[] = JSON.parse( + JSON.stringify([ + { + categorySlug: 'marketing', + consent: { + status: true, + history: [{ type: 'EXPLICIT', status: true }], + }, + }, + ]), + ); + expect(hasExplicitDecision(services)).toBe(true); + }); +}); + +describe('buildDetailFromServices', () => { + test('explicit history → type explicit, strict-AND ucCategory, per-service name keys', () => { + const services: UsercentricsV2Service[] = [ + makeV2Service('essential', true, 'explicit', 'Essential Cookie'), + makeV2Service('marketing', true, 'explicit', 'Google Ads'), + makeV2Service('marketing', false, 'implicit', 'Meta Pixel'), + ]; + + const detail = buildDetailFromServices(services); + + expect(detail.event).toBe('consent_status'); + expect(detail.type).toBe('explicit'); + // marketing has one denied service → category denied (strict AND). + expect(detail.ucCategory).toEqual({ essential: true, marketing: false }); + expect(detail['Essential Cookie']).toBe(true); + expect(detail['Google Ads']).toBe(true); + expect(detail['Meta Pixel']).toBe(false); + }); + + test('only-implicit history → type implicit', () => { + const services: UsercentricsV2Service[] = [ + makeV2Service('essential', true, 'implicit'), + makeV2Service('marketing', false, 'implicit'), + ]; -describe('V2 adapter (setupV2Adapter)', () => { + const detail = buildDetailFromServices(services); + + expect(detail.type).toBe('implicit'); + expect(detail.ucCategory).toEqual({ essential: true, marketing: false }); + }); +}); + +describe('setupV2Adapter (official events)', () => { let consentCalls: ConsentCall[]; let mockElb: ReturnType; @@ -44,204 +116,231 @@ describe('V2 adapter (setupV2Adapter)', () => { mockElb = createMockElb(consentCalls); }); - describe('post-init static read (implicit type, default explicitOnly)', () => { - test('default explicitOnly=true suppresses the static read', () => { - const mockWindow = createMockWindow(); - const services: UsercentricsV2Service[] = [ - { categorySlug: 'essential', consent: { status: true } }, - { categorySlug: 'marketing', consent: { status: false } }, - ]; - withUcUi(mockWindow, { - isInitialized: () => true, - getServicesBaseInfo: () => services, - }); - - const cleanup = setupV2Adapter({ - window: mockWindow as unknown as Window & typeof globalThis, - elb: mockElb, - settings: buildSettings(), // explicitOnly: true by default - logger: createMockLogger(), - }); - - // Static read emits 'implicit'; explicitOnly=true drops it. - expect(mockElb).not.toHaveBeenCalled(); - - cleanup(); + function run( + mockWindow: MockWindow, + settings: Settings = buildSettings(), + ): () => void { + return setupV2Adapter({ + window: mockWindow as unknown as Window & typeof globalThis, + elb: mockElb, + settings, + logger: createMockLogger(), }); + } - test('explicitOnly=false surfaces the static read as implicit snapshot', () => { - const mockWindow = createMockWindow(); - const services: UsercentricsV2Service[] = [ - { categorySlug: 'essential', consent: { status: true } }, - { categorySlug: 'marketing', consent: { status: false } }, - ]; - withUcUi(mockWindow, { - isInitialized: () => true, - getServicesBaseInfo: () => services, - }); - - const cleanup = setupV2Adapter({ - window: mockWindow as unknown as Window & typeof globalThis, - elb: mockElb, - settings: buildSettings({ explicitOnly: false }), - logger: createMockLogger(), - }); - - expect(mockElb).toHaveBeenCalledWith('walker consent', { - essential: true, - marketing: false, - }); - expect(mockElb).toHaveBeenCalledTimes(1); - - cleanup(); - }); + test('UC_UI_INITIALIZED with explicit history publishes (default explicitOnly)', () => { + const mockWindow = createMockWindow(); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'explicit'), + makeV2Service('marketing', false, 'explicit'), + ]), + ); - test('aggregates services by categorySlug with strict AND logic', () => { - const mockWindow = createMockWindow(); - // Two services in 'marketing': one true, one false → category is false. - // Two services in 'essential': both true → category is true. - // One service in 'functional': false → category is false. - const services: UsercentricsV2Service[] = [ - { categorySlug: 'essential', consent: { status: true } }, - { categorySlug: 'essential', consent: { status: true } }, - { categorySlug: 'marketing', consent: { status: true } }, - { categorySlug: 'marketing', consent: { status: false } }, - { categorySlug: 'functional', consent: { status: false } }, - ]; - withUcUi(mockWindow, { - isInitialized: () => true, - getServicesBaseInfo: () => services, - }); - - const cleanup = setupV2Adapter({ - window: mockWindow as unknown as Window & typeof globalThis, - elb: mockElb, - // explicitOnly=false so the static read is actually emitted here. - settings: buildSettings({ explicitOnly: false }), - logger: createMockLogger(), - }); - - expect(mockElb).toHaveBeenCalledWith('walker consent', { - essential: true, - marketing: false, - functional: false, - }); - expect(mockElb).toHaveBeenCalledTimes(1); - - cleanup(); + const cleanup = run(mockWindow); + + // The initial static read already published once because UC_UI is set. + expect(mockElb).toHaveBeenCalledTimes(1); + mockElb.mockClear(); + + mockWindow.__dispatchInitialized(); + + expect(mockElb).toHaveBeenCalledWith('walker consent', { + essential: true, + marketing: false, }); + expect(mockElb).toHaveBeenCalledTimes(1); + + cleanup(); + }); + + test('UC_UI_INITIALIZED with only-implicit history (first visit) does NOT publish', () => { + const mockWindow = createMockWindow(); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'implicit'), + makeV2Service('marketing', false, 'implicit'), + ]), + ); - test('skips static read when UC_UI.isInitialized() returns false', () => { - const mockWindow = createMockWindow(); - withUcUi(mockWindow, { - isInitialized: () => false, - getServicesBaseInfo: () => [ - { categorySlug: 'essential', consent: { status: true } }, - ], - }); - - const cleanup = setupV2Adapter({ - window: mockWindow as unknown as Window & typeof globalThis, - elb: mockElb, - settings: buildSettings({ explicitOnly: false }), - logger: createMockLogger(), - }); - - expect(mockElb).not.toHaveBeenCalled(); - - cleanup(); + const cleanup = run(mockWindow); + mockWindow.__dispatchInitialized(); + + expect(mockElb).not.toHaveBeenCalled(); + + cleanup(); + }); + + test('UC_UI_CMP_EVENT type ACCEPT_ALL re-reads and publishes', () => { + const mockWindow = createMockWindow(); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'explicit'), + makeV2Service('marketing', true, 'explicit'), + ]), + ); + + const cleanup = run(mockWindow); + mockElb.mockClear(); + + mockWindow.__dispatchCmpEvent({ source: 'button', type: 'ACCEPT_ALL' }); + + expect(mockElb).toHaveBeenCalledWith('walker consent', { + essential: true, + marketing: true, }); + expect(mockElb).toHaveBeenCalledTimes(1); + + cleanup(); + }); + + test('UC_UI_CMP_EVENT type CMP_SHOWN does NOT publish', () => { + const mockWindow = createMockWindow(); + mockWindow.__setUcUi( + makeUcUi([makeV2Service('marketing', true, 'explicit')]), + ); + + const cleanup = run(mockWindow); + mockElb.mockClear(); + + mockWindow.__dispatchCmpEvent({ source: 'first', type: 'CMP_SHOWN' }); + + expect(mockElb).not.toHaveBeenCalled(); - test('skips static read when UC_UI is absent', () => { - const mockWindow = createMockWindow(); - // No UC_UI attached. + cleanup(); + }); + + test('publishes when isInitialized is absent but getServicesBaseInfo is available', () => { + // Every V2 API method is optional; a deployment may expose + // getServicesBaseInfo without isInitialized. Consent must still publish + // rather than be silently suppressed by an absent optional method. + const mockWindow = createMockWindow(); + mockWindow.__setUcUi( + makeUcUi([makeV2Service('marketing', true, 'explicit')], { + isInitialized: undefined, + }), + ); - const cleanup = setupV2Adapter({ - window: mockWindow as unknown as Window & typeof globalThis, - elb: mockElb, - settings: buildSettings({ explicitOnly: false }), - logger: createMockLogger(), - }); + const cleanup = run(mockWindow); - expect(mockElb).not.toHaveBeenCalled(); + expect(mockElb).toHaveBeenCalledWith('walker consent', { marketing: true }); + expect(mockElb).toHaveBeenCalledTimes(1); + + cleanup(); + }); - cleanup(); + test('static read at init (already initialized) with explicit history publishes the made choice', () => { + const mockWindow = createMockWindow(); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'explicit'), + makeV2Service('marketing', false, 'explicit'), + ]), + ); + + const cleanup = run(mockWindow); + + expect(mockElb).toHaveBeenCalledWith('walker consent', { + essential: true, + marketing: false, }); + expect(mockElb).toHaveBeenCalledTimes(1); + + cleanup(); }); - describe('pre-init event listener', () => { - test('handles ucEvent dispatched after registration', () => { - const mockWindow = createMockWindow(); - // No UC_UI — simulate pre-init environment. + test('explicitOnly=false publishes even an implicit static snapshot', () => { + const mockWindow = createMockWindow(); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('essential', true, 'implicit'), + makeV2Service('marketing', false, 'implicit'), + ]), + ); - const cleanup = setupV2Adapter({ - window: mockWindow as unknown as Window & typeof globalThis, - elb: mockElb, - settings: buildSettings(), - logger: createMockLogger(), - }); + const cleanup = run(mockWindow, buildSettings({ explicitOnly: false })); - mockWindow.__dispatchEvent('ucEvent', { - event: 'consent_status', - type: 'explicit', - ucCategory: { essential: true, marketing: false }, - }); + expect(mockElb).toHaveBeenCalledWith('walker consent', { + essential: true, + marketing: false, + }); + expect(mockElb).toHaveBeenCalledTimes(1); - expect(mockElb).toHaveBeenCalledWith('walker consent', { - essential: true, - marketing: false, - }); - expect(mockElb).toHaveBeenCalledTimes(1); + cleanup(); + }); - cleanup(); + test('strict AND across services in one category (one denied → category denied)', () => { + const mockWindow = createMockWindow(); + mockWindow.__setUcUi( + makeUcUi([ + makeV2Service('marketing', true, 'explicit'), + makeV2Service('marketing', false, 'explicit'), + makeV2Service('essential', true, 'explicit'), + ]), + ); + + const cleanup = run(mockWindow); + + expect(mockElb).toHaveBeenCalledWith('walker consent', { + essential: true, + marketing: false, }); + expect(mockElb).toHaveBeenCalledTimes(1); + + cleanup(); + }); - test('respects custom eventName setting', () => { - const mockWindow = createMockWindow(); - - const cleanup = setupV2Adapter({ - window: mockWindow as unknown as Window & typeof globalThis, - elb: mockElb, - settings: buildSettings({ eventName: 'UC_SDK_EVENT' }), - logger: createMockLogger(), - }); - - mockWindow.__dispatchEvent('UC_SDK_EVENT', { - event: 'consent_status', - type: 'explicit', - ucCategory: { essential: true, marketing: false }, - }); - - expect(mockElb).toHaveBeenCalledTimes(1); - expect(mockElb).toHaveBeenCalledWith('walker consent', { - essential: true, - marketing: false, - }); - - cleanup(); + test('service-level: per-service name keys carry through buildDetailFromServices into parseConsent', () => { + // buildDetailFromServices surfaces each named service's status as a + // top-level key. parseConsent reads those service-level keys whenever the + // detail is service-level (ucCategory has a non-boolean entry), normalizing + // the names to consent keys. + const detail = buildDetailFromServices([ + makeV2Service('marketing', true, 'explicit', 'Google Analytics'), + makeV2Service('marketing', false, 'explicit', 'Meta Pixel'), + ]); + + expect(detail['Google Analytics']).toBe(true); + expect(detail['Meta Pixel']).toBe(false); + + // Force parseConsent's service-level branch with a non-boolean ucCategory + // entry so the top-level service keys are consumed. + const serviceLevelDetail = { + ...detail, + ucCategory: { marketing: 'partial' }, + }; + const state = parseConsent(serviceLevelDetail, buildSettings()); + + expect(state).toMatchObject({ + google_analytics: true, + meta_pixel: false, }); }); - describe('cleanup', () => { - test('cleanup removes the event listener', () => { - const mockWindow = createMockWindow(); + test('cleanup removes both UC_UI_INITIALIZED and UC_UI_CMP_EVENT listeners', () => { + const mockWindow = createMockWindow(); + mockWindow.__setUcUi( + makeUcUi([makeV2Service('marketing', true, 'explicit')]), + ); - const cleanup = setupV2Adapter({ - window: mockWindow as unknown as Window & typeof globalThis, - elb: mockElb, - settings: buildSettings(), - logger: createMockLogger(), - }); + const cleanup = run(mockWindow); + cleanup(); + mockElb.mockClear(); - cleanup(); + mockWindow.__dispatchInitialized(); + mockWindow.__dispatchCmpEvent({ source: 'button', type: 'ACCEPT_ALL' }); - mockWindow.__dispatchEvent('ucEvent', { - event: 'consent_status', - type: 'explicit', - ucCategory: { essential: true, marketing: false }, - }); + expect(mockElb).not.toHaveBeenCalled(); - expect(mockElb).not.toHaveBeenCalled(); - }); + expect(mockWindow.removeEventListener).toHaveBeenCalledWith( + 'UC_UI_INITIALIZED', + expect.any(Function), + ); + expect(mockWindow.removeEventListener).toHaveBeenCalledWith( + 'UC_UI_CMP_EVENT', + expect.any(Function), + ); + + cleanup(); }); }); diff --git a/packages/web/sources/cmps/usercentrics/src/__tests__/v3.test.ts b/packages/web/sources/cmps/usercentrics/src/__tests__/v3.test.ts index 0d6ecd45a..8f7a13aeb 100644 --- a/packages/web/sources/cmps/usercentrics/src/__tests__/v3.test.ts +++ b/packages/web/sources/cmps/usercentrics/src/__tests__/v3.test.ts @@ -22,7 +22,6 @@ import { */ function buildSettings(overrides: Partial = {}): Settings { return { - eventName: 'ucEvent', explicitOnly: true, categoryMap: {}, apiVersion: 'auto', @@ -75,23 +74,60 @@ function withUcCmp(mockWindow: MockWindow, ucCmp: MockUcCmp): void { } /** - * Dispatch a V3 event. The V3 event.detail is { source, type } — not the - * same shape as the V2 UsercentricsEventDetail. We go through the mock - * listener registry via a cast. + * Dispatch any registered window event by name, replaying every handler the + * adapter registered via addEventListener. Works for the V3 consent event + * (any custom name), the UC_UI_INITIALIZED lifecycle event, and so on, without + * needing a name-specific helper on the mock window. The mock's + * `addEventListener` is a jest.Mock, so we read its call log to find handlers. + */ +function dispatchWindowEvent( + mockWindow: MockWindow, + eventName: string, + event: Event, +): void { + const win = mockWindow as unknown as { + addEventListener: jest.Mock void]>; + removeEventListener: jest.Mock void]>; + }; + const removed = new Set( + win.removeEventListener.mock.calls + .filter(([name]) => name === eventName) + .map(([, handler]) => handler), + ); + // Replay only handlers that were added and not subsequently removed, so the + // helper honours cleanup() just like a real dispatch through the DOM bus. + win.addEventListener.mock.calls + .filter(([name]) => name === eventName) + .map(([, handler]) => handler) + .filter((handler) => !removed.has(handler)) + .forEach((handler) => handler(event)); +} + +/** + * Dispatch a V3 consent event. The V3 event.detail is { source, type } — not + * the same shape as the V2 UsercentricsEventDetail. */ function dispatchV3Event( mockWindow: MockWindow, eventName: string, detail: UsercentricsV3CmpEventDetail, ): void { - ( - mockWindow as unknown as { - __dispatchEvent: ( - event: string, - detail: UsercentricsV3CmpEventDetail, - ) => void; - } - ).__dispatchEvent(eventName, detail); + dispatchWindowEvent( + mockWindow, + eventName, + new CustomEvent(eventName, { detail }), + ); +} + +/** + * Dispatch the UC_UI_INITIALIZED lifecycle event (no detail payload). + */ +function dispatchInitialized(mockWindow: MockWindow): void { + dispatchWindowEvent( + mockWindow, + 'UC_UI_INITIALIZED', + new Event('UC_UI_INITIALIZED'), + ); } /** @@ -272,7 +308,7 @@ describe('V3 adapter (setupV3Adapter)', () => { }, } satisfies UsercentricsV3ConsentDetails); dispatchV3Event(mockWindow, 'UC_UI_CMP_EVENT', { - source: 'CMP', + source: 'button', type: 'ACCEPT_ALL', }); await flushPromises(); @@ -303,7 +339,7 @@ describe('V3 adapter (setupV3Adapter)', () => { }); describe('event-listener decision filter (Task C)', () => { - test('ACCEPT_ALL + source=CMP triggers publish', async () => { + test('publishes on UC_UI_CMP_EVENT with documented source (source:button, type:ACCEPT_ALL)', async () => { const mockWindow = createMockWindow(); const getConsentDetails = jest.fn().mockResolvedValue({ consent: buildConsentData({ type: 'EXPLICIT' }), @@ -323,19 +359,24 @@ describe('V3 adapter (setupV3Adapter)', () => { logger: createMockLogger(), }); + // Real SDK events carry a documented source ('button' here), NEVER 'CMP'. + // The adapter must gate on the decision type alone. dispatchV3Event(mockWindow, 'UC_UI_CMP_EVENT', { - source: 'CMP', + source: 'button', type: 'ACCEPT_ALL', }); await flushPromises(); expect(getConsentDetails).toHaveBeenCalledTimes(1); expect(mockElb).toHaveBeenCalledTimes(1); + expect(mockElb).toHaveBeenCalledWith('walker consent', { + essential: true, + }); cleanup(); }); - test('SAVE + source=CMP triggers publish', async () => { + test('publishes regardless of source (source:__ucCmp, type:SAVE)', async () => { const mockWindow = createMockWindow(); const getConsentDetails = jest.fn().mockResolvedValue({ consent: buildConsentData({ type: 'EXPLICIT' }), @@ -356,7 +397,7 @@ describe('V3 adapter (setupV3Adapter)', () => { }); dispatchV3Event(mockWindow, 'UC_UI_CMP_EVENT', { - source: 'CMP', + source: '__ucCmp', type: 'SAVE', }); await flushPromises(); @@ -367,7 +408,71 @@ describe('V3 adapter (setupV3Adapter)', () => { cleanup(); }); - test('DENY_ALL + source=CMP triggers publish', async () => { + test('ACCEPT_ALL triggers publish', async () => { + const mockWindow = createMockWindow(); + const getConsentDetails = jest.fn().mockResolvedValue({ + consent: buildConsentData({ type: 'EXPLICIT' }), + categories: { + essential: buildCategory('ALL_ACCEPTED', 'Essential'), + }, + } satisfies UsercentricsV3ConsentDetails); + withUcCmp(mockWindow, { + isInitialized: jest.fn().mockResolvedValue(false), + getConsentDetails, + }); + + const cleanup = await setupV3Adapter({ + window: mockWindow as unknown as Window & typeof globalThis, + elb: mockElb, + settings: buildSettings(), + logger: createMockLogger(), + }); + + dispatchV3Event(mockWindow, 'UC_UI_CMP_EVENT', { + source: 'button', + type: 'ACCEPT_ALL', + }); + await flushPromises(); + + expect(getConsentDetails).toHaveBeenCalledTimes(1); + expect(mockElb).toHaveBeenCalledTimes(1); + + cleanup(); + }); + + test('SAVE triggers publish', async () => { + const mockWindow = createMockWindow(); + const getConsentDetails = jest.fn().mockResolvedValue({ + consent: buildConsentData({ type: 'EXPLICIT' }), + categories: { + essential: buildCategory('ALL_ACCEPTED', 'Essential'), + }, + } satisfies UsercentricsV3ConsentDetails); + withUcCmp(mockWindow, { + isInitialized: jest.fn().mockResolvedValue(false), + getConsentDetails, + }); + + const cleanup = await setupV3Adapter({ + window: mockWindow as unknown as Window & typeof globalThis, + elb: mockElb, + settings: buildSettings(), + logger: createMockLogger(), + }); + + dispatchV3Event(mockWindow, 'UC_UI_CMP_EVENT', { + source: 'button', + type: 'SAVE', + }); + await flushPromises(); + + expect(getConsentDetails).toHaveBeenCalledTimes(1); + expect(mockElb).toHaveBeenCalledTimes(1); + + cleanup(); + }); + + test('DENY_ALL triggers publish', async () => { const mockWindow = createMockWindow(); const getConsentDetails = jest.fn().mockResolvedValue({ consent: buildConsentData({ type: 'EXPLICIT' }), @@ -388,7 +493,7 @@ describe('V3 adapter (setupV3Adapter)', () => { }); dispatchV3Event(mockWindow, 'UC_UI_CMP_EVENT', { - source: 'CMP', + source: 'button', type: 'DENY_ALL', }); await flushPromises(); @@ -402,7 +507,7 @@ describe('V3 adapter (setupV3Adapter)', () => { cleanup(); }); - test('CMP_SHOWN + source=CMP does NOT trigger publish', async () => { + test('CMP_SHOWN (non-decision) does NOT trigger publish', async () => { const mockWindow = createMockWindow(); const getConsentDetails = jest.fn(); withUcCmp(mockWindow, { @@ -418,7 +523,7 @@ describe('V3 adapter (setupV3Adapter)', () => { }); dispatchV3Event(mockWindow, 'UC_UI_CMP_EVENT', { - source: 'CMP', + source: 'button', type: 'CMP_SHOWN', }); await flushPromises(); @@ -429,7 +534,7 @@ describe('V3 adapter (setupV3Adapter)', () => { cleanup(); }); - test('event with source !== CMP does NOT trigger publish', async () => { + test('event without a decision type does NOT trigger publish', async () => { const mockWindow = createMockWindow(); const getConsentDetails = jest.fn(); withUcCmp(mockWindow, { @@ -444,9 +549,9 @@ describe('V3 adapter (setupV3Adapter)', () => { logger: createMockLogger(), }); + // Source is irrelevant; a missing/unknown type must not publish. dispatchV3Event(mockWindow, 'UC_UI_CMP_EVENT', { - source: 'SOMETHING_ELSE', - type: 'ACCEPT_ALL', + source: 'first', }); await flushPromises(); @@ -457,6 +562,69 @@ describe('V3 adapter (setupV3Adapter)', () => { }); }); + describe('UC_UI_INITIALIZED lifecycle (returning visitor)', () => { + test('UC_UI_INITIALIZED triggers a static read and publishes an existing EXPLICIT choice', async () => { + const mockWindow = createMockWindow(); + // Source loaded BEFORE __ucCmp existed: not initialized at setup time, + // so no static read runs during setup. The visitor already chose in a + // prior session, so no UC_UI_CMP_EVENT will fire either. + const getConsentDetails = jest.fn().mockResolvedValue({ + consent: buildConsentData({ type: 'EXPLICIT' }), + categories: { + essential: buildCategory('ALL_ACCEPTED', 'Essential'), + marketing: buildCategory('ALL_ACCEPTED', 'Marketing'), + }, + } satisfies UsercentricsV3ConsentDetails); + withUcCmp(mockWindow, { + isInitialized: jest.fn().mockResolvedValue(false), + getConsentDetails, + }); + + const cleanup = await setupV3Adapter({ + window: mockWindow as unknown as Window & typeof globalThis, + elb: mockElb, + settings: buildSettings(), + logger: createMockLogger(), + }); + + // No static read happened during setup. + expect(getConsentDetails).not.toHaveBeenCalled(); + expect(mockElb).not.toHaveBeenCalled(); + + // CMP becomes ready: the init listener performs the static read. + dispatchInitialized(mockWindow); + await flushPromises(); + + expect(getConsentDetails).toHaveBeenCalledTimes(1); + expect(mockElb).toHaveBeenCalledTimes(1); + expect(mockElb).toHaveBeenCalledWith('walker consent', { + essential: true, + marketing: true, + }); + + cleanup(); + }); + + test('UC_UI_INITIALIZED with absent __ucCmp does not throw or publish', async () => { + const mockWindow = createMockWindow(); + // No __ucCmp attached at all. + + const cleanup = await setupV3Adapter({ + window: mockWindow as unknown as Window & typeof globalThis, + elb: mockElb, + settings: buildSettings(), + logger: createMockLogger(), + }); + + dispatchInitialized(mockWindow); + await flushPromises(); + + expect(mockElb).not.toHaveBeenCalled(); + + cleanup(); + }); + }); + describe('explicitOnly filtering via ConsentData.type', () => { test('IMPLICIT consent + explicitOnly=true → no elb call', async () => { const mockWindow = createMockWindow(); @@ -561,9 +729,52 @@ describe('V3 adapter (setupV3Adapter)', () => { cleanup(); dispatchV3Event(mockWindow, 'UC_UI_CMP_EVENT', { - source: 'CMP', + source: 'button', + type: 'ACCEPT_ALL', + }); + await flushPromises(); + + expect(getConsentDetails).not.toHaveBeenCalled(); + expect(mockElb).not.toHaveBeenCalled(); + }); + + test('cleanup removes BOTH the consent event and UC_UI_INITIALIZED listeners', async () => { + const mockWindow = createMockWindow(); + const getConsentDetails = jest.fn().mockResolvedValue({ + consent: buildConsentData({ type: 'EXPLICIT' }), + categories: { + essential: buildCategory('ALL_ACCEPTED', 'Essential'), + }, + } satisfies UsercentricsV3ConsentDetails); + withUcCmp(mockWindow, { + isInitialized: jest.fn().mockResolvedValue(false), + getConsentDetails, + }); + + const cleanup = await setupV3Adapter({ + window: mockWindow as unknown as Window & typeof globalThis, + elb: mockElb, + settings: buildSettings(), + logger: createMockLogger(), + }); + + cleanup(); + + expect(mockWindow.removeEventListener).toHaveBeenCalledWith( + 'UC_UI_CMP_EVENT', + expect.any(Function), + ); + expect(mockWindow.removeEventListener).toHaveBeenCalledWith( + 'UC_UI_INITIALIZED', + expect.any(Function), + ); + + // After cleanup, neither event re-triggers a publish. + dispatchV3Event(mockWindow, 'UC_UI_CMP_EVENT', { + source: 'button', type: 'ACCEPT_ALL', }); + dispatchInitialized(mockWindow); await flushPromises(); expect(getConsentDetails).not.toHaveBeenCalled(); @@ -594,7 +805,7 @@ describe('V3 adapter (setupV3Adapter)', () => { // Default event name should NOT trigger anything. dispatchV3Event(mockWindow, 'UC_UI_CMP_EVENT', { - source: 'CMP', + source: 'button', type: 'ACCEPT_ALL', }); await flushPromises(); @@ -603,7 +814,7 @@ describe('V3 adapter (setupV3Adapter)', () => { // Custom event name SHOULD trigger. dispatchV3Event(mockWindow, 'MY_CUSTOM_UC_EVENT', { - source: 'CMP', + source: 'button', type: 'ACCEPT_ALL', }); await flushPromises(); diff --git a/packages/web/sources/cmps/usercentrics/src/examples/inputs.ts b/packages/web/sources/cmps/usercentrics/src/examples/inputs.ts index 32f2f0771..74b5f5ef3 100644 --- a/packages/web/sources/cmps/usercentrics/src/examples/inputs.ts +++ b/packages/web/sources/cmps/usercentrics/src/examples/inputs.ts @@ -1,112 +1,89 @@ -import type { UsercentricsEventDetail } from '../types'; +import type { UsercentricsV2Service } from '../types'; /** - * Example Usercentrics consent event detail inputs. + * Example Usercentrics V2 service inputs. * - * These represent real event.detail payloads from Usercentrics CMP. + * These mirror the shape returned by `window.UC_UI.getServicesBaseInfo()`: + * an array of services, each carrying a `categorySlug`, optional display + * `name`, and a `consent` block with the current `status` plus a decision + * `history`. The adapter derives the explicit/implicit gate from whether any + * history entry is `explicit`, and aggregates services into group-level + * consent via strict AND per `categorySlug`. */ /** - * Full consent - user accepted all categories (explicit) + * Full consent - every category accepted via an explicit decision + * (user clicked "Accept all"). */ -export const fullConsent: UsercentricsEventDetail = { - event: 'consent_status', - type: 'explicit', - action: 'onAcceptAllServices', - ucCategory: { - essential: true, - functional: true, - marketing: true, +export const servicesFullExplicit: UsercentricsV2Service[] = [ + { + categorySlug: 'essential', + consent: { status: true, history: [{ type: 'explicit', status: true }] }, }, - 'Google Analytics': true, - 'Google Ads Remarketing': true, -}; - -/** - * Partial consent - user accepted only essential and functional (explicit) - */ -export const partialConsent: UsercentricsEventDetail = { - event: 'consent_status', - type: 'explicit', - action: 'onUpdateServices', - ucCategory: { - essential: true, - functional: true, - marketing: false, + { + categorySlug: 'functional', + consent: { status: true, history: [{ type: 'explicit', status: true }] }, }, - 'Google Analytics': true, - 'Google Ads Remarketing': false, -}; - -/** - * Minimal consent - user denied everything except essential (explicit) - */ -export const minimalConsent: UsercentricsEventDetail = { - event: 'consent_status', - type: 'explicit', - action: 'onDenyAllServices', - ucCategory: { - essential: true, - functional: false, - marketing: false, + { + categorySlug: 'marketing', + consent: { status: true, history: [{ type: 'explicit', status: true }] }, }, - 'Google Analytics': false, - 'Google Ads Remarketing': false, -}; +]; /** - * Implicit consent - page load with default consent state - * (not an explicit user choice) + * Minimal consent - only essential accepted, others denied via an explicit + * decision (user clicked "Deny all"). */ -export const implicitConsent: UsercentricsEventDetail = { - event: 'consent_status', - type: 'implicit', - ucCategory: { - essential: true, - functional: false, - marketing: false, +export const servicesMinimalExplicit: UsercentricsV2Service[] = [ + { + categorySlug: 'essential', + consent: { status: true, history: [{ type: 'explicit', status: true }] }, }, - 'Google Analytics': false, - 'Google Ads Remarketing': false, -}; - -/** - * Explicit consent with uppercase type field (Usercentrics docs are - * inconsistent about casing - some show 'EXPLICIT', others 'explicit') - */ -export const fullConsentUpperCase: UsercentricsEventDetail = { - event: 'consent_status', - type: 'EXPLICIT', - action: 'onAcceptAllServices', - ucCategory: { - essential: true, - functional: true, - marketing: true, + { + categorySlug: 'functional', + consent: { status: false, history: [{ type: 'explicit', status: false }] }, + }, + { + categorySlug: 'marketing', + consent: { status: false, history: [{ type: 'explicit', status: false }] }, }, -}; +]; /** - * Service-level consent - ucCategory has mixed types (non-boolean values - * indicate individual service-level choice rather than group-level) + * Partial consent - essential and functional accepted, marketing denied, + * all via an explicit decision (user saved a custom selection). */ -export const serviceLevelConsent: UsercentricsEventDetail = { - event: 'consent_status', - type: 'explicit', - action: 'onUpdateServices', - ucCategory: { - essential: true, - functional: 'partial', // Non-boolean indicates mixed service choices - marketing: 'partial', +export const servicesPartialExplicit: UsercentricsV2Service[] = [ + { + categorySlug: 'essential', + consent: { status: true, history: [{ type: 'explicit', status: true }] }, + }, + { + categorySlug: 'functional', + consent: { status: true, history: [{ type: 'explicit', status: true }] }, + }, + { + categorySlug: 'marketing', + consent: { status: false, history: [{ type: 'explicit', status: false }] }, }, - 'Google Analytics': true, - 'Google Ads Remarketing': false, - Hotjar: true, -}; +]; /** - * Non-consent event (should be ignored) + * First-visit implicit state - the CMP reports page-load defaults with only + * implicit history. The default `explicitOnly` gate suppresses this, so no + * consent command is emitted. */ -export const nonConsentEvent: UsercentricsEventDetail = { - event: 'other_event', - type: 'explicit', -}; +export const servicesFirstVisitImplicit: UsercentricsV2Service[] = [ + { + categorySlug: 'essential', + consent: { status: true, history: [{ type: 'implicit', status: true }] }, + }, + { + categorySlug: 'functional', + consent: { status: false, history: [{ type: 'implicit', status: false }] }, + }, + { + categorySlug: 'marketing', + consent: { status: false, history: [{ type: 'implicit', status: false }] }, + }, +]; diff --git a/packages/web/sources/cmps/usercentrics/src/examples/outputs.ts b/packages/web/sources/cmps/usercentrics/src/examples/outputs.ts index 36a88be58..e2d72eb48 100644 --- a/packages/web/sources/cmps/usercentrics/src/examples/outputs.ts +++ b/packages/web/sources/cmps/usercentrics/src/examples/outputs.ts @@ -3,12 +3,13 @@ import type { WalkerOS } from '@walkeros/core'; /** * Expected walkerOS consent outputs. * - * These represent the consent state after parsing Usercentrics event details - * with no category mapping configured (pass-through). + * These represent the consent state emitted via `elb('walker consent', state)` + * after the V2 adapter aggregates `getServicesBaseInfo()` into group-level + * consent (no category mapping configured: pass-through by `categorySlug`). */ /** - * Full consent - all categories true (group-level) + * Full consent - all categories granted. */ export const fullConsentMapped: WalkerOS.Consent = { essential: true, @@ -17,7 +18,7 @@ export const fullConsentMapped: WalkerOS.Consent = { }; /** - * Partial consent - essential and functional true, marketing false + * Partial consent - essential and functional granted, marketing denied. */ export const partialConsentMapped: WalkerOS.Consent = { essential: true, @@ -26,7 +27,7 @@ export const partialConsentMapped: WalkerOS.Consent = { }; /** - * Minimal consent - only essential true + * Minimal consent - only essential granted. */ export const minimalConsentMapped: WalkerOS.Consent = { essential: true, @@ -35,22 +36,11 @@ export const minimalConsentMapped: WalkerOS.Consent = { }; /** - * Full consent with custom category mapping applied - * (essential->functional, functional->functional, marketing->marketing) + * Full consent with a custom category mapping applied + * (essential->functional, functional->analytics). */ export const fullConsentCustomMapped: WalkerOS.Consent = { functional: true, + analytics: true, marketing: true, }; - -/** - * Service-level consent - individual service booleans + boolean ucCategory entries - * (services normalized: lowercase, spaces to underscores) - * (ucCategory boolean entries mapped through categoryMap) - */ -export const serviceLevelMapped: WalkerOS.Consent = { - essential: true, - google_analytics: true, - google_ads_remarketing: false, - hotjar: true, -}; diff --git a/packages/web/sources/cmps/usercentrics/src/examples/step.ts b/packages/web/sources/cmps/usercentrics/src/examples/step.ts index 48b1a0d70..063956c78 100644 --- a/packages/web/sources/cmps/usercentrics/src/examples/step.ts +++ b/packages/web/sources/cmps/usercentrics/src/examples/step.ts @@ -1,114 +1,93 @@ import type { Flow } from '@walkeros/core'; +import { + servicesFullExplicit, + servicesMinimalExplicit, + servicesPartialExplicit, + servicesFirstVisitImplicit, +} from './inputs'; +import { + fullConsentMapped, + minimalConsentMapped, + fullConsentCustomMapped, +} from './outputs'; + +/** + * Step examples for the Usercentrics V2 path. + * + * `in` is the array returned by `window.UC_UI.getServicesBaseInfo()`. The test + * runner attaches it as the V2 API, then drives the official trigger named in + * `trigger.options.dispatch`: + * - 'init' (default): UC_UI is present when the source runs, so the static + * read at init emits the snapshot (also the `UC_UI_INITIALIZED` path). + * - 'cmp': UC_UI is attached only after init, then a `UC_UI_CMP_EVENT` + * decision (ACCEPT_ALL) re-reads and emits — the consent-change path. + */ export const fullConsent: Flow.StepExample = { title: 'Full consent', description: - 'A Usercentrics onAcceptAllServices event emits a walker consent command with essential, functional, and marketing granted.', - trigger: { type: 'consent' }, - in: { - event: 'consent_status', - type: 'explicit', - action: 'onAcceptAllServices', - ucCategory: { - essential: true, - functional: true, - marketing: true, - }, - }, - out: [ - [ - 'elb', - 'walker consent', - { - essential: true, - functional: true, - marketing: true, - }, - ], - ], + 'Usercentrics reports every category accepted via an explicit decision; the source emits a walker consent command granting essential, functional, and marketing.', + trigger: { type: 'consent', options: { dispatch: 'init' } }, + in: servicesFullExplicit, + out: [['elb', 'walker consent', fullConsentMapped]], }; export const minimalConsent: Flow.StepExample = { title: 'Minimal consent', description: - 'A Usercentrics onDenyAllServices event emits a walker consent command with only essential granted.', - trigger: { type: 'consent' }, - in: { - event: 'consent_status', - type: 'explicit', - action: 'onDenyAllServices', - ucCategory: { - essential: true, - functional: false, - marketing: false, - }, - }, + 'A "Deny all" explicit decision leaves only essential granted; functional and marketing are emitted as false.', + trigger: { type: 'consent', options: { dispatch: 'init' } }, + in: servicesMinimalExplicit, + out: [['elb', 'walker consent', minimalConsentMapped]], +}; + +export const returningVisitor: Flow.StepExample = { + title: 'Returning visitor static read', + description: + 'When the CMP is already initialized with a stored explicit decision, the static read at init re-publishes that choice without any further event.', + trigger: { type: 'consent', options: { dispatch: 'init' } }, + in: servicesPartialExplicit, out: [ [ 'elb', 'walker consent', { essential: true, - functional: false, + functional: true, marketing: false, }, ], ], }; +export const firstVisitImplicit: Flow.StepExample = { + title: 'First visit implicit (suppressed)', + description: + 'A first-visit snapshot carrying only implicit history is suppressed by the default explicitOnly gate, so no consent command is emitted.', + trigger: { type: 'consent', options: { dispatch: 'init' } }, + in: servicesFirstVisitImplicit, + out: [], +}; + +export const consentChange: Flow.StepExample = { + title: 'Consent change via CMP event', + description: + 'An ACCEPT_ALL decision fires UC_UI_CMP_EVENT; the source re-reads the services and emits the updated consent.', + trigger: { type: 'consent', options: { dispatch: 'cmp' } }, + in: servicesFullExplicit, + out: [['elb', 'walker consent', fullConsentMapped]], +}; + export const categoryMapOverride: Flow.StepExample = { title: 'Category map override', description: - 'Custom categoryMap remaps essential to functional and functional to analytics', - trigger: { type: 'consent' }, - in: { - event: 'consent_status', - type: 'explicit', - ucCategory: { - essential: true, - functional: true, - marketing: false, - }, - }, + 'A custom categoryMap remaps essential to functional and functional to analytics before emitting the walker consent command.', + trigger: { type: 'consent', options: { dispatch: 'init' } }, + in: servicesFullExplicit, mapping: { - categoryMap: { essential: 'functional', functional: 'analytics' }, - }, - out: [ - [ - 'elb', - 'walker consent', - { - functional: true, - analytics: true, - marketing: false, - }, - ], - ], -}; - -export const customEventName: Flow.StepExample = { - title: 'Custom event name', - description: 'Using UC_SDK_EVENT instead of ucEvent for Usercentrics SDK v2', - trigger: { type: 'consent', options: { eventName: 'UC_SDK_EVENT' } }, - in: { - event: 'consent_status', - type: 'explicit', - ucCategory: { - essential: true, - functional: true, - marketing: true, + settings: { + categoryMap: { essential: 'functional', functional: 'analytics' }, }, }, - mapping: { eventName: 'UC_SDK_EVENT' }, - out: [ - [ - 'elb', - 'walker consent', - { - essential: true, - functional: true, - marketing: true, - }, - ], - ], + out: [['elb', 'walker consent', fullConsentCustomMapped]], }; diff --git a/packages/web/sources/cmps/usercentrics/src/index.ts b/packages/web/sources/cmps/usercentrics/src/index.ts index d143417c8..14b9ad5fc 100644 --- a/packages/web/sources/cmps/usercentrics/src/index.ts +++ b/packages/web/sources/cmps/usercentrics/src/index.ts @@ -24,7 +24,6 @@ export * as SourceUsercentrics from './types'; * code: sourceUsercentrics, * config: { * settings: { - * eventName: 'ucEvent', * categoryMap: { * essential: 'functional', * functional: 'functional', @@ -48,7 +47,6 @@ export const sourceUsercentrics: Source.Init = async (context) => { // Merge user settings with defaults so adapters always see a fully-resolved Settings const settings: Settings = { - eventName: config?.settings?.eventName ?? 'ucEvent', categoryMap: config?.settings?.categoryMap ?? {}, explicitOnly: config?.settings?.explicitOnly ?? true, apiVersion: config?.settings?.apiVersion ?? 'auto', diff --git a/packages/web/sources/cmps/usercentrics/src/lib/v2.ts b/packages/web/sources/cmps/usercentrics/src/lib/v2.ts index c8459b91e..98414d791 100644 --- a/packages/web/sources/cmps/usercentrics/src/lib/v2.ts +++ b/packages/web/sources/cmps/usercentrics/src/lib/v2.ts @@ -3,6 +3,7 @@ import type { Settings, UsercentricsEventDetail, UsercentricsV2Service, + UsercentricsV3CmpEventDetail, } from '../types'; import { parseConsent } from './parseConsent'; @@ -13,6 +14,17 @@ export interface V2AdapterContext { logger: Logger.Instance; } +/** + * V3 CMP event `type` values that signal a real user decision. Other event + * types (e.g. CMP_SHOWN, VIEW_CHANGED) carry no consent decision and are + * ignored so the adapter only re-reads after an actual choice. + */ +const V2_DECISION_TYPES: ReadonlySet = new Set([ + 'ACCEPT_ALL', + 'DENY_ALL', + 'SAVE', +]); + /** * Aggregate V2 service array into a group-level ucCategory object using strict * AND logic. A category is `true` only if EVERY service in that category is @@ -35,61 +47,82 @@ function aggregateByCategory( } /** - * Build a synthetic UsercentricsEventDetail from the V2 static API. - * Marked as 'implicit' because UC_UI.isInitialized() === true does NOT prove - * the read was user-initiated — it only proves the SDK has finished loading. - * With settings.explicitOnly = true (the default), consumers correctly drop - * this detail. Set explicitOnly = false to surface the static snapshot. + * Whether any service carries an `explicit` entry in its consent history. An + * explicit entry is Usercentrics' own proof that the user actively decided + * (vs an implicit page-load default), so it correctly surfaces returning + * visitors whose stored decision was explicit. + */ +export function hasExplicitDecision( + services: UsercentricsV2Service[], +): boolean { + return services.some((s) => + (s.consent.history ?? []).some((h) => h.type?.toLowerCase() === 'explicit'), + ); +} + +/** + * Build a synthetic UsercentricsEventDetail from the V2 services array. The + * `type` is derived from Usercentrics' own consent history: `explicit` when any + * service records an explicit decision, `implicit` otherwise. Per-service name + * keys are surfaced so parseConsent can map service-level consent. */ -function buildDetailFromStatic( +export function buildDetailFromServices( services: UsercentricsV2Service[], ): UsercentricsEventDetail { - return { + const detail: UsercentricsEventDetail = { event: 'consent_status', - type: 'implicit', + type: hasExplicitDecision(services) ? 'explicit' : 'implicit', ucCategory: aggregateByCategory(services), }; + services.forEach((s) => { + if (s.name) detail[s.name] = s.consent.status; + }); + return detail; } /** - * Set up the V2 adapter: listens on the configured event AND performs a - * static read if UC_UI is already initialized. + * Set up the V2 adapter using Usercentrics' official events and getter. + * + * A single gated `read()` is reused by all triggers: the official + * `UC_UI_INITIALIZED` lifecycle event, the official `UC_UI_CMP_EVENT` decision + * events (filtered to real decisions), and a static read at setup time for the + * common case where the CMP is already initialized when the source runs. * - * Returns a cleanup function that removes the event listener. + * Returns a cleanup function that removes both event listeners. */ export function setupV2Adapter(ctx: V2AdapterContext): () => void { const { window: win, elb, settings, logger } = ctx; - const eventName = settings.eventName ?? 'ucEvent'; - - const handleDetail = (detail: UsercentricsEventDetail) => { - logger.debug('event received', detail); - - if (detail.event !== 'consent_status') return; - if (settings.explicitOnly && detail.type?.toLowerCase() !== 'explicit') - return; + const read = (): void => { + const api = win.UC_UI; + if (!api?.getServicesBaseInfo) return; + // isInitialized is optional on the V2 API; only honor it when present. + // Hard-gating on it would suppress all consent on deployments that expose + // getServicesBaseInfo without isInitialized. + if (api.isInitialized && !api.isInitialized()) return; + const services = api.getServicesBaseInfo(); + if (!services.length) return; + const detail = buildDetailFromServices(services); + logger.debug('consent read', detail); + if (settings.explicitOnly && detail.type !== 'explicit') return; const state = parseConsent(detail, settings); - if (Object.keys(state).length > 0) { - elb('walker consent', state); - } + if (Object.keys(state).length > 0) elb('walker consent', state); }; - const listener = (e: Event) => { - const custom = e as CustomEvent; - if (custom.detail) handleDetail(custom.detail); + const onInitialized = (): void => read(); + const onCmpEvent = (e: Event): void => { + const detail = (e as CustomEvent).detail; + if (!detail?.type || !V2_DECISION_TYPES.has(detail.type)) return; + read(); }; - win.addEventListener(eventName, listener); - // Static check: if UC_UI is already initialized, read current consent now. - const api = win.UC_UI; - if (api?.isInitialized?.() && api.getServicesBaseInfo) { - const services = api.getServicesBaseInfo(); - if (services.length > 0) { - handleDetail(buildDetailFromStatic(services)); - } - } + win.addEventListener('UC_UI_INITIALIZED', onInitialized); + win.addEventListener('UC_UI_CMP_EVENT', onCmpEvent); + + read(); // static read: CMP already initialized when source runs return () => { - win.removeEventListener(eventName, listener); + win.removeEventListener('UC_UI_INITIALIZED', onInitialized); + win.removeEventListener('UC_UI_CMP_EVENT', onCmpEvent); }; } diff --git a/packages/web/sources/cmps/usercentrics/src/lib/v3.ts b/packages/web/sources/cmps/usercentrics/src/lib/v3.ts index 6632c43d6..8580f3d65 100644 --- a/packages/web/sources/cmps/usercentrics/src/lib/v3.ts +++ b/packages/web/sources/cmps/usercentrics/src/lib/v3.ts @@ -90,19 +90,28 @@ export async function setupV3Adapter( const listener = (event: Event): void => { const cmp = win.__ucCmp; if (!cmp) return; - const custom = event as CustomEvent; - const detail = custom.detail; + const detail = (event as CustomEvent).detail; // Filter to decision events only: the CMP emits a broad set of events // (CMP_SHOWN, UI_INITIALIZED, VIEW_CHANGED, …) on this bus. Without - // filtering, every view toggle would re-publish consent. - if (!detail || detail.source !== 'CMP') return; - if (!detail.type || !V3_DECISION_TYPES.has(detail.type)) return; + // filtering, every view toggle would re-publish consent. The decision + // `type` is the only reliable signal; `detail.source` is one of + // none|button|first|second|embeddings|__ucCmp and must NOT be gated on. + if (!detail?.type || !V3_DECISION_TYPES.has(detail.type)) return; publishConsent(cmp).catch(() => { // Swallow — a failed V3 fetch should not break the page. }); }; win.addEventListener(eventName, listener); + // When the source loads BEFORE __ucCmp, no UC_UI_CMP_EVENT fires for a + // choice made in a prior session. UC_UI_INITIALIZED signals the CMP is ready, + // so perform the static read then to publish any already-made decision. + const onInitialized = (): void => { + const cmp = win.__ucCmp; + if (cmp) publishConsent(cmp).catch(() => {}); + }; + win.addEventListener('UC_UI_INITIALIZED', onInitialized); + // Static check: if __ucCmp is already initialized, fetch now. // Wrapped in try/catch so transient API failures during setup don't // reject setupV3Adapter() and tear down the whole flow — the listener @@ -121,5 +130,6 @@ export async function setupV3Adapter( return () => { win.removeEventListener(eventName, listener); + win.removeEventListener('UC_UI_INITIALIZED', onInitialized); }; } diff --git a/packages/web/sources/cmps/usercentrics/src/schemas/settings.ts b/packages/web/sources/cmps/usercentrics/src/schemas/settings.ts index db470d5bb..60f28b0af 100644 --- a/packages/web/sources/cmps/usercentrics/src/schemas/settings.ts +++ b/packages/web/sources/cmps/usercentrics/src/schemas/settings.ts @@ -5,13 +5,6 @@ import { z } from '@walkeros/core/dev'; */ export const SettingsSchema = z .object({ - eventName: z - .string() - .describe( - "Window event name to listen for, configured in the Usercentrics admin (Implementation > Data Layer & Events). Use 'UC_SDK_EVENT' for the built-in Browser SDK event. Default: 'ucEvent'.", - ) - .optional(), - categoryMap: z .record(z.string(), z.string()) .describe( @@ -22,7 +15,7 @@ export const SettingsSchema = z explicitOnly: z .boolean() .describe( - "Only process consent_status events where type is 'explicit'. Ignores implicit/default page-load events. Default: true.", + 'Only publish when the user has actively decided (V3: consent.type EXPLICIT; V2: an EXPLICIT entry in service consent history). Implicit/default page-load states are suppressed. Set false to publish any snapshot including implicit. Default: true.', ) .optional(), }) diff --git a/packages/web/sources/cmps/usercentrics/src/types/index.ts b/packages/web/sources/cmps/usercentrics/src/types/index.ts index adcbce23b..105591b0b 100644 --- a/packages/web/sources/cmps/usercentrics/src/types/index.ts +++ b/packages/web/sources/cmps/usercentrics/src/types/index.ts @@ -28,10 +28,23 @@ export interface UsercentricsEventDetail { declare global { interface WindowEventMap { ucEvent: CustomEvent; + UC_UI_INITIALIZED: Event; UC_UI_CMP_EVENT: CustomEvent; } } +/** Usercentrics CONSENT_TYPE values (history entry type / V3 consent type). */ +export type UsercentricsConsentType = 'explicit' | 'implicit'; + +/** A single entry in a V2 service's consent history (getServicesBaseInfo). */ +export interface UsercentricsV2ConsentHistoryEntry { + type: UsercentricsConsentType; + status: boolean; + /** CONSENT_ACTION, e.g. 'onAcceptAllServices' | 'onInitialPageLoad'. Not read for the gate. */ + action?: string; + timestamp?: number; +} + /** * Usercentrics V2 service info shape returned by `UC_UI.getServicesBaseInfo()`. * Only the fields we use are typed - minimal surface, not a full V2 API mirror. @@ -39,9 +52,13 @@ declare global { export interface UsercentricsV2Service { /** Category slug: 'essential' | 'functional' | 'marketing' | custom */ categorySlug: string; + /** Service display name (used for service-level consent keys) */ + name?: string; /** Consent state for this service */ consent: { status: boolean; + /** Consent decision history; an 'explicit' entry proves a real user decision. */ + history?: UsercentricsV2ConsentHistoryEntry[]; }; } @@ -54,6 +71,7 @@ export interface UsercentricsV2Service { export interface UsercentricsV2Api { isInitialized?: () => boolean; getServicesBaseInfo?: () => UsercentricsV2Service[]; + getServicesFullInfo?: () => Promise; areAllConsentsAccepted?: () => boolean; } @@ -111,8 +129,16 @@ export interface UsercentricsV3Api { * `source: 'CMP'` + a decision `type` tells us a user action has been taken. */ export interface UsercentricsV3CmpEventDetail { - source?: string; + source?: + | 'none' + | 'button' + | 'first' + | 'second' + | 'embeddings' + | '__ucCmp' + | string; type?: string; + abTestVariant?: string; } declare global { @@ -136,15 +162,6 @@ declare global { * Settings for Usercentrics source */ export interface Settings { - /** - * Window event name to listen for. - * Configured in Usercentrics admin under Implementation > Data Layer & Events. - * Can also be set to 'UC_SDK_EVENT' for the built-in Browser SDK event. - * - * Default: 'ucEvent' - */ - eventName?: string; - /** * Map Usercentrics categories to walkerOS consent groups. * Keys: Usercentrics category names (from ucCategory) @@ -160,11 +177,10 @@ export interface Settings { categoryMap?: Record; /** - * Only process explicit consent (user made a choice). - * When true: Ignores events where type !== 'explicit' - * When false: Processes any consent_status event including implicit/defaults - * - * Default: true + * Only publish when the user has actively decided. V3: consent.type === + * EXPLICIT. V2: an EXPLICIT entry exists in service consent history. + * Implicit/default page-load states are suppressed. Set false to publish any + * consent snapshot including implicit. Default: true. */ explicitOnly?: boolean; diff --git a/packages/web/sources/session/src/__tests__/index.integration.test.ts b/packages/web/sources/session/src/__tests__/index.integration.test.ts index 983d1df1b..4e5fcac83 100644 --- a/packages/web/sources/session/src/__tests__/index.integration.test.ts +++ b/packages/web/sources/session/src/__tests__/index.integration.test.ts @@ -136,3 +136,70 @@ describe('Session Source: collector-enforced exactly-once (I12)', () => { expect(sessionStartCount(captured)).toBe(1); }); }); + +describe('Session Source: ungated path respects run', () => { + beforeEach(() => { + webCore.__store.clear(); + + // jsdom: a fresh navigate entry so the storage path treats this as a visit. + Object.defineProperty(window, 'performance', { + value: { + getEntriesByType: jest.fn().mockReturnValue([{ type: 'navigate' }]), + }, + writable: true, + configurable: true, + }); + }); + + // No `consent` setting: the source has no consent rule to replay at the run + // barrier. The emit must instead wait for the run lifecycle, otherwise the + // `session start` pushed during init() (while !allowed) is dropped at the + // dormant destination gate and never delivered. + async function startUngatedFlow(options: { + run: boolean; + captured: WalkerOS.Event[]; + }): Promise { + const { collector } = await startFlow({ + run: options.run, + sources: { + session: { + code: sourceSession, + config: { settings: { storage: true } }, + }, + }, + destinations: { + capture: { + code: { + type: 'capture', + config: {}, + push: (event: WalkerOS.Event): void => { + options.captured.push(event); + }, + }, + }, + }, + }); + + return collector; + } + + test('run:false — session start is delivered at run, not dropped pre-run', async () => { + const captured: WalkerOS.Event[] = []; + const collector = await startUngatedFlow({ run: false, captured }); + + // Pre-run: collector is not allowed yet, so nothing is delivered. + expect(sessionStartCount(captured)).toBe(0); + + await collector.command('run'); + + // The ungated emit lands in the now-allowed pipeline exactly once. + expect(sessionStartCount(captured)).toBe(1); + }); + + test('run:true — session start is delivered exactly once at startup run', async () => { + const captured: WalkerOS.Event[] = []; + await startUngatedFlow({ run: true, captured }); + + expect(sessionStartCount(captured)).toBe(1); + }); +}); diff --git a/packages/web/sources/session/src/__tests__/index.test.ts b/packages/web/sources/session/src/__tests__/index.test.ts index a4e97278f..e94e88196 100644 --- a/packages/web/sources/session/src/__tests__/index.test.ts +++ b/packages/web/sources/session/src/__tests__/index.test.ts @@ -55,11 +55,18 @@ describe('Session Source', () => { }); describe('Session Start', () => { - test('calls sessionStart on initialization', async () => { + test('ungated path registers a run subscription at init (emit deferred to run)', async () => { await createSessionSource(collector); - // Session start should have been called, which calls command('user', ...) - expect(mockCommand).toHaveBeenCalled(); + // No consent configured: the emit is deferred to the run lifecycle, so + // init registers a single on('run') rule rather than emitting at init + // (which would be dropped at the collector's dormant gate). + const runRegistrations = mockCommand.mock.calls.filter( + ([cmd, data]) => + cmd === 'on' && + (data as { type?: string } | undefined)?.type === 'run', + ); + expect(runRegistrations).toHaveLength(1); }); }); diff --git a/packages/web/sources/session/src/__tests__/stepExamples.test.ts b/packages/web/sources/session/src/__tests__/stepExamples.test.ts index 4ec745bdd..4fb4def9a 100644 --- a/packages/web/sources/session/src/__tests__/stepExamples.test.ts +++ b/packages/web/sources/session/src/__tests__/stepExamples.test.ts @@ -104,12 +104,25 @@ describe('Step Examples', () => { ); } - const mockElb = jest.fn(async () => ({ - ok: true, - successful: [], - failed: [], - queued: [], - })) as unknown as jest.MockedFunction; + // The ungated path defers its emit to an on('run') subscription. Capture + // those rules here and fire them below to simulate the collector's run + // lifecycle, so the example output reflects the emitted user/session/ + // session-start calls rather than the internal registration. + const runRules: Array<() => unknown> = []; + const captured: unknown[][] = []; + + const mockElb = jest.fn(async (...args: unknown[]) => { + const [action, data] = args as [ + unknown, + { type?: string; rules?: Array<() => unknown> } | undefined, + ]; + if (action === 'on' && data?.type === 'run') { + runRules.push(...(data.rules || [])); + } else { + captured.push(['elb', ...args]); + } + return { ok: true, successful: [], failed: [], queued: [] }; + }) as unknown as jest.MockedFunction; const collectorStub: Collector.Instance = { allowed: true, @@ -134,14 +147,14 @@ describe('Step Examples', () => { // Session detection runs in init(), not the factory. await source.init?.(); + // Fire the run lifecycle: the ungated path emits here, not at init(). + for (const rule of runRules) await rule(); + // Yield to pick up any deferred pushes - for (let i = 0; i < 10 && mockElb.mock.calls.length === 0; i++) { + for (let i = 0; i < 10 && captured.length === 0; i++) { await Promise.resolve(); } - const captured = mockElb.mock.calls.map( - (args) => ['elb', ...args] as unknown[], - ); expect(captured).toEqual(example.out); }); }); diff --git a/packages/web/sources/session/src/index.ts b/packages/web/sources/session/src/index.ts index 6bac7de56..ed41c7c6b 100644 --- a/packages/web/sources/session/src/index.ts +++ b/packages/web/sources/session/src/index.ts @@ -38,14 +38,7 @@ export const sourceSession: Source.Init = async (context) => { command, }; - // Run session detection in init() (Pass 2 of initSources), not the factory - // (Pass 1), so construction stays side-effect free. When `settings.consent` - // is set this registers a single consent rule with the collector; the - // collector then guarantees exactly-once delivery per state change, so the - // source does not need to react to consent events itself. Deferring to init() - // keeps that single registration but moves the emit out of construction, - // where it would race source merge order. - const init = async (): Promise => { + const runSessionStart = (): void => { sessionStart({ ...settings, window: env.window, @@ -54,6 +47,25 @@ export const sourceSession: Source.Init = async (context) => { }); }; + // Session detection runs in init() (Pass 2 of initSources), not the factory + // (Pass 1), so construction stays side-effect free. + // + // Consent-gated (settings.consent): sessionStart registers a single consent + // rule with the collector, which replays it at the run barrier and guarantees + // exactly-once delivery, so the source does not react to consent itself. + // + // Ungated: the emit must wait for the run lifecycle. Calling sessionStart in + // init() would push `session start` while the collector is not yet `allowed`, + // dropping it at the dormant destination gate. Registering an on('run') rule + // defers the emit into the now-allowed pipeline. + const init = async (): Promise => { + if (settings.consent) { + runSessionStart(); + } else { + await command('on', { type: 'run', rules: [() => runSessionStart()] }); + } + }; + return { type: 'session', config: fullConfig, diff --git a/skills/walkeros-create-cmp-source/SKILL.md b/skills/walkeros-create-cmp-source/SKILL.md index 60d13b642..ba9c84904 100644 --- a/skills/walkeros-create-cmp-source/SKILL.md +++ b/skills/walkeros-create-cmp-source/SKILL.md @@ -137,19 +137,19 @@ destroy implementation. Fill in this matrix for your CMP. Reference implementations for comparison: -| Decision | CookieFirst | Usercentrics | CookiePro | -| ----------------------- | --------------------------------------- | -------------------------------- | ------------------------------------------------ | -| Already loaded? | `window.CookieFirst.consent` | None (events only) | `window.OneTrust` + `window.OptanonActiveGroups` | -| Init listener | `cf_init` event | Same as change event (`ucEvent`) | `OptanonWrapper` callback (self-unwrap) | -| Change listener | `cf_consent` event | `ucEvent` / `UC_SDK_EVENT` | `OneTrustGroupsUpdated` event | -| Consent shape | Boolean map `{ category: bool }` | Mixed object (groups + services) | Comma-separated IDs `,C0001,C0003,` | -| Category naming | Human-readable | Admin-configured | Opaque IDs (C0001-C0005) | -| Explicit check | `consent === null` | `detail.type` (case-insensitive) | `IsAlertBoxClosed()` | -| Default categoryMap | Populated (human names to walkerOS) | Empty (pass-through) | Populated (opaque IDs need mapping) | -| Number of change events | 1 (`cf_consent`) | 1 (`ucEvent`) | 2 (`OptanonWrapper` + `OneTrustGroupsUpdated`) | -| Consent layers | Single (categories only) | Dual (groups + services) | Single (categories only) | -| Consent access | Property (`window.CookieFirst.consent`) | Event detail (`event.detail`) | Property (`window.OptanonActiveGroups`) | -| Event registration | `addEventListener` | `addEventListener` | Callback assignment + `addEventListener` | +| Decision | CookieFirst | Usercentrics | CookiePro | +| ----------------------- | --------------------------------------- | ------------------------------------------------------------------------------------------------ | ------------------------------------------------ | +| Already loaded? | `window.CookieFirst.consent` | None (events only) | `window.OneTrust` + `window.OptanonActiveGroups` | +| Init listener | `cf_init` event | Same as change event (`ucEvent`) | `OptanonWrapper` callback (self-unwrap) | +| Change listener | `cf_consent` event | `ucEvent` / `UC_SDK_EVENT` | `OneTrustGroupsUpdated` event | +| Consent shape | Boolean map `{ category: bool }` | Mixed object (groups + services) | Comma-separated IDs `,C0001,C0003,` | +| Category naming | Human-readable | Admin-configured | Opaque IDs (C0001-C0005) | +| Explicit check | `consent === null` | Official consent metadata (V3 `consent.type`; V2 an `EXPLICIT` entry in service consent history) | `IsAlertBoxClosed()` | +| Default categoryMap | Populated (human names to walkerOS) | Empty (pass-through) | Populated (opaque IDs need mapping) | +| Number of change events | 1 (`cf_consent`) | 1 (`ucEvent`) | 2 (`OptanonWrapper` + `OneTrustGroupsUpdated`) | +| Consent layers | Single (categories only) | Dual (groups + services) | Single (categories only) | +| Consent access | Property (`window.CookieFirst.consent`) | Event detail (`event.detail`) | Property (`window.OptanonActiveGroups`) | +| Event registration | `addEventListener` | `addEventListener` | Callback assignment + `addEventListener` | ### Gate: Research complete @@ -700,6 +700,14 @@ What if the CMP loads before the source? What about `explicitOnly: false`? Each CMP has different timing behavior -- document it explicitly under "Timing considerations." +For "previous-choice" detection (was this an active user decision or a +first-visit default?), read the CMP's official consent metadata, the consent +type or decision history (e.g. Usercentrics V3 `consent.type`, V2 the service +consent history), not a per-pageload event `type` field. A per-pageload event +`type` conflates first-visit defaults with a returning visitor's restored +choice, so a returning visitor's prior consent would be dropped under +`explicitOnly`. + ### 6. Test consent revocation end-to-end Full grant -> revoke -> verify denied (explicit `false` values). This is the diff --git a/turbo.json b/turbo.json index ae9d5f616..fc3bda56e 100644 --- a/turbo.json +++ b/turbo.json @@ -52,6 +52,7 @@ "outputLogs": "errors-only" }, "test": { + "dependsOn": ["^build"], "inputs": [ "src/**/*.ts", "src/**/*.tsx", diff --git a/website/docs/apps/cli.mdx b/website/docs/apps/cli.mdx index 0706308d7..7fbd5344b 100644 --- a/website/docs/apps/cli.mdx +++ b/website/docs/apps/cli.mdx @@ -1994,6 +1994,8 @@ These options work with all commands: | `WALKEROS_APP_URL` | Base URL override (default: `https://app.walkeros.io`) | | `WALKEROS_DEPLOY_TOKEN` | Deploy token for container heartbeat authentication | | `WALKEROS_CLIENT_TYPE` | Override client identity sent to the app. Defaults to `cli`; set to `runner` for long-lived flow runners. Used by the runtime image. | +| `WALKEROS_OBSERVE_LEVEL` | Baseline telemetry level for `walkeros run` (`off`, `standard`, or `trace`). Invalid values are warned about and ignored. | +| `WALKEROS_CONFIG_FROZEN` | Set to `1` or `true` to make `walkeros run` serve the bundle as an immutable snapshot: secrets are still injected at boot, but config hot-swap and heartbeat are disabled. | ## API version compatibility diff --git a/website/docs/apps/runner.mdx b/website/docs/apps/runner.mdx index a4e31ca84..0946d0849 100644 --- a/website/docs/apps/runner.mdx +++ b/website/docs/apps/runner.mdx @@ -146,6 +146,8 @@ The runner treats secret fetch failures differently depending on the cause: | **500 (server error)** | Warning: the runner continues without secrets. | | **Failure during hot-swap** | The swap is skipped entirely. The runner keeps running the current flow with existing secrets and retries on the next poll. | +Transient fetch failures (request timeouts, network errors, and 5xx responses) are retried a few times with bounded, jittered backoff before the behavior above applies. This covers the secret, config, and bundle fetches at startup, so a brief blip while the container starts does not fail the run. The secret and bundle fetches are also each bounded by a timeout, so an unresponsive API cannot stall startup indefinitely. + :::note Secrets only apply in mode C/D (remote config). In local modes, set environment variables directly on the container with `-e` or in your Docker Compose file. ::: @@ -280,9 +282,11 @@ When `WALKEROS_TOKEN` and `PROJECT_ID` are set, the runner sends periodic heartb - **Endpoint**: `POST /api/projects/:projectId/runners/heartbeat` - **Interval**: Every `HEARTBEAT_INTERVAL` seconds (default 60) -- **Payload**: instance ID, flow ID, config version, CLI version, uptime, mode, event counters (delta since last successful heartbeat) +- **Payload**: instance ID, flow ID, config version, CLI version, uptime, mode, event counters (delta since last successful heartbeat), plus recent runtime errors and recent log output (redacted at the runner before sending) - **Fire-and-forget**: Failures are logged, never crash the runner. Counter deltas accumulate on failure and are included in the next successful heartbeat. +The recent errors and logs let the dashboard surface a deployed flow's runtime errors and log output without any external log tooling. They are captured in bounded in-memory buffers and secrets are redacted before they leave the runner. + Runners appear in the project dashboard after their first heartbeat. diff --git a/website/docs/sources/web/cmps/usercentrics/index.mdx b/website/docs/sources/web/cmps/usercentrics/index.mdx index b92d2f2ea..60c7bb7ef 100644 --- a/website/docs/sources/web/cmps/usercentrics/index.mdx +++ b/website/docs/sources/web/cmps/usercentrics/index.mdx @@ -8,7 +8,9 @@ import Configuration from '@site/src/components/snippets/_configuration.mdx'; # Usercentrics -Integrates [Usercentrics](https://usercentrics.com/) consent management with walkerOS by listening for a configured window event and mapping category or service consent state to walkerOS consent groups. +Integrates [Usercentrics](https://usercentrics.com/) consent management with walkerOS using the official Usercentrics events and consent getters, mapping category or service consent state to walkerOS consent groups. + +We recommend the source package below over a hand-written event listener: it handles version detection, returning-visitor restore, and explicit-consent gating for you. ## Installation @@ -26,7 +28,6 @@ await startFlow({ code: sourceUsercentrics, config: { settings: { - eventName: 'ucEvent', // Must match your Usercentrics admin config categoryMap: { essential: 'functional', functional: 'functional', @@ -41,13 +42,16 @@ await startFlow({ -| Setting | Type | Default | Description | -| -------------- | ------------------------ | ------------------- | -------------------------------------------------------- | -| `apiVersion` | `'auto' \| 'v2' \| 'v3'` | `'auto'` | Which Usercentrics API to target (auto-detects by default)| -| `eventName` | `string` | `'ucEvent'` | V2 window event name configured in Usercentrics admin | -| `v3EventName` | `string` | `'UC_UI_CMP_EVENT'` | V3 window event name (overridable for custom admin configs)| -| `categoryMap` | `Record` | `{}` | Maps Usercentrics categories to walkerOS consent groups | -| `explicitOnly` | `boolean` | `true` | Only process explicit consent (user made a choice) | +| Setting | Type | Default | Description | +| -------------- | ------------------------ | ------------------- | ----------------------------------------------------------- | +| `apiVersion` | `'auto' \| 'v2' \| 'v3'` | `'auto'` | Which Usercentrics API to target (auto-detects by default) | +| `categoryMap` | `Record` | `{}` | Maps Usercentrics categories to walkerOS consent groups | +| `explicitOnly` | `boolean` | `true` | Only publish when the user has actively decided | +| `v3EventName` | `string` | `'UC_UI_CMP_EVENT'` | V3 event name, override only for a custom admin event | + +There is no configurable data-layer event setting: the source listens to the +always-emitted official Usercentrics events, so no `eventName` configuration or +Usercentrics admin window-event setup is required. ### V2 vs V3 support @@ -55,25 +59,16 @@ The source supports both Usercentrics V2 (`window.UC_UI`) and V3 (`window.__ucCmp`) APIs. With the default `apiVersion: 'auto'`, detection runs at init: -- If the CMP is already initialized, consent is read **statically** from the - available API — this fixes the race where V3's `UC_UI_CMP_EVENT` fires before - walkerOS loads. -- If no CMP is present yet, listeners for **both** V2 and V3 events are - registered so late-loading CMPs are still caught. +- If the CMP is already initialized, consent is read **statically** through the + official getters (V2 `UC_UI.getServicesBaseInfo()`, V3 + `__ucCmp.getConsentDetails()`). +- If no CMP is present yet, the source listens for `UC_UI_INITIALIZED` and reads + the current state once the CMP signals it is ready, so late-loading CMPs are + still caught. - When both APIs are available, V3 is preferred. Set `apiVersion: 'v2'` or `'v3'` to force a specific integration. -### Usercentrics setup - -Configure a **Window Event** in your Usercentrics admin: -Implementation > Data Layer & Events > Window Event Name (e.g., `ucEvent`). - -![Usercentrics window event implementation](./usercentrics-implementation-window_event.png) - -Alternatively, set `eventName: 'UC_SDK_EVENT'` to use the built-in Browser SDK -event (no admin configuration required). - ### Custom mapping example ```typescript @@ -83,7 +78,6 @@ await startFlow({ code: sourceUsercentrics, config: { settings: { - eventName: 'ucEvent', categoryMap: { essential: 'functional', functional: 'functional', @@ -99,35 +93,40 @@ await startFlow({ ## How it works -1. **API detection**: Reads consent statically from `window.UC_UI` (V2) or - `window.__ucCmp` (V3) if already initialized, otherwise registers event - listeners for both APIs so post-init events are still captured. +The source uses Usercentrics' official integration surface across both API +versions: + +1. **Already initialized**: if the CMP loaded before the source, consent is read + statically through the official getters (V2 `UC_UI.getServicesBaseInfo()`, V3 + `__ucCmp.getConsentDetails()`). + +2. **CMP loads after the source**: the source listens for `UC_UI_INITIALIZED` + and reads the current consent state once the CMP is ready. -2. **Group vs. service detection**: Checks if `ucCategory` values are all - booleans: - - **Group-level**: Uses `ucCategory` as consent state (maps categories via - `categoryMap`) - - **Service-level**: Extracts individual service booleans from `event.detail` - (normalized to `lowercase_underscores`) and applies `categoryMap` to - boolean `ucCategory` entries +3. **User decisions**: the source listens for `UC_UI_CMP_EVENT` (consent actions + `ACCEPT_ALL`, `DENY_ALL`, and `SAVE`) and republishes the updated state. -3. **Explicit filtering**: By default, only processes events where - `type === 'explicit'` (user actively made a choice). Set - `explicitOnly: false` to also process implicit/default consent. +4. **Category mapping**: maps categories via `categoryMap` and calls + `elb('walker consent', state)` with the mapped consent state. -4. **Consent command**: Calls `elb('walker consent', state)` with the mapped - consent state. +### Explicit consent +By default (`explicitOnly: true`), the source publishes only states the user has +actively decided. It reads this from the official consent metadata: V3 +`consent.type === EXPLICIT`, and V2 an `EXPLICIT` entry in the service consent +history. First-visit defaults stay suppressed. Set `explicitOnly: false` to also +publish implicit/default consent. ### Timing considerations -The source should be initialized before the Usercentrics script loads to avoid -missing the initial consent event. When using `explicitOnly: true` (default), -this is not a concern since the implicit init event is filtered anyway. For -`explicitOnly: false`, ensure the consent source has no `require` constraints -so it initializes immediately. +A returning visitor's prior choice is applied on page load, either from the +static getter read (CMP already initialized) or on `UC_UI_INITIALIZED` (CMP loads +later). First-visit defaults stay suppressed under the default +`explicitOnly: true`. Set `explicitOnly: false` to publish any snapshot, +including implicit defaults. Ensure the consent source has no `require` +constraints so it initializes immediately. ## Reference -- [Usercentrics custom events documentation](https://support.usercentrics.com/hc/en-us/articles/17104002464668-How-can-I-create-a-custom-event) +- [Usercentrics developer documentation](https://usercentrics.com/docs/) - [Source code](https://github.com/elbwalker/walkerOS/tree/main/packages/web/sources/cmps/usercentrics)