From 74cc585a0c5d80ffda3f1310fe2f17b9befff277 Mon Sep 17 00:00:00 2001 From: Daniel G Wilson Date: Sun, 2 Aug 2026 10:05:05 +0000 Subject: [PATCH 1/7] feat(pricing): dated operator-editable rate table + pure cost estimators Add src/pricing.ts: a single, dated, operator-editable per-model + E2B desktop rate table plus two pure, deterministic estimators (estimateActorCost / estimateDesktopCost) with injectable rates for tests. Unknown model/no-usage/no-duration => DECLARED ABSENT (estimatedCostUsd: null + a reason), never a guessed or silent-zero cost. Add the additive, optional ActorTrace.estimatedCost field (humanish.actor-estimated-cost.v1), kept distinct from the reserved provider-returned tokenUsage.costUsd so an estimate is never confused with an authoritative charge. Export the pricing surface from src/index.ts. Co-Authored-By: Claude Opus 4.8 --- src/actor-contract.ts | 10 +++ src/index.ts | 13 ++++ src/pricing.ts | 172 ++++++++++++++++++++++++++++++++++++++++++ tests/pricing.test.ts | 135 +++++++++++++++++++++++++++++++++ 4 files changed, 330 insertions(+) create mode 100644 src/pricing.ts create mode 100644 tests/pricing.test.ts diff --git a/src/actor-contract.ts b/src/actor-contract.ts index 93af3bb..f5f4ca5 100644 --- a/src/actor-contract.ts +++ b/src/actor-contract.ts @@ -1,4 +1,5 @@ import type { CodexAppServerRunResult, CodexAppServerStatus, CodexAppServerTrace } from "./codex-app-server.js"; +import type { ActorEstimatedCost } from "./pricing.js"; import { redactText } from "./redaction.js"; // The provider-neutral evidence schema. Codex item/* events, Claude @@ -129,6 +130,15 @@ export interface ActorTrace { counts: Record; items: ActorTraceItem[]; tokenUsage?: ActorTokenUsage; + /** + * ADDITIVE + OPTIONAL token-derived cost ESTIMATE for this lane (humanish.actor-estimated-cost.v1). + * Distinct from `tokenUsage.costUsd`, which is RESERVED for a real provider-returned charge: a + * bare `costUsd` always means "the provider billed this", while `estimatedCost.estimatedCostUsd` + * is a rate-table multiply named honestly as an estimate (invariant 6). Absent on codex/scripted + * lanes and on every pre-existing bundle — its absence is tolerated by verify (fail-open on + * display). A `null` estimatedCostUsd is DECLARED ABSENT (unknown rate / no usage), never 0. + */ + estimatedCost?: ActorEstimatedCost; capabilities: ActorCapabilities; } diff --git a/src/index.ts b/src/index.ts index 112e79d..18109ca 100644 --- a/src/index.ts +++ b/src/index.ts @@ -63,6 +63,19 @@ export type { BrowserLabScoringContext } from "./adapter-extension.js"; export type { RedactionHooks } from "./redaction.js"; +export { + DESKTOP_RATE, + MODEL_RATES, + PRICING_SCHEMA, + estimateActorCost, + estimateDesktopCost +} from "./pricing.js"; +export type { + ActorEstimatedCost, + DesktopCostEstimate, + DesktopRate, + ModelRate +} from "./pricing.js"; export { normalizeCliArgv } from "./argv.js"; export { CODEX_APP_SERVER_UI_SCHEMA, diff --git a/src/pricing.ts b/src/pricing.ts new file mode 100644 index 0000000..172ec51 --- /dev/null +++ b/src/pricing.ts @@ -0,0 +1,172 @@ +// OPERATOR-EDITABLE ESTIMATES — NOT authoritative prices. Providers change pricing without +// notice. When they do, update BOTH the number AND the `asOf` date on the affected entry. +// Every dollar figure humanish derives from this table is surfaced/persisted as an ESTIMATE, +// labeled "estimated (rates as of )", and is NEVER presented as an exact charge. +// +// This module is PURE (node builtins only, no deps) and deterministic: the two estimator +// functions take an OPTIONAL injected rate table/rate so tests drive assertions with a fake +// sheet and never depend on the live numbers below. An UNKNOWN model/desktop rate yields a +// DECLARED-ABSENT estimate (estimatedCostUsd: null + a reason), NEVER a guessed or silent-zero +// cost (invariant 5). A bare `costUsd` elsewhere in the contract means a provider actually +// billed that amount; the token-derived estimate here always lives under `estimatedCostUsd` +// so a reader can never confuse an estimate for an authoritative charge (invariant 6). + +// A type-only import — erased at compile time, so the pricing <-> actor-contract cycle is not a +// runtime cycle. +import type { ActorTokenUsage } from "./actor-contract.js"; + +export const PRICING_SCHEMA = "humanish.pricing.v1"; +export const ACTOR_ESTIMATED_COST_SCHEMA = "humanish.actor-estimated-cost.v1"; + +export interface ModelRate { + /** USD per input token (the per-1M equivalent is noted in the comment beside each entry). */ + inputUsdPerToken: number; + /** USD per output token. */ + outputUsdPerToken: number; + /** "YYYY-MM-DD" the entry was last checked against `source`. */ + asOf: string; + /** Public pricing page the number came from (a comment/URL, never a secret). */ + source: string; + /** true = a stand-in NOT copied from a live sheet; the estimate carries this flag so a + * placeholder rate is never mistaken for a confirmed one. */ + placeholder?: boolean; +} + +export interface DesktopRate { + usdPerMinute: number; + asOf: string; + source: string; + placeholder?: boolean; +} + +/** + * The token-derived cost ESTIMATE for one actor lane. `estimatedCostUsd: null` = DECLARED ABSENT + * (unknown rate or no token usage) — never coerced to 0. A non-null figure ALWAYS carries its + * pricing provenance (`ratesAsOf` + `source`) so the mechanism (a rate-table multiply) matches + * the claim (an estimate, not a charge). Defined here so the rate table and the field that + * consumes it live together; `ActorTrace` imports it type-only. + */ +export interface ActorEstimatedCost { + schema: typeof ACTOR_ESTIMATED_COST_SCHEMA; + /** null = declared absent (no rate for the model / no token usage). */ + estimatedCostUsd: number | null; + reason?: "no_rate_for_model" | "no_token_usage"; + /** Pricing provenance date; null iff estimatedCostUsd is null. */ + ratesAsOf: string | null; + /** The pricing-page URL/comment that produced the rate. */ + source?: string; + /** The model id the estimate was keyed on. */ + modelId?: string; + /** true when the rate is a stand-in, not a live sheet. */ + placeholder?: boolean; + breakdown?: { inputUsd: number; outputUsd: number; inputTokens: number; outputTokens: number }; +} + +/** The desktop-minute cost ESTIMATE (host-side create->teardown span * a per-minute rate). Same + * null-discipline as ActorEstimatedCost: `estimatedCostUsd: null` = not measured (no duration). */ +export interface DesktopCostEstimate { + estimatedCostUsd: number | null; + reason?: "no_duration"; + ratesAsOf: string | null; + source?: string; + /** The billed minutes the estimate was keyed on; null when no duration was measured. */ + minutes: number | null; + placeholder?: boolean; +} + +// Per-model rates, keyed on the model id that lands in trace.ids.model (lookup is +// case-insensitive on a trimmed id). An id NOT present here is DECLARED ABSENT, never guessed. +export const MODEL_RATES: Record = { + // OpenAI computer-use-preview (the classic CUA model). ~$3 / 1M input, ~$12 / 1M output. + // source: openai.com/api/pricing (verify — providers change without notice). + "computer-use-preview": { + inputUsdPerToken: 3e-6, + outputUsdPerToken: 12e-6, + asOf: "2026-08-01", + source: "openai.com/api/pricing (computer-use-preview)" + }, + // gpt-5.5 = the shipped CUA default (DEFAULT_OPENAI_CU_MODEL). PLACEHOLDER shaped like a + // GPT-5-class rate (~$1.25 / 1M input, ~$10 / 1M output) until confirmed against the live sheet. + "gpt-5.5": { + inputUsdPerToken: 1.25e-6, + outputUsdPerToken: 10e-6, + asOf: "2026-08-01", + source: "PLACEHOLDER — confirm at openai.com/api/pricing", + placeholder: true + } +}; + +// E2B desktop sandbox compute, billed per-second by vCPU+RAM. PLACEHOLDER ~ $0.10 / hr for a +// ~2 vCPU desktop => ~$0.00167 / minute. Confirm at e2b.dev/pricing. +export const DESKTOP_RATE: DesktopRate = { + usdPerMinute: 0.00167, + asOf: "2026-08-01", + source: "PLACEHOLDER — confirm at e2b.dev/pricing", + placeholder: true +}; + +/** Round a USD figure to 6 decimals so a float-accumulated total never carries spurious + * precision. This mirrors the SPIRIT of the terminal ledger's private roundUsd (6dp) without + * importing it — pricing stays a standalone pure module. */ +export function round6(n: number): number { + return Math.round(n * 1e6) / 1e6; +} + +/** + * Estimate one actor lane's model-token cost from its trace tokenUsage + model id. Deterministic; + * the rate table is injectable (tests pass a fake sheet). Returns a DECLARED-ABSENT estimate + * (estimatedCostUsd: null + a reason) for a missing rate or missing usage — never a guessed cost. + */ +export function estimateActorCost( + tokenUsage: ActorTokenUsage | undefined, + modelId: string | undefined, + rates: Record = MODEL_RATES +): ActorEstimatedCost { + if (!tokenUsage || (tokenUsage.input === undefined && tokenUsage.output === undefined)) { + return { schema: ACTOR_ESTIMATED_COST_SCHEMA, estimatedCostUsd: null, reason: "no_token_usage", ratesAsOf: null }; + } + const rate = modelId ? rates[modelId.trim().toLowerCase()] : undefined; + if (!rate) { + return { + schema: ACTOR_ESTIMATED_COST_SCHEMA, + estimatedCostUsd: null, + reason: "no_rate_for_model", + ratesAsOf: null, + ...(modelId ? { modelId } : {}) + }; + } + const inTok = tokenUsage.input ?? 0; + const outTok = tokenUsage.output ?? 0; + const inputUsd = round6(inTok * rate.inputUsdPerToken); + const outputUsd = round6(outTok * rate.outputUsdPerToken); + return { + schema: ACTOR_ESTIMATED_COST_SCHEMA, + estimatedCostUsd: round6(inputUsd + outputUsd), + ratesAsOf: rate.asOf, + source: rate.source, + ...(modelId ? { modelId } : {}), + ...(rate.placeholder ? { placeholder: true } : {}), + breakdown: { inputUsd, outputUsd, inputTokens: inTok, outputTokens: outTok } + }; +} + +/** + * Estimate the E2B desktop-minute cost from a host-side create->teardown span (minutes). The rate + * is injectable. Returns a DECLARED-ABSENT estimate (null + "no_duration") when no duration was + * measured (no sandbox / unmeasurable span) — never a guessed 0. + */ +export function estimateDesktopCost( + minutes: number | undefined, + rate: DesktopRate = DESKTOP_RATE +): DesktopCostEstimate { + if (minutes === undefined || !(minutes >= 0)) { + return { estimatedCostUsd: null, reason: "no_duration", ratesAsOf: null, minutes: null }; + } + return { + estimatedCostUsd: round6(minutes * rate.usdPerMinute), + ratesAsOf: rate.asOf, + source: rate.source, + minutes: round6(minutes), + ...(rate.placeholder ? { placeholder: true } : {}) + }; +} diff --git a/tests/pricing.test.ts b/tests/pricing.test.ts new file mode 100644 index 0000000..310d7c8 --- /dev/null +++ b/tests/pricing.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from "vitest"; + +import { + ACTOR_ESTIMATED_COST_SCHEMA, + DESKTOP_RATE, + MODEL_RATES, + PRICING_SCHEMA, + estimateActorCost, + estimateDesktopCost, + round6, + type DesktopRate, + type ModelRate +} from "../src/pricing.js"; + +// A fake sheet so every assertion is driven by injected numbers, never the live table. +const FAKE_RATES: Record = { + "test-model": { + inputUsdPerToken: 2e-6, + outputUsdPerToken: 5e-6, + asOf: "2024-01-01", + source: "fake-sheet://test-model" + }, + "placeholder-model": { + inputUsdPerToken: 1e-6, + outputUsdPerToken: 1e-6, + asOf: "2024-02-02", + source: "fake-sheet://placeholder", + placeholder: true + } +}; +const FAKE_DESKTOP: DesktopRate = { usdPerMinute: 0.01, asOf: "2024-03-03", source: "fake-sheet://desktop" }; + +describe("pricing schema constants", () => { + it("names both schema tags at v1", () => { + expect(PRICING_SCHEMA).toBe("humanish.pricing.v1"); + expect(ACTOR_ESTIMATED_COST_SCHEMA).toBe("humanish.actor-estimated-cost.v1"); + }); + + it("keeps the shipped rate table keyed lowercase with the CUA default present", () => { + // The shipped default resolves to gpt-5.5 (DEFAULT_OPENAI_CU_MODEL); it must be priceable so a + // capped run is not refused by default. The classic computer-use-preview is a confirmed rate. + expect(MODEL_RATES["gpt-5.5"]?.placeholder).toBe(true); + expect(MODEL_RATES["computer-use-preview"]?.placeholder).toBeUndefined(); + expect(DESKTOP_RATE.placeholder).toBe(true); + for (const key of Object.keys(MODEL_RATES)) { + expect(key).toBe(key.toLowerCase()); + } + }); +}); + +describe("estimateActorCost", () => { + it("computes an EXACT rate-table multiply for a known model and carries provenance + breakdown", () => { + const est = estimateActorCost({ input: 1000, output: 200 }, "test-model", FAKE_RATES); + const inputUsd = round6(1000 * 2e-6); + const outputUsd = round6(200 * 5e-6); + expect(est.schema).toBe("humanish.actor-estimated-cost.v1"); + expect(est.estimatedCostUsd).toBe(round6(inputUsd + outputUsd)); + expect(est.ratesAsOf).toBe("2024-01-01"); + expect(est.source).toBe("fake-sheet://test-model"); + expect(est.modelId).toBe("test-model"); + expect(est.reason).toBeUndefined(); + expect(est.breakdown).toEqual({ inputUsd, outputUsd, inputTokens: 1000, outputTokens: 200 }); + }); + + it("is case-insensitive and trims the model id before lookup", () => { + const est = estimateActorCost({ input: 10, output: 0 }, " TEST-Model ", FAKE_RATES); + expect(est.estimatedCostUsd).toBe(round6(10 * 2e-6)); + expect(est.ratesAsOf).toBe("2024-01-01"); + }); + + it("DECLARES ABSENT (null + no_rate_for_model) for an unknown model — never a guessed cost", () => { + const est = estimateActorCost({ input: 1000, output: 200 }, "no-such-model", FAKE_RATES); + expect(est.estimatedCostUsd).toBeNull(); + expect(est.reason).toBe("no_rate_for_model"); + expect(est.ratesAsOf).toBeNull(); + expect(est.source).toBeUndefined(); + expect(est.modelId).toBe("no-such-model"); + }); + + it("DECLARES ABSENT (null + no_token_usage) for undefined or empty token usage", () => { + for (const usage of [undefined, {}, { total: 5 } as const]) { + const est = estimateActorCost(usage, "test-model", FAKE_RATES); + expect(est.estimatedCostUsd).toBeNull(); + expect(est.reason).toBe("no_token_usage"); + expect(est.ratesAsOf).toBeNull(); + } + }); + + it("propagates the placeholder flag from the rate into the estimate", () => { + const est = estimateActorCost({ input: 5, output: 5 }, "placeholder-model", FAKE_RATES); + expect(est.placeholder).toBe(true); + expect(est.estimatedCostUsd).toBe(round6(5 * 1e-6 + 5 * 1e-6)); + const known = estimateActorCost({ input: 5, output: 5 }, "test-model", FAKE_RATES); + expect(known.placeholder).toBeUndefined(); + }); + + it("treats a missing input or output token count as 0 (not absent) when the other is present", () => { + const est = estimateActorCost({ input: 100 }, "test-model", FAKE_RATES); + expect(est.estimatedCostUsd).toBe(round6(100 * 2e-6)); + expect(est.breakdown).toEqual({ inputUsd: round6(100 * 2e-6), outputUsd: 0, inputTokens: 100, outputTokens: 0 }); + }); +}); + +describe("estimateDesktopCost", () => { + it("computes minutes * usdPerMinute rounded, with provenance", () => { + const est = estimateDesktopCost(3, FAKE_DESKTOP); + expect(est.estimatedCostUsd).toBe(round6(3 * 0.01)); + expect(est.ratesAsOf).toBe("2024-03-03"); + expect(est.source).toBe("fake-sheet://desktop"); + expect(est.minutes).toBe(3); + }); + + it("DECLARES ABSENT for undefined, NaN, or negative minutes", () => { + for (const minutes of [undefined, Number.NaN, -1]) { + const est = estimateDesktopCost(minutes, FAKE_DESKTOP); + expect(est.estimatedCostUsd).toBeNull(); + expect(est.reason).toBe("no_duration"); + expect(est.ratesAsOf).toBeNull(); + expect(est.minutes).toBeNull(); + } + }); + + it("propagates the desktop placeholder flag", () => { + const est = estimateDesktopCost(2, { ...FAKE_DESKTOP, placeholder: true }); + expect(est.placeholder).toBe(true); + }); +}); + +describe("round6", () => { + it("kills float accumulation drift", () => { + // 0.1 + 0.2 = 0.30000000000000004; the ledger must not carry that spurious tail. + expect(round6(0.1 + 0.2)).toBe(0.3); + expect(round6(11.535069 + 0.070428)).toBe(11.605497); + }); +}); From 0c7eae5ac1b857c7d9fa610e03b8d57aebfb21d6 Mon Sep 17 00:00:00 2001 From: Daniel G Wilson Date: Sun, 2 Aug 2026 10:09:12 +0000 Subject: [PATCH 2/7] feat(run): additive run-cost-summary contract + fail-closed verify labeling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the optional, additive RunBundle.cost (humanish.run-cost-summary.v1) with RunCostLine/RunCostSummary: the sum of every lane's model-token estimate plus the E2B desktop-minute estimate, carrying the terminal ledger's null-discipline (a present-but-unpriceable line is null + a reason, contributes nothing to the total; an all-null summary has a null total, never 0). humanish.run-bundle.v1 stays v1 — every field is additive/optional so old bundles remain byte-stable and loadable. verifyPreparedRun gains a "cost estimate labeling" check (costLabelingFindings): absence PASSES (fail-open on display), but a CLAIMED dollar figure without its ratesAsOf date + source, or a total that does not equal round6(sum of only the non-null lines), FAILS. Magnitude is never inspected — a correctly-labeled huge estimate still passes. Co-Authored-By: Claude Opus 4.8 --- src/index.ts | 2 + src/run.ts | 140 ++++++++++++++++++++++++++++++++++++++++++++++ tests/run.test.ts | 138 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 279 insertions(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index 18109ca..9480a71 100644 --- a/src/index.ts +++ b/src/index.ts @@ -150,6 +150,8 @@ export type { RunAdapterScore, RunAttributionClass, RunBundle, + RunCostLine, + RunCostSummary, RunDesktopGeometry, RunEvent, RunFeedbackCandidate, diff --git a/src/run.ts b/src/run.ts index a7f7ce1..3d89bf6 100644 --- a/src/run.ts +++ b/src/run.ts @@ -34,6 +34,7 @@ import { mapWithConcurrency } from "./concurrency.js"; import { screenshotEvidenceError } from "./image-evidence.js"; import { buildObserverData } from "./observer-data.js"; import { parseResolvedPersona, personaToDirectives, renderPersonaPromptSection, type ResolvedPersona } from "./persona.js"; +import { round6 } from "./pricing.js"; import { containsSensitive, digestText, redactText, redactToSecretLabel, tailText } from "./redaction.js"; import type { E2BDesktopModule } from "./e2b-desktop-launch.js"; import { @@ -753,6 +754,58 @@ export interface RunBundle { * resource lease. Optional + additive; core never enumerates provider accounts. */ providerResources?: RunProviderResource[]; + /** + * OPTIONAL, ADDITIVE run-level cost ESTIMATE (humanish.run-cost-summary.v1): the sum of every + * lane's model-token estimate PLUS the E2B desktop-minute estimate, carrying the SAME + * null-discipline the terminal cost ledger already ships. Absent on every pre-existing bundle + * and on dry-runs that invent no spend (byte-stable). Every dollar figure here is an ESTIMATE, + * never an authoritative charge; verify asserts its LABELING/provenance, never its magnitude. + */ + cost?: RunCostSummary; +} + +/** + * One contributing cost line of a RunCostSummary. A line is PRESENT even when it cannot be priced + * (records that we TRIED and could not) — an unpriceable line carries estimatedCostUsd: null + a + * `reason` and contributes NOTHING to the summary total (invariant 5). `estimatedCostUsd` is NEVER + * coerced to 0. + */ +export interface RunCostLine { + kind: "model-tokens" | "desktop-minutes"; + laneId?: string; + modelId?: string; + /** null = NOT MEASURED / no rate; never coerced to 0. */ + estimatedCostUsd: number | null; + reason?: "no_rate_for_model" | "no_rate_for_desktop" | "no_token_usage" | "no_duration"; + /** Pricing provenance date; non-null iff estimatedCostUsd is non-null. */ + ratesAsOf: string | null; + source?: string; + placeholder?: boolean; +} + +/** + * The run-level cost ESTIMATE. `estimatedTotalUsd` is the rounded sum of ONLY the non-null + * `breakdown` lines; it is null iff EVERY line is null (never 0-coerced). `fullyEstimated` is + * false when any applicable line is null (the total is then a LOWER BOUND). Every non-null dollar + * figure carries `ratesAsOf`; `placeholder` is true when any contributing rate is a stand-in. + */ +export interface RunCostSummary { + schema: "humanish.run-cost-summary.v1"; + currency: "usd"; + /** Sum of the KNOWN (non-null) lines; null iff every applicable line is null. */ + estimatedTotalUsd: number | null; + /** Max asOf across contributing rates; null when nothing was priced. */ + ratesAsOf: string | null; + /** false when any applicable line is null (the total is a lower bound). */ + fullyEstimated: boolean; + /** true when any contributing rate is a placeholder (a stand-in, not a live sheet). */ + placeholder: boolean; + breakdown: RunCostLine[]; + tokenUsage: { input: number; output: number; total: number }; + /** Host-side create->teardown span in minutes; null when no sandbox was created. */ + desktopMinutes: number | null; + /** Honest "estimated; unmeasured" statement. */ + note: string; } export interface RunProviderResource { @@ -3893,6 +3946,18 @@ async function verifyPreparedRun( ? "rerun bundles either are absent or link selected lanes to prior lane status and a fan-out rerun event" : `rerun lineage findings: ${rerunFindings.join(", ")}` }); + // Cost is ADVISORY on magnitude, FAIL-CLOSED on labeling/provenance (claims match mechanism). + // Absence PASSES (fail-open on display); a claimed dollar figure without its ratesAsOf date + + // source, or a total that does not match its known lines, FAILS. Magnitude is never inspected — + // a correctly-labeled huge estimate still passes. + const costFindings = isRunBundle(bundle) ? costLabelingFindings(bundle) : []; + checks.push({ + name: "cost estimate labeling", + ok: costFindings.length === 0, + message: costFindings.length === 0 + ? "cost figures are absent, or every claimed estimate carries its ratesAsOf date + source and the total matches its known lines (estimates never presented as exact)" + : `cost labeling findings: ${costFindings.join(", ")}` + }); const ok = checks.every((check) => check.ok); const warnings = isRunBundle(bundle) @@ -5424,6 +5489,81 @@ function rerunLineageFindings(bundle: RunBundle): string[] { return findings; } +/** + * Verify the LABELING/provenance of any cost figure a bundle CLAIMS — never its magnitude. Returns + * [] (pass) unless a dollar claim lacks its provenance (invariant 6) or a total misreports its + * known lines. ABSENCE always passes (fail-open on display, discipline #3): a bundle with no cost, + * a null estimate, or a lane without estimatedCost is fine. A NON-NULL figure must carry its + * ratesAsOf date + source; a NUMBER total must equal round6(sum of ONLY the non-null lines) and a + * null line may never be coerced to 0. A null estimate must be declared honestly (a reason + null + * ratesAsOf), mirroring the terminal no-spend proof's null-discipline. + */ +function costLabelingFindings(bundle: RunBundle): string[] { + const findings: string[] = []; + + const cost = bundle.cost; + if (cost) { + if (cost.schema !== "humanish.run-cost-summary.v1") { + findings.push(`run cost summary schema is ${String(cost.schema)}, expected humanish.run-cost-summary.v1`); + } + let knownSum = 0; + let anyKnown = false; + for (const [index, line] of (cost.breakdown ?? []).entries()) { + if (line.estimatedCostUsd === null) { + continue; + } + anyKnown = true; + knownSum += line.estimatedCostUsd; + if (typeof line.ratesAsOf !== "string" || line.ratesAsOf.length === 0) { + findings.push(`cost breakdown line ${index} (${line.kind}) claims $${line.estimatedCostUsd} without a ratesAsOf date`); + } + if (typeof line.source !== "string" || line.source.length === 0) { + findings.push(`cost breakdown line ${index} (${line.kind}) claims $${line.estimatedCostUsd} without a pricing source`); + } + } + if (cost.estimatedTotalUsd !== null) { + if (typeof cost.ratesAsOf !== "string" || cost.ratesAsOf.length === 0) { + findings.push("run cost summary claims a number estimatedTotalUsd without a ratesAsOf date"); + } + if (round6(cost.estimatedTotalUsd) !== round6(knownSum)) { + findings.push(`run cost estimatedTotalUsd ${cost.estimatedTotalUsd} does not equal the sum of its known breakdown lines (${round6(knownSum)})`); + } + } else if (anyKnown) { + // Every-line-null is the only honest null total; a null total beside a known line hides spend. + findings.push("run cost estimatedTotalUsd is null but a breakdown line carries a known (non-null) cost"); + } + } + + for (const stream of bundle.streams) { + const estimate = stream.actor?.estimatedCost; + if (!estimate) { + continue; + } + const laneLabel = stream.laneId ?? stream.id; + if (estimate.schema !== "humanish.actor-estimated-cost.v1") { + findings.push(`lane ${laneLabel} actor estimatedCost schema is ${String(estimate.schema)}, expected humanish.actor-estimated-cost.v1`); + } + if (estimate.estimatedCostUsd !== null) { + if (typeof estimate.ratesAsOf !== "string" || estimate.ratesAsOf.length === 0) { + findings.push(`lane ${laneLabel} claims a model-token cost $${estimate.estimatedCostUsd} without a ratesAsOf date`); + } + if (typeof estimate.source !== "string" || estimate.source.length === 0) { + findings.push(`lane ${laneLabel} claims a model-token cost $${estimate.estimatedCostUsd} without a pricing source`); + } + } else { + // Declared-absent honesty (invariant 5): a null estimate must say WHY and carry null ratesAsOf. + if (estimate.reason === undefined) { + findings.push(`lane ${laneLabel} records a null cost estimate without a reason`); + } + if (estimate.ratesAsOf !== null) { + findings.push(`lane ${laneLabel} records a null cost estimate but carries a non-null ratesAsOf`); + } + } + } + + return findings; +} + // SEQUENTIAL: the three disclosures a sequential shared-world bundle MUST pin (verify fails closed // if any is absent — omission overclaims): sequential turns only, no concurrency/races handled, and // a checkpoint delta is attributed to the TURN it followed, not a specific action (correlation). diff --git a/tests/run.test.ts b/tests/run.test.ts index c4cb595..96a05fe 100644 --- a/tests/run.test.ts +++ b/tests/run.test.ts @@ -23,6 +23,7 @@ import { readReview, runDryRun, verifyRun, + type RunCostSummary, type RunSubjectProvenance, type RunSubjectStateStepRecord } from "../src/run.js"; @@ -2965,7 +2966,7 @@ function cuaActorTrace(args: { async function writeCuaRunFixture( cwd: string, runId: string, - args: { dryRun: boolean; trace?: ActorTrace; subject?: RunSubjectProvenance; forceReviewVerdict?: "pass" | "fail" | "blocked" | "timed_out" | "contract_proof_only" } + args: { dryRun: boolean; trace?: ActorTrace; subject?: RunSubjectProvenance; cost?: RunCostSummary; forceReviewVerdict?: "pass" | "fail" | "blocked" | "timed_out" | "contract_proof_only" } ): Promise { const session: CuaLoopResult | undefined = args.trace ? { @@ -2994,6 +2995,9 @@ async function writeCuaRunFixture( if (args.subject) { bundle.subject = args.subject; } + if (args.cost) { + bundle.cost = args.cost; + } if (args.forceReviewVerdict) { bundle.review.verdict = args.forceReviewVerdict; } @@ -3472,3 +3476,135 @@ describe("verify: subject provenance (local-tree)", () => { }); }); }); + +describe("verify: cost estimate labeling", () => { + // An ENGAGED live trace so the review verdict is a genuine pass; that isolates the cost check + // from the actor-engagement gate. Cost is ADVISORY on magnitude, FAIL-CLOSED on provenance. + const engagedTrace = (estimatedCost?: ActorTrace["estimatedCost"]): ActorTrace => { + const trace = cuaActorTrace({ + counts: { turns: 2, actions: 1, screenshots: 0, reasonings: 0, messages: 1, idleTurns: 0, noProgressTurns: 0 }, + items: [ + { id: "action-001", kind: "ui_action", lifecycle: "completed", title: "click (11, 22)" }, + { id: "message-001", kind: "message", lifecycle: "completed", title: "message", text: "Done." } + ] + }); + if (estimatedCost) trace.estimatedCost = estimatedCost; + return trace; + }; + const costCheck = (verify: Awaited>) => + verify.checks.find((entry) => entry.name === "cost estimate labeling"); + const labeledSummary = (overrides: Partial = {}): RunCostSummary => ({ + schema: "humanish.run-cost-summary.v1", + currency: "usd", + estimatedTotalUsd: 11.6, + ratesAsOf: "2026-08-01", + fullyEstimated: false, + placeholder: false, + breakdown: [ + { kind: "model-tokens", laneId: "lane-01", modelId: "computer-use-preview", estimatedCostUsd: 11.6, ratesAsOf: "2026-08-01", source: "openai.com/api/pricing (computer-use-preview)" }, + { kind: "desktop-minutes", estimatedCostUsd: null, reason: "no_duration", ratesAsOf: null } + ], + tokenUsage: { input: 3843523, output: 5869, total: 3849392 }, + desktopMinutes: null, + note: "Estimated model-token cost; desktop minutes unmeasured.", + ...overrides + }); + + it("passes when the bundle carries NO cost at all (fail-open on absence)", async () => { + await withFixtureCopy(async (cwd) => { + await writeCuaRunFixture(cwd, "cost-absent", { dryRun: false, trace: engagedTrace() }); + const verify = await verifyRun(cwd, "cost-absent"); + expect(costCheck(verify)?.ok).toBe(true); + expect(verify.ok).toBe(true); + }); + }); + + it("passes a properly-labeled estimate and a HUGE but correctly-labeled estimate (magnitude never fails)", async () => { + await withFixtureCopy(async (cwd) => { + await writeCuaRunFixture(cwd, "cost-labeled", { dryRun: false, trace: engagedTrace(), cost: labeledSummary() }); + const labeled = await verifyRun(cwd, "cost-labeled"); + expect(costCheck(labeled)?.ok).toBe(true); + expect(labeled.ok).toBe(true); + + await writeCuaRunFixture(cwd, "cost-huge", { + dryRun: false, + trace: engagedTrace(), + cost: labeledSummary({ + estimatedTotalUsd: 1_000_000, + breakdown: [ + { kind: "model-tokens", laneId: "lane-01", modelId: "computer-use-preview", estimatedCostUsd: 1_000_000, ratesAsOf: "2026-08-01", source: "openai.com/api/pricing" } + ] + }) + }); + const huge = await verifyRun(cwd, "cost-huge"); + expect(costCheck(huge)?.ok).toBe(true); + expect(huge.ok).toBe(true); + }); + }); + + it("FAILS a number total that lacks its ratesAsOf date (a token-derived charge without provenance)", async () => { + await withFixtureCopy(async (cwd) => { + await writeCuaRunFixture(cwd, "cost-no-rates", { + dryRun: false, + trace: engagedTrace(), + cost: labeledSummary({ ratesAsOf: null }) + }); + const verify = await verifyRun(cwd, "cost-no-rates"); + expect(costCheck(verify)?.ok).toBe(false); + expect(verify.ok).toBe(false); + }); + }); + + it("FAILS a total that does not equal the sum of its known breakdown lines", async () => { + await withFixtureCopy(async (cwd) => { + await writeCuaRunFixture(cwd, "cost-mismatch", { + dryRun: false, + trace: engagedTrace(), + cost: labeledSummary({ estimatedTotalUsd: 99.99 }) + }); + const verify = await verifyRun(cwd, "cost-mismatch"); + expect(costCheck(verify)?.ok).toBe(false); + }); + }); + + it("FAILS a null total sitting beside a known (non-null) breakdown line", async () => { + await withFixtureCopy(async (cwd) => { + await writeCuaRunFixture(cwd, "cost-null-hides", { + dryRun: false, + trace: engagedTrace(), + cost: labeledSummary({ estimatedTotalUsd: null }) + }); + const verify = await verifyRun(cwd, "cost-null-hides"); + expect(costCheck(verify)?.ok).toBe(false); + }); + }); + + it("asserts per-actor estimate labeling: a number estimate without ratesAsOf fails; a declared-absent null passes", async () => { + await withFixtureCopy(async (cwd) => { + await writeCuaRunFixture(cwd, "actor-cost-bad", { + dryRun: false, + trace: engagedTrace({ + schema: "humanish.actor-estimated-cost.v1", + estimatedCostUsd: 4.86, + ratesAsOf: null, + source: "openai.com/api/pricing" + }) + }); + expect(costCheck(await verifyRun(cwd, "actor-cost-bad"))?.ok).toBe(false); + + await writeCuaRunFixture(cwd, "actor-cost-null", { + dryRun: false, + trace: engagedTrace({ + schema: "humanish.actor-estimated-cost.v1", + estimatedCostUsd: null, + reason: "no_rate_for_model", + ratesAsOf: null, + modelId: "mystery-model" + }) + }); + const nullVerify = await verifyRun(cwd, "actor-cost-null"); + expect(costCheck(nullVerify)?.ok).toBe(true); + expect(nullVerify.ok).toBe(true); + }); + }); +}); From 1d23eef38a80f19c5e05d8ed50273be247d0cb2e Mon Sep 17 00:00:00 2001 From: Daniel G Wilson Date: Sun, 2 Aug 2026 10:11:53 +0000 Subject: [PATCH 3/7] feat(cua): fail-closed maxUsd spend cap in the computer-use loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add optional maxUsd + an injected pure estimateTurnCostUsd to CuaLoopOptions (threaded through CuaActorSessionOptions). When maxUsd is set, the loop aborts (completionReason "budget_reached") the moment the running estimated spend crosses it — placed right after per-turn usage accumulation and BEFORE the next provider.nextTurn, so a model stuck retrying cannot spend one extra turn past the cap (the runaway-retry guard). Absent maxUsd is a no-op (byte-stable), and a null/unpriceable estimate can never trip the cap (never a silent-zero abort). The estimator is injected so the loop stays free of the operator rate table and the cap is deterministic in tests. Co-Authored-By: Claude Opus 4.8 --- src/computer-use-actor.ts | 8 +++- src/computer-use.ts | 32 +++++++++++++- tests/computer-use.test.ts | 85 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 2 deletions(-) diff --git a/src/computer-use-actor.ts b/src/computer-use-actor.ts index f64b2c1..805fd6e 100644 --- a/src/computer-use-actor.ts +++ b/src/computer-use-actor.ts @@ -75,6 +75,10 @@ export interface CuaActorSessionOptions { writeScreenshot?: (name: string, bytes: Buffer) => Promise; /** Deterministic harness-owned stop guards evaluated between model turns. */ stopWhen?: StopWhen; + /** FAIL-CLOSED spend cap (USD) threaded to the loop; absent = uncapped. See CuaLoopOptions.maxUsd. */ + maxUsd?: number; + /** Injected pure per-turn cost estimator paired with `maxUsd`. See CuaLoopOptions.estimateTurnCostUsd. */ + estimateTurnCostUsd?: (input: number, output: number) => number | null; } export async function runCuaActorSession(options: CuaActorSessionOptions): Promise { @@ -96,7 +100,9 @@ export async function runCuaActorSession(options: CuaActorSessionOptions): Promi ...(options.redactScreenshots === undefined ? {} : { redactScreenshots: options.redactScreenshots }), ...(options.scrubText === undefined ? {} : { scrubText: options.scrubText }), ...(options.writeScreenshot === undefined ? {} : { writeScreenshot: options.writeScreenshot }), - ...(options.stopWhen === undefined ? {} : { stopWhen: options.stopWhen }) + ...(options.stopWhen === undefined ? {} : { stopWhen: options.stopWhen }), + ...(options.maxUsd === undefined ? {} : { maxUsd: options.maxUsd }), + ...(options.estimateTurnCostUsd === undefined ? {} : { estimateTurnCostUsd: options.estimateTurnCostUsd }) }; return runComputerUseLoop(loopOptions); diff --git a/src/computer-use.ts b/src/computer-use.ts index c8c1c4c..2e40652 100644 --- a/src/computer-use.ts +++ b/src/computer-use.ts @@ -215,6 +215,22 @@ export interface CuaLoopOptions { * wandering after the product already reached an app-visible endpoint. */ stopWhen?: StopWhen; + /** + * FAIL-CLOSED spend cap (USD). When set, the loop aborts (completionReason "budget_reached") + * the moment the running ESTIMATED spend crosses it, BEFORE the next provider turn — the + * runaway-retry-loop guard. Absent = uncapped (the historical CUA behavior). maxUsd: 0 means + * no-spend (any measurable estimate > 0 aborts). Enforcement needs a measurable estimate, so + * the lab refuses a cap on an unpriced model at PREFLIGHT rather than running uncapped. + */ + maxUsd?: number; + /** + * Injected PURE per-turn cost estimator (keeps the loop free of the operator rate table and + * makes the cap deterministic in tests). Given running (input, output) token totals, returns the + * estimated USD, or null when unpriceable. Only consulted when `maxUsd` is set. A null estimate + * mid-run cannot trip the cap — preflight already guaranteed a rate exists, so a null here is a + * vanished-rate harness condition, not a silent uncapped pass. + */ + estimateTurnCostUsd?: (input: number, output: number) => number | null; } export interface CuaLoopResult { @@ -432,7 +448,9 @@ export async function runComputerUseLoop(options: CuaLoopOptions): Promise text, writeScreenshot = async (name) => `screenshots/${name}`, - stopWhen + stopWhen, + maxUsd, + estimateTurnCostUsd } = options; const noProgressRecoverySteps = Math.min(Math.max(1, noProgressSteps - 1), 3); const idleRecoverySteps = Math.min(Math.max(1, idleSteps - 1), 3); @@ -599,6 +617,18 @@ export async function runComputerUseLoop(options: CuaLoopOptions): Promise maxUsd) { + completionReason = "budget_reached"; + reason = `estimated spend $${running} crossed execution.caps.maxUsd=$${maxUsd}; aborted fail-closed before the next model turn`; + break; + } + } if (turn.reasoning) { items.push({ id: nextId("reasoning"), diff --git a/tests/computer-use.test.ts b/tests/computer-use.test.ts index 91c31a9..eb4eed2 100644 --- a/tests/computer-use.test.ts +++ b/tests/computer-use.test.ts @@ -989,3 +989,88 @@ describe("runComputerUseLoop vision-provider frame guard (issue #148, RUNG 3)", expect(result.completionReason).toBe("goal_satisfied"); }); }); + +describe("runComputerUseLoop fail-closed maxUsd cap", () => { + // A non-idle turn that also reports fixed per-turn usage; a RepeatProvider emits it forever so + // only the cap (or a backstop) can stop the loop. + const usageTurn: CuaTurn = { + actions: [{ kind: "click", x: 10, y: 20 }], + pendingSafetyChecks: [], + done: false, + usage: { input: 100, output: 50 } + }; + // An injected PURE estimator with a fake rate (no live pricing table): $0.001 per token. + const estimateTurnCostUsd = (input: number, output: number): number => (input + output) * 0.001; + + it("aborts fail-closed the moment the running estimate crosses maxUsd, BEFORE the next model turn", async () => { + const provider = new RepeatProvider(usageTurn); + const executor = new SignatureExecutor(["s0", "s1", "s2", "s3", "s4", "s5"]); + + const result = await runComputerUseLoop({ + instructions: "Drive until the budget cap fires.", + provider, + executor, + persona, + redaction: defaultRedactionHooks, + timeoutMs: 10_000_000, + now: monotonicClock(), + maxUsd: 0.35, + estimateTurnCostUsd + }); + + // Cumulative estimate: turn1 $0.15, turn2 $0.30, turn3 $0.45 > $0.35 → break at turn 3. + expect(result.completionReason).toBe("budget_reached"); + expect(result.status).toBe("passed"); + expect(result.reason).toContain("crossed execution.caps.maxUsd=$0.35"); + // The cap fires BEFORE the next provider.nextTurn: exactly 3 turns were requested, no 4th. + expect(provider.seen).toHaveLength(3); + expect(result.trace.tokenUsage).toEqual({ input: 300, output: 150, total: 450 }); + }); + + it("is a no-op when maxUsd is unset — the loop runs to its natural completion unchanged", async () => { + const provider = new ScriptedProvider([ + { actions: [{ kind: "click", x: 10, y: 20 }], pendingSafetyChecks: [], done: false, usage: { input: 100, output: 50 } }, + { actions: [], pendingSafetyChecks: [], done: true, message: "Done.", usage: { input: 100, output: 50 } } + ]); + const executor = new SignatureExecutor(["s0", "s1", "s2"]); + + const result = await runComputerUseLoop({ + instructions: "Finish naturally.", + provider, + executor, + persona, + redaction: defaultRedactionHooks, + timeoutMs: 10_000_000, + now: monotonicClock(), + // estimator present but no cap → the cap branch is never consulted. + estimateTurnCostUsd + }); + + expect(result.completionReason).toBe("goal_satisfied"); + expect(result.reason).toBe("Done."); + expect(result.trace.tokenUsage).toEqual({ input: 200, output: 100, total: 300 }); + }); + + it("cannot trip on a null (unpriceable) estimate mid-run — it runs to natural completion", async () => { + const provider = new ScriptedProvider([ + { actions: [{ kind: "click", x: 10, y: 20 }], pendingSafetyChecks: [], done: false, usage: { input: 100, output: 50 } }, + { actions: [], pendingSafetyChecks: [], done: true, message: "Done.", usage: { input: 100, output: 50 } } + ]); + const executor = new SignatureExecutor(["s0", "s1", "s2"]); + + const result = await runComputerUseLoop({ + instructions: "Finish naturally.", + provider, + executor, + persona, + redaction: defaultRedactionHooks, + timeoutMs: 10_000_000, + now: monotonicClock(), + maxUsd: 0, + // A vanished rate returns null; the cap must NOT abort on it (never a silent-zero abort). + estimateTurnCostUsd: () => null + }); + + expect(result.completionReason).toBe("goal_satisfied"); + }); +}); From 9aac740fdc30b883b784c117a6d3e3b93b8124ce Mon Sep 17 00:00:00 2001 From: Daniel G Wilson Date: Sun, 2 Aug 2026 10:25:43 +0000 Subject: [PATCH 4/7] feat(cua-lab): per-lane + run-level estimated cost, desktop-minutes, unpriced-cap preflight Attach a token-derived estimatedCost to each CUA lane's trace at the lab boundary (before persist), measure the host-side E2B desktop create->teardown span via an injected clock (deterministic in tests), and assemble a run-level RunCostSummary (per-lane model-token lines + one aggregate desktop-minute line) on both the single-lane and fan-out bundles. The null-discipline is honored end-to-end: an unpriced model yields a present-but-null line that contributes nothing to the total; dry-run and in-process lanes invent no spend (cost omitted, byte-stable). Wire execution.caps.maxUsd into the loop as the fail-closed abort (with an injected pure per-turn estimator), reusing the terminal lane's LabScenarioCaps shape (parsed via the existing parseCaps; inert/warned off the CUA route). Resolve the invariant tension fail-closed at PREFLIGHT: a maxUsd cap on a model src/pricing.ts cannot price is refused (HUMANISH_CUA_LAB_UNPRICED_CAP) before any sandbox rather than run uncapped. Thread the clock into the concurrent shared-world lane too. Co-Authored-By: Claude Opus 4.8 --- src/concurrent-shared-world-lab.ts | 1 + src/cua-actor-lab.ts | 196 ++++++++++++++++++++++++++++- src/lab-config.ts | 19 +++ tests/cua-actor-lab.fanout.test.ts | 59 +++++++++ tests/cua-actor-lab.test.ts | 166 ++++++++++++++++++++++++ 5 files changed, 439 insertions(+), 2 deletions(-) diff --git a/src/concurrent-shared-world-lab.ts b/src/concurrent-shared-world-lab.ts index 2ae2f75..b5267f1 100644 --- a/src/concurrent-shared-world-lab.ts +++ b/src/concurrent-shared-world-lab.ts @@ -713,6 +713,7 @@ export async function runConcurrentSharedWorld(options: RunConcurrentSharedWorld redactScreenshots, scrubKnownValues, runSession, + now, hooks: cuaHooks, // Concurrent lanes are independent evidence seats: a requested-vs-verified screen // mismatch is recorded as separate facts + a warning instead of failing the lane's diff --git a/src/cua-actor-lab.ts b/src/cua-actor-lab.ts index 8e646cb..816cba9 100644 --- a/src/cua-actor-lab.ts +++ b/src/cua-actor-lab.ts @@ -40,6 +40,7 @@ import { } from "./browser-evidence-hygiene.js"; import type { CuaActorSessionOptions } from "./computer-use-actor.js"; import type { CuaExecutor, CuaLoopResult, CuaProvider } from "./computer-use.js"; +import { DEFAULT_OPENAI_CU_MODEL } from "./openai-responses-cu.js"; import type { E2BDesktopLike } from "./e2b-desktop-executor.js"; import { createDesktopSandbox, @@ -114,9 +115,12 @@ import { type RunSimulationStatus, type RunStream, type RunProviderResource, + type RunCostLine, + type RunCostSummary, type RunSubjectProvenance, type RunSubjectStateStepRecord } from "./run.js"; +import { estimateActorCost, estimateDesktopCost, MODEL_RATES, round6 } from "./pricing.js"; export const CUA_ACTOR_LAB_SCHEMA = "humanish.cua-lab-result.v2"; @@ -259,6 +263,10 @@ export interface CuaActorLabHooks extends BrowserLabAdapterHooks { buildProvider?: (ctx: { config: LabConfig; actor: CuaActorDescriptor }) => Promise; env?: Record; renderObserverFn?: typeof renderObserver; + /** Injected clock (ms) for the host-side E2B desktop create->teardown span measurement that + * feeds the desktop-minute cost estimate. Defaults to Date.now; tests inject a frozen/stepped + * clock so the desktop-minute line is deterministic. */ + now?: () => number; /** Injected clock/sleep for the detached-step polling (tests only). */ detachedTimers?: DetachedTimers; /** @@ -390,6 +398,10 @@ export type CuaActorLabErrorCode = | "HUMANISH_CUA_LAB_FANOUT_INVALID" | "HUMANISH_CUA_LAB_RERUN_INVALID" | "HUMANISH_CUA_LAB_DEVICE_GEOMETRY" + // A fail-closed spend cap (execution.caps.maxUsd) was set but src/pricing.ts has no rate for the + // resolved model, so the cap could not be enforced. Refused at preflight (before any sandbox) + // rather than run uncapped — an unenforceable cap is more dangerous than none. + | "HUMANISH_CUA_LAB_UNPRICED_CAP" // watch --expose (tunnel-edge auth) validation + tunnel-startup failures surfaced by runCuaBackend // before or around the run. Carried on the CUA lab envelope so `watch --expose` refusals // render through the same formatter as any other CUA lab failure. @@ -910,6 +922,9 @@ export interface CuaLaneDeps { redactScreenshots: boolean; scrubKnownValues: (text: string) => string; runSession: (options: CuaActorSessionOptions) => Promise; + /** Injected clock (ms). Used to measure the host-side E2B desktop create->teardown span so the + * desktop-minute cost estimate is deterministic in tests. Defaults to Date.now. */ + now: () => number; hooks: CuaActorLabHooks; /** Lane-0 only: signal the pipeline gate after provisioning succeeds (true) or fails (false). */ signalProvisioned?: (ok: boolean) => void; @@ -929,6 +944,10 @@ export interface LaneRunOutcome { session?: CuaLoopResult; sessionError?: string; sandboxId?: string; + /** Host-side E2B desktop create->teardown span (ms). An APPROXIMATION of E2B's server-side + * billed lifetime (server-side kill-on-timeout can extend it) — so the derived dollar figure is + * doubly an estimate. Absent on the in-process route (no sandbox) and on dry-run. */ + desktopDurationMs?: number; killed: boolean; streamUrlPresent: boolean; screenshots: string[]; @@ -1644,6 +1663,11 @@ export async function runCuaLane(spec: CuaLaneSpec, deps: CuaLaneDeps): Promise< let sessionError: string | undefined; let failureCode: CuaActorLabErrorCode | undefined; let sandboxId: string | undefined; + // Host-side E2B desktop billed-span endpoints, measured via the injected clock. Captured right + // after create() succeeds and again in the finally after teardown resolves (both the killed and + // kept-for-debug paths), so the desktop-minute cost estimate reflects the honest lifetime. + let sandboxCreatedAtMs: number | undefined; + let sandboxTornDownAtMs: number | undefined; let killed = false; let streamUrl: string | undefined; let subjectCommit: string | undefined; @@ -1694,6 +1718,8 @@ export async function runCuaLane(spec: CuaLaneSpec, deps: CuaLaneDeps): Promise< lifecycle: { onTimeout: "kill" } }, config.execution?.desktop?.template); sandboxId = desktop.sandboxId; + // The billed span starts the instant the sandbox exists. + sandboxCreatedAtMs = deps.now(); if (deps.hooks.prepareDesktop) { await deps.hooks.prepareDesktop(desktop, { laneId: spec.laneId, laneIndex: spec.laneIndex, laneCount: deps.laneCount }); @@ -1817,6 +1843,12 @@ export async function runCuaLane(spec: CuaLaneSpec, deps: CuaLaneDeps): Promise< warnings.push(`Live desktop stream unavailable (run continues; evidence still captured): ${redactText(deps.scrubKnownValues(toErrorMessage(error)))}`); } + // The FAIL-CLOSED spend cap (execution.caps.maxUsd) is wired into the loop as maxUsd + an + // injected pure per-turn estimator keyed on the resolved model. Preflight already refused a + // cap on an unpriced model, so the estimate is measurable whenever a cap is in force. The + // model id here matches provider.version (openai-responses-cu resolves the default when unset). + const capModelId = config.actors[0]?.model ?? DEFAULT_OPENAI_CU_MODEL; + const maxUsd = config.execution?.caps?.maxUsd; const sessionOptions: CuaActorSessionOptions = { instructions: spec.instructions, persona: spec.persona, @@ -1825,6 +1857,13 @@ export async function runCuaLane(spec: CuaLaneSpec, deps: CuaLaneDeps): Promise< apiKey: deps.openaiApiKey, ...(config.actors[0]?.model ? { model: config.actors[0]!.model } : {}) }, + ...(maxUsd === undefined + ? {} + : { + maxUsd, + estimateTurnCostUsd: (input: number, output: number): number | null => + estimateActorCost({ input, output }, capModelId).estimatedCostUsd + }), desktop: desktop as unknown as E2BDesktopLike, ...(launchedBrowserFamily === "chromium" ? { @@ -1910,10 +1949,24 @@ export async function runCuaLane(spec: CuaLaneSpec, deps: CuaLaneDeps): Promise< } else { warnings.push("Installed @e2b/desktop SDK does not expose Sandbox.kill; server-side kill-on-timeout will reclaim the sandbox."); } + // Close the billed span for BOTH the killed and kept-for-debug paths (a kept sandbox is + // still billed until its server-side timeout, so the honest span ends here either way). + sandboxTornDownAtMs = deps.now(); } } + // Host-side approximation of the E2B desktop's billed lifetime; feeds the desktop-minute cost + // estimate. Never negative. + const desktopDurationMs = sandboxCreatedAtMs !== undefined && sandboxTornDownAtMs !== undefined + ? Math.max(0, sandboxTornDownAtMs - sandboxCreatedAtMs) + : undefined; + if (session) { + // Per-lane model-token cost ESTIMATE, attached to the trace before it is persisted (the model + // id is authoritative here — provider.version). Kept at the lab boundary so the pure loop + // never depends on the operator rate table. estimateActorCost declares absent (null) for an + // unknown rate / missing usage rather than guessing. + session.trace.estimatedCost = estimateActorCost(session.trace.tokenUsage, session.trace.ids.model); await writeContainedOutputFile(deps.artifactRoot, spec.traceArtifactPath, `${JSON.stringify(session.trace, null, 2)}\n`, "utf8"); if (session.trace.redaction.screenshots === "raw") { warnings.push("Screenshots are full-fidelity (raw) for local use — the bundle stays in gitignored .humanish and nothing scans these pixels; review them before sharing anywhere. Set policies.redactScreenshots: true to blur a share-as-is bundle."); @@ -1944,6 +1997,7 @@ export async function runCuaLane(spec: CuaLaneSpec, deps: CuaLaneDeps): Promise< ...(session ? { session } : {}), ...(sessionError === undefined ? {} : { sessionError }), ...(sandboxId === undefined ? {} : { sandboxId }), + ...(desktopDurationMs === undefined ? {} : { desktopDurationMs }), killed, streamUrlPresent: streamUrl !== undefined, screenshots, @@ -2470,6 +2524,22 @@ export async function runCuaActorLab(options: RunCuaActorLabOptions): Promise; + desktopMinutes: number | undefined; +}): RunCostSummary | undefined { + const breakdown: RunCostLine[] = []; + let sumInput = 0; + let sumOutput = 0; + + for (const lane of args.lanes) { + const usage = lane.trace.tokenUsage; + if (usage) { + sumInput += usage.input ?? 0; + sumOutput += usage.output ?? 0; + } + const est = lane.trace.estimatedCost; + if (!est) { + continue; + } + breakdown.push({ + kind: "model-tokens", + ...(lane.laneId === undefined ? {} : { laneId: lane.laneId }), + ...(est.modelId === undefined ? {} : { modelId: est.modelId }), + estimatedCostUsd: est.estimatedCostUsd, + ...(est.reason === undefined ? {} : { reason: est.reason }), + ratesAsOf: est.ratesAsOf, + ...(est.source === undefined ? {} : { source: est.source }), + ...(est.placeholder ? { placeholder: true } : {}) + }); + } + + if (args.desktopMinutes !== undefined) { + const desktop = estimateDesktopCost(args.desktopMinutes); + breakdown.push({ + kind: "desktop-minutes", + estimatedCostUsd: desktop.estimatedCostUsd, + ...(desktop.reason === undefined ? {} : { reason: desktop.reason }), + ratesAsOf: desktop.ratesAsOf, + ...(desktop.source === undefined ? {} : { source: desktop.source }), + ...(desktop.placeholder ? { placeholder: true } : {}) + }); + } + + if (breakdown.length === 0) { + return undefined; + } + + let knownSum = 0; + let anyKnown = false; + let anyNull = false; + let placeholder = false; + let maxRatesAsOf: string | null = null; + for (const line of breakdown) { + if (line.estimatedCostUsd === null) { + anyNull = true; + continue; + } + anyKnown = true; + knownSum += line.estimatedCostUsd; + if (line.placeholder) placeholder = true; + if (line.ratesAsOf !== null && (maxRatesAsOf === null || line.ratesAsOf > maxRatesAsOf)) { + maxRatesAsOf = line.ratesAsOf; + } + } + const estimatedTotalUsd = anyKnown ? round6(knownSum) : null; + const note = estimatedTotalUsd === null + ? `No priced spend lines this run — every cost line is DECLARED ABSENT (unknown rate / no usage / no duration); nothing is guessed. Add a rate to src/pricing.ts to estimate this model.` + : `Estimated ${estimatedTotalUsd} USD total${anyNull ? " (LOWER BOUND — some lines unmeasured/unpriced)" : ""}${placeholder ? "; includes PLACEHOLDER rate(s) — confirm before trusting the magnitude" : ""}. Every figure is an ESTIMATE (rates as of ${maxRatesAsOf}), a rate-table multiply, NOT an authoritative provider charge.`; + + return { + schema: "humanish.run-cost-summary.v1", + currency: "usd", + estimatedTotalUsd, + ratesAsOf: maxRatesAsOf, + fullyEstimated: !anyNull, + placeholder, + breakdown, + tokenUsage: { input: sumInput, output: sumOutput, total: sumInput + sumOutput }, + desktopMinutes: args.desktopMinutes ?? null, + note + }; +} + +// Convert a host-side desktop span (ms) into billed minutes, or undefined when no sandbox ran. +function desktopSpanToMinutes(desktopDurationMs: number | undefined): number | undefined { + return desktopDurationMs === undefined ? undefined : desktopDurationMs / 60_000; +} + export function buildCuaBundle(args: { actorId: string; appUrl: string; @@ -3431,8 +3601,16 @@ export function buildCuaBundle(args: { /** Completed subject-phase records (clone/upload/extract/install/build/ready/state groups) * to fold into bundle.events, so run.json carries real phase timing after the fact. */ phaseEvents?: SubjectPhaseEvent[]; + /** Host-side E2B desktop billed span for this lane, in minutes (from LaneRunOutcome + * desktopDurationMs). Absent when no sandbox ran (in-process/dry-run) → no desktop cost line. */ + desktopMinutes?: number; }): RunBundle { const publicAppUrl = publicSafeAppUrlLabel(args.appUrl); + // Run-level cost ESTIMATE (advisory; omitted when nothing was priced and no sandbox ran). + const cost = buildCuaCostSummary({ + lanes: args.session ? [{ ...(args.laneId === undefined ? {} : { laneId: args.laneId }), trace: args.session.trace }] : [], + desktopMinutes: args.desktopMinutes + }); const status: RunSimulationStatus = args.inProgress === true ? "running" : args.session @@ -3712,7 +3890,8 @@ export function buildCuaBundle(args: { // honest on app-url bundles too — the caller minted the URL, its state is the caller's. // CuaSubjectProvenanceArg's two variants (clone, local-tree) are already RunSubjectProvenance- // shaped, so no reconstruction is needed beyond the app-url fallback. - subject: args.subjectProvenance ?? { source: "app-url", state: { provenance: "undeclared" } } + subject: args.subjectProvenance ?? { source: "app-url", state: { provenance: "undeclared" } }, + ...(cost === undefined ? {} : { cost }) }; } @@ -4113,6 +4292,18 @@ export function buildCuaFanoutBundle(args: { laneId: outcome.spec.laneId })); + // Run-level cost ESTIMATE: one model-token line per lane that ran a session (from its persisted + // trace.estimatedCost) + one aggregate desktop-minutes line summing each lane's OWN sandbox span + // (per-lane worlds => no shared provisioning to double-count). Omitted on a pure dry-run. + const costLanes = specs + .map((spec, index) => ({ laneId: spec.laneId, outcome: outcomes?.[index] })) + .filter((entry): entry is { laneId: string; outcome: LaneRunOutcome } => entry.outcome?.session !== undefined) + .map((entry) => ({ laneId: entry.laneId, trace: entry.outcome.session!.trace })); + const desktopMinutesTotal = (outcomes ?? []).some((outcome) => outcome.desktopDurationMs !== undefined) + ? (outcomes ?? []).reduce((sum, outcome) => sum + (outcome.desktopDurationMs ?? 0), 0) / 60_000 + : undefined; + const cost = buildCuaCostSummary({ lanes: costLanes, desktopMinutes: desktopMinutesTotal }); + return { schema: RUN_BUNDLE_SCHEMA, runId: args.runId, @@ -4169,7 +4360,8 @@ export function buildCuaFanoutBundle(args: { ? {} : { desktopBrowser: { requested: configuredBrowser, ...(unanimousResolvedBrowser === undefined ? {} : { resolved: unanimousResolvedBrowser }) } }), ...(providerResources.length === 0 ? {} : { providerResources }), - subject: args.aggregateSubject + subject: args.aggregateSubject, + ...(cost === undefined ? {} : { cost }) }; } diff --git a/src/lab-config.ts b/src/lab-config.ts index a9b0d83..b92f5ce 100644 --- a/src/lab-config.ts +++ b/src/lab-config.ts @@ -445,6 +445,15 @@ export interface LabExecution { /** FORWARD-DECLARED. */ concurrency?: number; desktop?: LabExecutionDesktop; + /** + * Blast-radius budget for the computer-use lane. CONSUMED on the CUA route: `caps.maxUsd`, when + * set, is a FAIL-CLOSED abort — the session stops the moment its running ESTIMATED spend crosses + * it (the runaway-retry guard), and a cap on a model src/pricing.ts cannot price is REFUSED at + * preflight rather than run uncapped. Absent = UNCAPPED (the historical CUA behavior); + * maxUsd: 0 = no-spend (any measurable estimate > 0 aborts). Inert (warned) on non-CUA routes. + * Reuses the same LabScenarioCaps shape as the terminal lane's `scenario.caps` (not a fork). + */ + caps?: LabScenarioCaps; /** `terminal-product` route: the terminal transport + stdin posture. Consumed on that route. */ terminal?: LabExecutionTerminal; /** `terminal-product` route: the in-sandbox agent's runtime-auth channel. Live runs inject the @@ -1234,6 +1243,9 @@ function forwardDeclaredWarnings(config: LabConfig): string[] { // execution.concurrency is CONSUMED on the cua route (it bounds in-flight fan-out lanes); // inert (warned) everywhere else. if (config.execution?.concurrency !== undefined && !routesToCua) inert.push("execution.concurrency"); + // execution.caps is CONSUMED on the cua route (maxUsd is the fail-closed spend abort); inert + // (warned) everywhere else so a misplaced budget field is never trusted to cap a route it cannot. + if (config.execution?.caps && !routesToCua) inert.push("execution.caps (the fail-closed spend abort is a computer-use route capability; needs a computer-use actor on e2b-desktop)"); // terminal-product consumes subject.product, scenario.caps, execution.{terminal,runtimeAuth}: // dry-run records the contract; live execution enforces caps and command-scoped auth. On every // OTHER route they are inert and must warn so a @@ -2041,6 +2053,13 @@ function parseExecution(raw: unknown): { ok: true; value: LabExecution | undefin return desktopResult; } if (desktopResult.value) execution.desktop = desktopResult.value; + // Reuse the terminal lane's caps parser (same shape, not a fork) — a malformed budget is a hard + // error, never silently dropped (a cap that silently does nothing would be a safety lie). + const capsResult = parseCaps(raw.caps); + if (!capsResult.ok) { + return capsResult; + } + if (capsResult.value) execution.caps = capsResult.value; const terminalResult = parseTerminal(raw.terminal); if (!terminalResult.ok) { return terminalResult; diff --git a/tests/cua-actor-lab.fanout.test.ts b/tests/cua-actor-lab.fanout.test.ts index da15dc7..4beca9e 100644 --- a/tests/cua-actor-lab.fanout.test.ts +++ b/tests/cua-actor-lab.fanout.test.ts @@ -949,3 +949,62 @@ describe("cua fan-out — engine fail-closed guards", () => { expect(result.error?.code).toBe("HUMANISH_CUA_LAB_FANOUT_INVALID"); }); }); + +describe("cua fan-out — cost estimate (sum lane token lines + one aggregate desktop line)", () => { + let cwd: string; + beforeEach(async () => { cwd = await mkdtemp(path.join(tmpdir(), "humanish-fanout-cost-")); }); + afterEach(async () => { await rm(cwd, { recursive: true, force: true }); }); + + const usageSession = (input: number, output: number): unknown[] => [ + { id: "resp_1", output: [{ type: "computer_call", call_id: "c1", actions: [{ type: "click", x: 11, y: 22 }] }], usage: { input_tokens: input, output_tokens: output } }, + { id: "resp_2", output: [{ type: "message", content: [{ type: "output_text", text: "Done." }] }], usage: { input_tokens: 0, output_tokens: 0 } } + ]; + + it("emits one model-tokens line per lane and a SINGLE aggregate desktop-minutes line, summing without double-counting", async () => { + const handle = makeFanoutModule(); + const config = fanoutConfig({ + concurrency: 2, + lanes: [ + { id: "role-a", persona: "role-a", device: "desktop", instruction: "Explore role A." }, + { id: "role-b", persona: "role-b", device: "desktop", instruction: "Explore role B." } + ] + }); + const outcome = await runLab(config, { + cwd, + cuaHooks: { + env: { OPENAI_API_KEY: "test-openai-key", E2B_API_KEY: "test-e2b-key" }, + loadDesktopModule: async () => handle.module, + runSession: async (options: CuaActorSessionOptions) => + runCuaActorSession({ ...options, openai: { ...options.openai, apiKey: "test-openai-key", fetchFn: scriptedFetch(usageSession(1000, 200)) } }) + } + }); + expect(outcome.backend).toBe("cua"); + if (outcome.backend !== "cua") return; + const result = outcome.result; + expect(result.ok).toBe(true); + + const bundle = JSON.parse(await readFile(path.join(cwd, ".humanish", "runs", result.runId, "run.json"), "utf8")); + const cost = bundle.cost; + expect(cost.schema).toBe("humanish.run-cost-summary.v1"); + + const modelLines = cost.breakdown.filter((l: { kind: string }) => l.kind === "model-tokens"); + const desktopLines = cost.breakdown.filter((l: { kind: string }) => l.kind === "desktop-minutes"); + // One model-tokens line PER lane, keyed by laneId; exactly ONE aggregate desktop line. + expect(modelLines).toHaveLength(2); + expect(new Set(modelLines.map((l: { laneId?: string }) => l.laneId))).toEqual(new Set(["role-a", "role-b"])); + expect(desktopLines).toHaveLength(1); + + // Token usage summed across BOTH lanes (2 * {input:1000, output:200}). + expect(cost.tokenUsage).toEqual({ input: 2000, output: 400, total: 2400 }); + + // The total is exactly the sum of the KNOWN lines — the invariant verify also asserts. + const knownSum = cost.breakdown + .filter((l: { estimatedCostUsd: number | null }) => l.estimatedCostUsd !== null) + .reduce((s: number, l: { estimatedCostUsd: number }) => s + l.estimatedCostUsd, 0); + expect(cost.estimatedTotalUsd).toBeCloseTo(Math.round(knownSum * 1e6) / 1e6, 9); + + const verify = await verifyRun(cwd, result.runId); + expect(verify.checks.find((c) => c.name === "cost estimate labeling")?.ok).toBe(true); + expect(verify.ok).toBe(true); + }); +}); diff --git a/tests/cua-actor-lab.test.ts b/tests/cua-actor-lab.test.ts index 13ee296..e7ebe4c 100644 --- a/tests/cua-actor-lab.test.ts +++ b/tests/cua-actor-lab.test.ts @@ -3220,3 +3220,169 @@ describe("runCuaActorLab budget/timeout semantics + live serve", () => { expect(persisted).not.toContain("stream.invalid"); }); }); + +describe("runCuaActorLab cost estimates", () => { + let cwd: string; + beforeEach(async () => { cwd = await mkdtemp(path.join(tmpdir(), "humanish-cua-cost-")); }); + afterEach(async () => { await rm(cwd, { recursive: true, force: true }); }); + + // A stepped clock: runCuaLane reads it exactly twice — right after create() and right after + // teardown — so delta == one step == the deterministic billed span. + function steppedClock(stepMs: number): () => number { + let t = 0; + return () => (t += stepMs); + } + // A scripted OpenAI Responses session that reports token usage, so a real estimate is produced. + const usageSession = (input: number, output: number): unknown[] => [ + { id: "resp_1", output: [{ type: "computer_call", call_id: "c1", actions: [{ type: "click", x: 11, y: 22 }] }], usage: { input_tokens: input, output_tokens: output } }, + { id: "resp_2", output: [{ type: "message", content: [{ type: "output_text", text: "Done." }] }], usage: { input_tokens: 0, output_tokens: 0 } } + ]; + function configWithModel(model?: string, caps?: { maxUsd?: number }): LabConfig { + const parsed = parseLabConfig({ + schema: LAB_CONFIG_SCHEMA, + id: "cua-cost-proof", + title: "CUA cost proof", + subject: { source: "app-url", appUrl: "http://127.0.0.1:3000/" }, + actors: [{ type: "openai-computer-use", persona: "first-time-visitor", mission: "Explore the app and stop.", ...(model ? { model } : {}) }], + execution: { target: "e2b-desktop", timeoutMs: 60_000, desktop: { resolution: [1280, 800] }, ...(caps ? { caps } : {}) }, + scenario: { mode: "live" } + }); + if (!parsed.ok) throw new Error(parsed.error.message); + return parsed.config; + } + const readBundle = async (runId: string): Promise => + JSON.parse(await readFile(path.join(cwd, ".humanish", "runs", runId, "run.json"), "utf8")); + + it("attaches a labeled per-lane + run-level estimated cost with provenance and deterministic desktop-minutes; verify passes", async () => { + const { module, killed } = makeFakeModule(makeFakeSandbox()); + const result = await runCuaActorLab({ + cwd, + config: configWithModel(), // default resolves to gpt-5.5 (a documented PLACEHOLDER rate) + dryRun: false, + hooks: { + env: { OPENAI_API_KEY: "k", E2B_API_KEY: "k" }, + loadDesktopModule: async () => module, + now: steppedClock(60_000), // create → 60000ms, teardown → 120000ms → 1 billed minute + runSession: async (o) => runCuaActorSession({ ...o, openai: { ...o.openai, apiKey: "k", fetchFn: scriptedFetch(usageSession(2_000_000, 4_000)) } }) + } + }); + expect(result.ok).toBe(true); + expect(killed).toEqual(["fake-sandbox-001"]); + + const bundle = await readBundle(result.runId); + // Per-actor estimate on the persisted trace: labeled with provenance, placeholder flagged. + const est = bundle.streams[0].actor.estimatedCost; + expect(est.schema).toBe("humanish.actor-estimated-cost.v1"); + // gpt-5.5 placeholder: 2_000_000*1.25e-6 + 4_000*10e-6 = 2.5 + 0.04 = 2.54. + expect(est.estimatedCostUsd).toBeCloseTo(2.54, 6); + expect(est.ratesAsOf).toBe("2026-08-01"); + expect(est.source).toContain("openai.com/api/pricing"); + expect(est.placeholder).toBe(true); + expect(est.modelId).toBe("gpt-5.5"); + + const cost = bundle.cost; + expect(cost.schema).toBe("humanish.run-cost-summary.v1"); + expect(cost.currency).toBe("usd"); + expect(cost.desktopMinutes).toBe(1); + expect(cost.tokenUsage).toEqual({ input: 2_000_000, output: 4_000, total: 2_004_000 }); + const modelLine = cost.breakdown.find((l: any) => l.kind === "model-tokens"); + const desktopLine = cost.breakdown.find((l: any) => l.kind === "desktop-minutes"); + expect(modelLine.estimatedCostUsd).toBeCloseTo(2.54, 6); + expect(modelLine.ratesAsOf).toBe("2026-08-01"); + expect(modelLine.source).toContain("openai.com/api/pricing"); + expect(desktopLine.estimatedCostUsd).toBeCloseTo(0.00167, 6); + expect(cost.estimatedTotalUsd).toBeCloseTo(2.54167, 6); + expect(cost.placeholder).toBe(true); + expect(cost.fullyEstimated).toBe(true); + expect(cost.ratesAsOf).toBe("2026-08-01"); + + const verify = await verifyRun(cwd, result.runId); + expect(verify.checks.find((c) => c.name === "cost estimate labeling")?.ok).toBe(true); + expect(verify.ok).toBe(true); + }); + + it("DECLARES ABSENT (null + reason) for an unpriced model and sums ONLY the known lines into the total", async () => { + const { module } = makeFakeModule(makeFakeSandbox()); + const result = await runCuaActorLab({ + cwd, + config: configWithModel("gpt-4o-unpriced-xyz"), + dryRun: false, + hooks: { + env: { OPENAI_API_KEY: "k", E2B_API_KEY: "k" }, + loadDesktopModule: async () => module, + now: steppedClock(60_000), + runSession: async (o) => runCuaActorSession({ ...o, openai: { ...o.openai, apiKey: "k", fetchFn: scriptedFetch(usageSession(1000, 200)) } }) + } + }); + + const bundle = await readBundle(result.runId); + const est = bundle.streams[0].actor.estimatedCost; + expect(est.estimatedCostUsd).toBeNull(); + expect(est.reason).toBe("no_rate_for_model"); + expect(est.ratesAsOf).toBeNull(); + + const cost = bundle.cost; + const modelLine = cost.breakdown.find((l: any) => l.kind === "model-tokens"); + const desktopLine = cost.breakdown.find((l: any) => l.kind === "desktop-minutes"); + expect(modelLine.estimatedCostUsd).toBeNull(); + expect(modelLine.reason).toBe("no_rate_for_model"); + // The total is the desktop line ALONE — the null model line is never coerced to 0. + expect(cost.estimatedTotalUsd).toBeCloseTo(desktopLine.estimatedCostUsd, 6); + expect(cost.fullyEstimated).toBe(false); + // token usage is still summed even though the model could not be priced. + expect(cost.tokenUsage).toEqual({ input: 1000, output: 200, total: 1200 }); + + const verify = await verifyRun(cwd, result.runId); + expect(verify.checks.find((c) => c.name === "cost estimate labeling")?.ok).toBe(true); + }); + + it("DRY-RUN invents no spend: the bundle carries no cost block", async () => { + const result = await runCuaActorLab({ cwd, config: configWithModel(), dryRun: true, runId: "cost-dry-run" }); + expect(result.ok).toBe(true); + const bundle = await readBundle("cost-dry-run"); + expect(bundle.cost).toBeUndefined(); + }); + + it("refuses a maxUsd cap on a model src/pricing.ts cannot price, BEFORE creating any sandbox", async () => { + const { module, created } = makeFakeModule(makeFakeSandbox()); + const result = await runCuaActorLab({ + cwd, + config: configWithModel("gpt-4o-unpriced-xyz", { maxUsd: 5 }), + dryRun: false, + hooks: { + env: { OPENAI_API_KEY: "k", E2B_API_KEY: "k" }, + loadDesktopModule: async () => module, + runSession: async () => { throw new Error("a session must never run under an unenforceable cap"); } + } + }); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("HUMANISH_CUA_LAB_UNPRICED_CAP"); + expect(created).toHaveLength(0); + }); + + it("accepts a maxUsd cap on a PRICED model and runs — a priced cap is wired, never a refusal", async () => { + const { module } = makeFakeModule(makeFakeSandbox()); + const result = await runCuaActorLab({ + cwd, + config: configWithModel("computer-use-preview", { maxUsd: 50 }), + dryRun: false, + hooks: { + env: { OPENAI_API_KEY: "k", E2B_API_KEY: "k" }, + loadDesktopModule: async () => module, + now: steppedClock(60_000), + runSession: async (o) => runCuaActorSession({ ...o, openai: { ...o.openai, apiKey: "k", fetchFn: scriptedFetch(usageSession(1000, 200)) } }) + } + }); + expect(result.error?.code).not.toBe("HUMANISH_CUA_LAB_UNPRICED_CAP"); + expect(result.ok).toBe(true); + const bundle = await readBundle(result.runId); + // computer-use-preview is a CONFIRMED (non-placeholder) MODEL rate — the per-actor estimate + // and its model-tokens line carry no placeholder flag. + expect(bundle.streams[0].actor.estimatedCost.placeholder).toBeUndefined(); + const modelLine = bundle.cost.breakdown.find((l: any) => l.kind === "model-tokens"); + expect(modelLine.placeholder).toBeUndefined(); + // The summary is STILL flagged placeholder because the E2B DESKTOP rate is a placeholder — an + // honest signal that a stand-in rate contributed to the total. + expect(bundle.cost.placeholder).toBe(true); + }); +}); From 7f0a9dc93495007cbeb9ee104a6e433682942a13 Mon Sep 17 00:00:00 2001 From: Daniel G Wilson Date: Sun, 2 Aug 2026 10:32:43 +0000 Subject: [PATCH 5/7] feat(observer): labeled cost estimates in the Observer + serve library Project the run-level cost summary through buildObserverData to ObserverData.cost, and render it in the Observer: a run-total stat in the status bar + run-details popover, and a per-lane figure in the focus side panel. Every dollar reads "~$X estimated (rates as of )" (never a bare "$X"), tags a placeholder rate, and reads "not estimated ()" when declared absent. The serve library index + history feed carry a labeled per-run "~$X est." token with a rates-as-of tooltip; buildHistoryIndex projects estimatedTotalUsd/ratesAsOf/placeholder and the LibraryHistory row widens to match. Pages stay self-contained (no new assets/fetch). Co-Authored-By: Claude Opus 4.8 --- src/observer-assets.ts | 40 ++++++++++++-- src/observer-data.ts | 9 +++- src/observer-library.ts | 13 ++++- src/observer.ts | 8 ++- tests/observer-serve.test.ts | 69 ++++++++++++++++++++++++ tests/observer.test.ts | 102 ++++++++++++++++++++++++++++++++++- 6 files changed, 232 insertions(+), 9 deletions(-) diff --git a/src/observer-assets.ts b/src/observer-assets.ts index 0c7c4e3..e8698f5 100644 --- a/src/observer-assets.ts +++ b/src/observer-assets.ts @@ -1223,6 +1223,30 @@ export function observerClientJs(): string { function laneEvents(s) { return s.timeline || []; } function laneArtifacts(s) { return s.artifacts || []; } + // ---------------------------------------------------------------- cost (ALWAYS labeled estimated) + function fmtCostUsd(v) { + if (typeof v !== "number" || !isFinite(v)) return null; + return v >= 0.01 ? v.toFixed(2) : String(v); + } + // A single honest label for any cost figure. A number ALWAYS reads "~$X estimated (rates as of + // )" (never a bare "$X"); a placeholder rate is tagged; a null is "not estimated ()". + function costText(usd, ratesAsOf, placeholder, reason) { + if (usd == null) return "not estimated (" + (reason || "unmeasured") + ")"; + var s = "~$" + (fmtCostUsd(usd) || "0") + " estimated (rates as of " + (ratesAsOf || "unknown") + ")"; + if (placeholder) s += " · placeholder rate"; + return s; + } + function runCostText() { + var c = currentData.cost; + if (!c) return ""; + return costText(c.estimatedTotalUsd, c.ratesAsOf, c.placeholder, "no priced lines"); + } + function laneCostText(s) { + var est = s && s.actor && s.actor.estimatedCost; + if (!est) return ""; + return costText(est.estimatedCostUsd, est.ratesAsOf, est.placeholder, est.reason); + } + function geometrySize(value) { var width = value && Number(value.width); var height = value && Number(value.height); @@ -2192,7 +2216,9 @@ export function observerClientJs(): string { + '
Persona
' + esc((run.persona && run.persona.name) || "Synthetic persona") + ' is attempting this lane as a realistic setup operator.
' + '
Scenario
' + esc((run.scenario && run.scenario.goal) || laneSummary(s)) + '
' + '
' + pip(s.status, live) + '' + (live ? "Now" : "Last step") + '
' - + '
' + esc(laneStep(s)) + '
' + buildMeaningfulUseCard(s) + '' + + '
' + esc(laneStep(s)) + '
' + + (laneCostText(s) ? '
Est. cost
' + esc(laneCostText(s)) + '
' : "") + + buildMeaningfulUseCard(s) + '' + '
' + tabs.map(function (t) { return ''; }).join("") + '
' @@ -2231,6 +2257,7 @@ export function observerClientJs(): string { + '' + '' + counts.map(function (c) { return '' + c.count + '' + c.label + ''; }).join("") + '' + '' + esc("mode: " + (run.mode || "unknown") + " · " + (run.runId || "")) + '' + + (currentData.cost ? '' + esc("cost: " + runCostText()) + '' : "") + '
' + ''; + + '' + esc(meta) + ''; }).join(""); return '