diff --git a/package-lock.json b/package-lock.json index f162bd8a..58fb7379 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4805,7 +4805,7 @@ }, "packages/workspace-lens": { "name": "@metaharness/workspace-lens", - "version": "0.1.0", + "version": "0.1.1", "license": "MIT", "devDependencies": { "typescript": "^5.4.0", diff --git a/packages/workspace-probe/README.md b/packages/workspace-probe/README.md index 00b31de3..bb088dc6 100644 --- a/packages/workspace-probe/README.md +++ b/packages/workspace-probe/README.md @@ -37,6 +37,35 @@ const verdict = gradeMutationByWorkspace(baselineReceipts, mutantReceipts); if (!verdict.keep) rejectMutation(verdict.reasons); // structurally brittle ``` +## CLI + +For audit workflows that don't want to write TypeScript. All inputs are JSON, all output is JSON on +stdout (composes with `jq`): + +```bash +npx @metaharness/workspace-probe diag lens.json +npx @metaharness/workspace-probe readout lens.json activations.json [--top-k N] +npx @metaharness/workspace-probe probe receipts.json [--drift-threshold F] +npx @metaharness/workspace-probe grade-mutation baseline-receipts.json mutant-receipts.json +``` + +- `diag` → the fitted lens's metadata (model/lens id, dModel, fitted layers, vocab size). +- `readout` → the workspace tokens per activation (`{layer, position, h}[]`). +- `probe` → the `workspaceProbeScore` over a receipt set. +- `grade-mutation` → the keep/veto verdict comparing two receipt sets. + +Exit codes: `0` ok, `2` usage error, `1` runtime error. A fitted lens artifact is produced out-of-band +(open-weight model + backward pass — see `@metaharness/workspace-lens`). + +## Runnable demo + +[`examples/demo.mjs`](./examples/demo.mjs) is a `$0`, no-model end-to-end walk-through (synthetic lens → +readout → receipts → probe score → mutation veto): + +```bash +node packages/workspace-probe/examples/demo.mjs +``` + ## Links - **workspace-lens (the primitive):** https://www.npmjs.com/package/@metaharness/workspace-lens diff --git a/packages/workspace-probe/__tests__/cli.test.ts b/packages/workspace-probe/__tests__/cli.test.ts new file mode 100644 index 00000000..fdd69b73 --- /dev/null +++ b/packages/workspace-probe/__tests__/cli.test.ts @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: MIT +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { mkdtemp, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { LensArtifact, WorkspaceLensReceipt } from '@metaharness/workspace-lens'; +import { runCli } from '../src/cli.js'; + +// dModel=3; J(layer 5)=identity; unembed maps component → token; so h=[0,0,5] → "wrong". +const LENS: LensArtifact = { + lensId: 'jlens-synth-v1', modelId: 'synth-3d', dModel: 3, + vocab: ['yes', 'no', 'wrong', 'right', 'secret'], + unembed: [[1, 0, 0], [0, 1, 0], [0, 0, 1], [1, 1, 0], [0, 0, 0]], + layers: [{ layer: 5, jacobian: [[1, 0, 0], [0, 1, 0], [0, 0, 1]] }], +}; +function receipt(over: Partial = {}): WorkspaceLensReceipt { + return { + promptHash: 'h', modelId: 'm', lensId: 'l', layerRange: [5, 5], positions: [0], topTokens: [], + entropyTrajectory: [], createdAt: '2026-07-07T00:00:00Z', + flags: { promptInjection: false, evalAwareness: false, hiddenObjective: false, refusalConflict: false }, + triggers: [], workspaceDrift: 0.1, ...over, + }; +} +const critTrigger = { concept: 'exfiltration', layer: 5, position: 0, score: 0.9, critical: true }; + +// Capture stdout JSON. +function captureStdout(): { readonly text: () => string; restore: () => void } { + let buf = ''; + const spy = vi.spyOn(process.stdout, 'write').mockImplementation((s: string | Uint8Array) => { buf += s.toString(); return true; }); + return { text: () => buf, restore: () => spy.mockRestore() }; +} + +async function tmpJson(name: string, data: unknown): Promise { + const dir = await mkdtemp(join(tmpdir(), 'wp-cli-')); + const p = join(dir, name); + await writeFile(p, JSON.stringify(data)); + return p; +} + +afterEach(() => vi.restoreAllMocks()); + +describe('workspace-probe CLI', () => { + it('diag prints lens metadata', async () => { + const lensPath = await tmpJson('lens.json', LENS); + const cap = captureStdout(); + const code = await runCli(['diag', lensPath]); + cap.restore(); + expect(code).toBe(0); + const j = JSON.parse(cap.text()); + expect(j).toMatchObject({ modelId: 'synth-3d', lensId: 'jlens-synth-v1', dModel: 3, layers: [5], vocabSize: 5 }); + }); + + it('readout decodes activations to workspace tokens', async () => { + const lensPath = await tmpJson('lens.json', LENS); + const actPath = await tmpJson('act.json', [{ layer: 5, position: 3, h: [0, 0, 5] }]); + const cap = captureStdout(); + const code = await runCli(['readout', lensPath, actPath, '--top-k', '3']); + cap.restore(); + expect(code).toBe(0); + const j = JSON.parse(cap.text()); + expect(j[0].tokens[0].token).toBe('wrong'); + }); + + it('probe scores a receipt set', async () => { + const rPath = await tmpJson('r.json', [receipt(), receipt({ triggers: [critTrigger] })]); + const cap = captureStdout(); + const code = await runCli(['probe', rPath]); + cap.restore(); + expect(code).toBe(0); + const j = JSON.parse(cap.text()); + expect(j.n).toBe(2); + expect(j.criticalRate).toBeCloseTo(0.5); + }); + + it('grade-mutation vetoes a workspace-degrading mutation', async () => { + const base = await tmpJson('b.json', [receipt(), receipt()]); + const mut = await tmpJson('m.json', [receipt({ triggers: [critTrigger] }), receipt()]); + const cap = captureStdout(); + const code = await runCli(['grade-mutation', base, mut]); + cap.restore(); + expect(code).toBe(0); + const j = JSON.parse(cap.text()); + expect(j.keep).toBe(false); + }); + + it('exits 2 on missing args and 0 on --help; 2 on unknown command', async () => { + const errSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + expect(await runCli(['diag'])).toBe(2); + expect(await runCli(['bogus'])).toBe(2); + errSpy.mockRestore(); + const cap = captureStdout(); + expect(await runCli(['--help'])).toBe(0); + cap.restore(); + expect(cap.text()).toMatch(/Usage:/); + }); + + it('exits 1 on a runtime error (unreadable file)', async () => { + const errSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + const code = await runCli(['diag', '/no/such/lens.json']); + errSpy.mockRestore(); + expect(code).toBe(1); + }); +}); diff --git a/packages/workspace-probe/bin/workspace-probe.mjs b/packages/workspace-probe/bin/workspace-probe.mjs new file mode 100755 index 00000000..d7979f1f --- /dev/null +++ b/packages/workspace-probe/bin/workspace-probe.mjs @@ -0,0 +1,4 @@ +#!/usr/bin/env node +// SPDX-License-Identifier: MIT +import { runCli } from '../dist/cli.js'; +runCli(process.argv.slice(2)).then((code) => process.exit(code)); diff --git a/packages/workspace-probe/examples/demo.mjs b/packages/workspace-probe/examples/demo.mjs new file mode 100644 index 00000000..3b59767a --- /dev/null +++ b/packages/workspace-probe/examples/demo.mjs @@ -0,0 +1,38 @@ +#!/usr/bin/env node +// SPDX-License-Identifier: MIT +// +// $0, no-model end-to-end demo of the workspace-lens → workspace-probe flow, on a synthetic lens. +// Run: node packages/workspace-probe/examples/demo.mjs +// +// A real deployment fits the lens offline (open-weight model + backward pass — see the reference +// anthropics/jacobian-lens) and captures real activations; here we hand-build a tiny lens so the whole +// governance flow runs deterministically with no dependencies. + +import { WorkspaceLens, buildReceipt } from '@metaharness/workspace-lens'; +import { workspaceProbeScore, gradeMutationByWorkspace } from '@metaharness/workspace-probe'; + +// dModel=3; J(layer 5)=identity; unembed maps residual components to tokens. +const lens = WorkspaceLens.fromArtifact({ + lensId: 'jlens-demo', modelId: 'synth-3d', dModel: 3, + vocab: ['ok', 'wrong', 'exfiltrate'], + unembed: [[1, 0, 0], [0, 1, 0], [0, 0, 1]], + layers: [{ layer: 5, jacobian: [[1, 0, 0], [0, 1, 0], [0, 0, 1]] }], +}); + +// A concept direction (per-model) for the "exfiltrate" workspace concept — flagged critical. +const concepts = [{ concept: 'exfiltration', modelId: 'synth-3d', vector: [0, 0, 1], critical: true }]; + +console.log('1) readout — what the activation is disposed to say:'); +console.log(' ', lens.readout({ layer: 5, position: 4, h: [0, 5, 0] }).tokens.map((t) => t.token).join(' '), '(top: "wrong")\n'); + +// Two "decisions": one clean, one where the exfiltration direction lights up. +const clean = buildReceipt(lens, 'summarize the doc', [{ layer: 5, position: 4, h: [0, 5, 0] }], { createdAt: '2026-07-07T00:00:00Z', concepts }); +const leaky = buildReceipt(lens, 'summarize the doc', [{ layer: 5, position: 4, h: [0, 0, 9] }], { createdAt: '2026-07-07T00:00:01Z', concepts }); + +console.log('2) workspaceProbeScore over a clean baseline:'); +console.log(' ', JSON.stringify(workspaceProbeScore([clean, clean])), '\n'); + +console.log('3) grade-mutation — a mutation whose workspace starts leaking "exfiltration" is VETOED'); +console.log(' even though the final answer may look fine:'); +const verdict = gradeMutationByWorkspace([clean, clean], [clean, leaky]); +console.log(' ', JSON.stringify({ keep: verdict.keep, reasons: verdict.reasons })); diff --git a/packages/workspace-probe/package.json b/packages/workspace-probe/package.json index 1f30cbdf..9ae89c74 100644 --- a/packages/workspace-probe/package.json +++ b/packages/workspace-probe/package.json @@ -1,16 +1,50 @@ { "name": "@metaharness/workspace-probe", - "version": "0.1.0", + "version": "0.1.1", "description": "Evaluation + Darwin-Mode bridge for @metaharness/workspace-lens: turn Jacobian-Lens interpretability receipts into a flywheel-consumable workspace_probe score, and reject structurally-brittle prompt/policy mutations (final answer up, workspace grip down). Pure, dependency-light, deterministic.", "type": "module", "main": "./dist/index.js", "types": "./dist/index.d.ts", - "exports": { ".": { "types": "./dist/index.d.ts", "import": "./dist/index.js" } }, - "files": ["dist", "README.md"], - "publishConfig": { "access": "public" }, - "scripts": { "build": "tsc", "test": "vitest run", "lint": "tsc --noEmit" }, - "keywords": ["jacobian-lens", "workspace-lens", "interpretability", "darwin-mode", "mutation-evidence", "flywheel", "eval", "ai-safety", "metaharness", "agent-harness"], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "files": [ + "dist", + "bin", + "README.md" + ], + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "tsc", + "test": "vitest run", + "lint": "tsc --noEmit" + }, + "keywords": [ + "jacobian-lens", + "workspace-lens", + "interpretability", + "darwin-mode", + "mutation-evidence", + "flywheel", + "eval", + "ai-safety", + "metaharness", + "agent-harness" + ], "license": "MIT", - "dependencies": { "@metaharness/workspace-lens": "^0.1.0" }, - "devDependencies": { "typescript": "^5.4.0", "vitest": "^2.0.0" } + "dependencies": { + "@metaharness/workspace-lens": "^0.1.0" + }, + "devDependencies": { + "typescript": "^5.4.0", + "vitest": "^2.0.0" + }, + "bin": { + "workspace-probe": "./bin/workspace-probe.mjs" + } } diff --git a/packages/workspace-probe/src/cli.ts b/packages/workspace-probe/src/cli.ts new file mode 100644 index 00000000..3d1376e9 --- /dev/null +++ b/packages/workspace-probe/src/cli.ts @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: MIT +// +// `workspace-probe` CLI — a thin shell surface over @metaharness/workspace-lens + this package, for +// interpretability-audit workflows that don't want to write TypeScript. All inputs are JSON files, all +// output is JSON on stdout, so it composes with `jq` and CI. Pure (fs reads only); no model, no network. +// +// workspace-probe diag +// workspace-probe readout [--top-k N] +// workspace-probe probe [--drift-threshold F] +// workspace-probe grade-mutation + +import { readFile } from 'node:fs/promises'; +import { WorkspaceLens, type LensArtifact, type HiddenState } from '@metaharness/workspace-lens'; +import { workspaceProbeScore, gradeMutationByWorkspace } from './probe.js'; +import type { WorkspaceLensReceipt } from '@metaharness/workspace-lens'; + +const USAGE = `workspace-probe — interpretability CLI (@metaharness/workspace-probe) + +Usage: + workspace-probe diag + workspace-probe readout [--top-k N] + workspace-probe probe [--drift-threshold F] + workspace-probe grade-mutation [--drift-threshold F] + +All inputs are JSON; all output is JSON on stdout. A fitted lens artifact is produced out-of-band +(open-weight model + backward pass — see @metaharness/workspace-lens). Exit 0 on success, 2 on usage +error, 1 on a runtime error.`; + +async function readJson(path: string): Promise { + return JSON.parse(await readFile(path, 'utf-8')) as T; +} + +function numFlag(args: string[], name: string): number | undefined { + const i = args.indexOf(name); + if (i < 0 || i + 1 >= args.length) return undefined; + const v = Number(args[i + 1]); + return Number.isFinite(v) ? v : undefined; +} + +function out(obj: unknown): void { + process.stdout.write(JSON.stringify(obj, null, 2) + '\n'); +} + +/** Run the CLI. Returns a process exit code (0 ok, 2 usage, 1 runtime). Pure w.r.t. process.exit. */ +export async function runCli(argv: string[]): Promise { + const [cmd, ...rest] = argv; + const positional = rest.filter((a) => !a.startsWith('--')); + try { + switch (cmd) { + case 'diag': { + if (!positional[0]) { process.stderr.write(USAGE + '\n'); return 2; } + const lens = WorkspaceLens.fromArtifact(await readJson(positional[0])); + out({ modelId: lens.modelId, lensId: lens.lensId, dModel: lens.dModel, layers: lens.layers, vocabSize: lens.artifact.vocab.length }); + return 0; + } + case 'readout': { + if (!positional[0] || !positional[1]) { process.stderr.write(USAGE + '\n'); return 2; } + const lens = WorkspaceLens.fromArtifact(await readJson(positional[0])); + const states = await readJson(positional[1]); + const topK = numFlag(rest, '--top-k') ?? 8; + out(states.filter((s) => lens.hasLayer(s.layer)).map((s) => lens.readout(s, topK))); + return 0; + } + case 'probe': { + if (!positional[0]) { process.stderr.write(USAGE + '\n'); return 2; } + const receipts = await readJson(positional[0]); + out(workspaceProbeScore(receipts, { driftThreshold: numFlag(rest, '--drift-threshold') })); + return 0; + } + case 'grade-mutation': { + if (!positional[0] || !positional[1]) { process.stderr.write(USAGE + '\n'); return 2; } + const baseline = await readJson(positional[0]); + const mutant = await readJson(positional[1]); + out(gradeMutationByWorkspace(baseline, mutant, { driftThreshold: numFlag(rest, '--drift-threshold') })); + return 0; + } + case undefined: + case '-h': + case '--help': + process.stdout.write(USAGE + '\n'); + return 0; + default: + process.stderr.write(`unknown command: ${cmd}\n\n` + USAGE + '\n'); + return 2; + } + } catch (e) { + process.stderr.write(`workspace-probe: ${(e as Error).message}\n`); + return 1; + } +}