From 6bb2152cd36711ad7d1d1d0530073638019858da Mon Sep 17 00:00:00 2001 From: sepehr-safari Date: Thu, 9 Jul 2026 03:22:35 +0300 Subject: [PATCH] =?UTF-8?q?feat(core):=20rich=20scenario=20assertions=20?= =?UTF-8?q?=E2=80=94=208=20assertion=20types=20+=20evaluateScenario()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #86 --- .changeset/v03-assertions.md | 10 + CURRENT_STATE.md | 1 + packages/toolkit/src/core/assertions.test.ts | 636 +++++++++++++++++++ packages/toolkit/src/core/assertions.ts | 438 +++++++++++++ packages/toolkit/src/core/index.ts | 4 + packages/toolkit/src/core/types.ts | 47 ++ packages/toolkit/src/index.ts | 4 + 7 files changed, 1140 insertions(+) create mode 100644 .changeset/v03-assertions.md create mode 100644 packages/toolkit/src/core/assertions.test.ts create mode 100644 packages/toolkit/src/core/assertions.ts diff --git a/.changeset/v03-assertions.md b/.changeset/v03-assertions.md new file mode 100644 index 0000000..754adcf --- /dev/null +++ b/.changeset/v03-assertions.md @@ -0,0 +1,10 @@ +--- +'@ocpp-debugkit/toolkit': minor +--- + +Add rich scenario assertions — 8 declarative assertion types for scenario evaluation: +- `event_order`, `event_count`, `payload_field`, `timing`, `session_state`, + `failure_severity`, `no_failures`, `failure_count` + +New API: `runAssertions()`, `evaluateScenario()`. Backward compatible — +existing scenarios with only `expectedFailures` work unchanged. diff --git a/CURRENT_STATE.md b/CURRENT_STATE.md index 6a7f841..84258f5 100644 --- a/CURRENT_STATE.md +++ b/CURRENT_STATE.md @@ -201,6 +201,7 @@ integration examples, and contributor onboarding. `SUSPICIOUS_SESSION_DURATION`, `SLOW_RESPONSE`, `HEARTBEAT_INTERVAL_VIOLATION`, `METER_VALUE_ANOMALY`, `UNRESPONSIVE_CSMS` (15 total rules) - ✅ Trace diffing — `diffTraces()` API (Issue #85) +- ✅ Rich scenario assertions — 8 assertion types, `evaluateScenario()` (Issue #86) ## What's Next diff --git a/packages/toolkit/src/core/assertions.test.ts b/packages/toolkit/src/core/assertions.test.ts new file mode 100644 index 0000000..62c362d --- /dev/null +++ b/packages/toolkit/src/core/assertions.test.ts @@ -0,0 +1,636 @@ +import { describe, it, expect } from 'vitest'; +import { runAssertions, evaluateScenario } from './assertions.js'; +import { buildSessionTimeline } from './timeline.js'; +import { detectFailures } from './detection.js'; +import type { Event, RawOcppMessage, Scenario, ScenarioAssertion } from './types.js'; + +// Helpers +function makeEvent( + id: string, + messageId: string, + messageType: 'Call' | 'CallResult' | 'CallError', + action: string | null, + payload: unknown = {}, + timestamp: number | null = null, + direction: 'CS_TO_CSMS' | 'CSMS_TO_CS' | 'UNKNOWN' = 'CS_TO_CSMS', +): Event { + let rawMessage: RawOcppMessage; + if (messageType === 'Call') { + rawMessage = [2, messageId, action as string, payload]; + } else if (messageType === 'CallResult') { + rawMessage = [3, messageId, payload]; + } else { + rawMessage = [4, messageId, 'Error', 'desc', payload]; + } + return { + id, + messageId, + timestamp, + direction, + messageType, + action, + payload, + errorCode: messageType === 'CallError' ? 'Error' : null, + errorDescription: messageType === 'CallError' ? 'desc' : null, + rawMessage, + }; +} + +function makeContext(events: Event[]) { + const sessions = buildSessionTimeline(events); + const failures = detectFailures(events, sessions); + return { events, sessions, failures }; +} + +// A normal session for testing (all Calls have matching CallResults, includes Heartbeat) +function normalSessionEvents(): Event[] { + return [ + makeEvent('e1', 'm1', 'Call', 'BootNotification', { chargePointSerialNumber: 'CS-001' }, 1000), + makeEvent( + 'e2', + 'm1', + 'CallResult', + null, + { status: 'Accepted', interval: 60 }, + 1500, + 'CSMS_TO_CS', + ), + makeEvent('e3', 'm2', 'Call', 'Authorize', { idTag: 'TAG-001' }, 2000), + makeEvent( + 'e4', + 'm2', + 'CallResult', + null, + { idTagInfo: { status: 'Accepted' } }, + 2500, + 'CSMS_TO_CS', + ), + makeEvent('e5', 'm3', 'Call', 'StartTransaction', { connectorId: 1, idTag: 'TAG-001' }, 3000), + makeEvent( + 'e6', + 'm3', + 'CallResult', + null, + { idTagInfo: { status: 'Accepted' }, transactionId: 42 }, + 3500, + 'CSMS_TO_CS', + ), + makeEvent( + 'e7', + 'm4', + 'Call', + 'MeterValues', + { + connectorId: 1, + transactionId: 42, + meterValue: [{ sampledValue: [{ value: '100' }] }], + }, + 4000, + ), + makeEvent('e8', 'm4', 'CallResult', null, {}, 4100, 'CSMS_TO_CS'), + makeEvent('e9', 'm5', 'Call', 'Heartbeat', {}, 5000), + makeEvent( + 'e10', + 'm5', + 'CallResult', + null, + { currentTime: '2024-01-15T10:00:05Z' }, + 5100, + 'CSMS_TO_CS', + ), + makeEvent( + 'e11', + 'm6', + 'Call', + 'StopTransaction', + { transactionId: 42, reason: 'Local' }, + 3000 + 30 * 60 * 1000, + ), + makeEvent( + 'e12', + 'm6', + 'CallResult', + null, + { idTagInfo: { status: 'Accepted' } }, + 3000 + 30 * 60 * 1000 + 500, + 'CSMS_TO_CS', + ), + ]; +} + +describe('assertions', () => { + describe('event_order', () => { + it('passes when actions appear in order', () => { + const ctx = makeContext(normalSessionEvents()); + const assertion: ScenarioAssertion = { + type: 'event_order', + params: { actions: ['BootNotification', 'Authorize', 'StartTransaction'] }, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(true); + }); + + it('fails when actions are out of order', () => { + const ctx = makeContext(normalSessionEvents()); + const assertion: ScenarioAssertion = { + type: 'event_order', + params: { actions: ['StartTransaction', 'BootNotification'] }, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(false); + }); + + it('fails when action is missing', () => { + const ctx = makeContext(normalSessionEvents()); + const assertion: ScenarioAssertion = { + type: 'event_order', + params: { actions: ['BootNotification', 'DataTransfer'] }, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(false); + }); + }); + + describe('event_count', () => { + it('passes when count is within range', () => { + const ctx = makeContext(normalSessionEvents()); + const assertion: ScenarioAssertion = { + type: 'event_count', + params: { min: 5, max: 20 }, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(true); + }); + + it('passes when counting specific action', () => { + const ctx = makeContext(normalSessionEvents()); + const assertion: ScenarioAssertion = { + type: 'event_count', + params: { min: 1, max: 1, action: 'StartTransaction' }, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(true); + }); + + it('fails when count is below min', () => { + const ctx = makeContext(normalSessionEvents()); + const assertion: ScenarioAssertion = { + type: 'event_count', + params: { min: 100 }, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(false); + }); + + it('fails when count is above max', () => { + const ctx = makeContext(normalSessionEvents()); + const assertion: ScenarioAssertion = { + type: 'event_count', + params: { max: 3 }, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(false); + }); + }); + + describe('payload_field', () => { + it('passes when field equals expected value', () => { + const ctx = makeContext(normalSessionEvents()); + const assertion: ScenarioAssertion = { + type: 'payload_field', + params: { action: 'Authorize', field: 'idTag', equals: 'TAG-001' }, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(true); + }); + + it('passes when checking nested field', () => { + const ctx = makeContext(normalSessionEvents()); + const assertion: ScenarioAssertion = { + type: 'payload_field', + params: { action: 'Authorize', field: 'idTagInfo.status', equals: 'Accepted' }, + }; + // Note: Authorize is a Call, idTagInfo.status is in the CallResult response + // This should fail because the Call payload only has idTag, not idTagInfo + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(false); + }); + + it('fails when field value does not match', () => { + const ctx = makeContext(normalSessionEvents()); + const assertion: ScenarioAssertion = { + type: 'payload_field', + params: { action: 'Authorize', field: 'idTag', equals: 'WRONG-TAG' }, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(false); + }); + + it('fails when action not found', () => { + const ctx = makeContext(normalSessionEvents()); + const assertion: ScenarioAssertion = { + type: 'payload_field', + params: { action: 'DataTransfer', field: 'idTag' }, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(false); + }); + }); + + describe('timing', () => { + it('passes when gap is within range', () => { + const ctx = makeContext(normalSessionEvents()); + const assertion: ScenarioAssertion = { + type: 'timing', + params: { actionA: 'BootNotification', actionB: 'StartTransaction', maxGapMs: 5000 }, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(true); + }); + + it('fails when gap exceeds maxGapMs', () => { + const ctx = makeContext(normalSessionEvents()); + const assertion: ScenarioAssertion = { + type: 'timing', + params: { actionA: 'BootNotification', actionB: 'StopTransaction', maxGapMs: 1000 }, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(false); + }); + + it('fails when gap is below minGapMs', () => { + const ctx = makeContext(normalSessionEvents()); + const assertion: ScenarioAssertion = { + type: 'timing', + params: { actionA: 'BootNotification', actionB: 'Authorize', minGapMs: 10000 }, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(false); + }); + }); + + describe('session_state', () => { + it('passes when session has expected status', () => { + const ctx = makeContext(normalSessionEvents()); + const assertion: ScenarioAssertion = { + type: 'session_state', + params: { expected: 'completed' }, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(true); + }); + + it('fails when session status does not match', () => { + const ctx = makeContext(normalSessionEvents()); + const assertion: ScenarioAssertion = { + type: 'session_state', + params: { expected: 'aborted' }, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(false); + }); + }); + + describe('failure_severity', () => { + it('passes when failure has expected severity', () => { + // Use a trace with FAILED_AUTHORIZATION (severity: warning) + const events = [ + makeEvent( + 'e1', + 'm1', + 'Call', + 'BootNotification', + { chargePointSerialNumber: 'CS-001' }, + 1000, + ), + makeEvent( + 'e2', + 'm1', + 'CallResult', + null, + { status: 'Accepted', interval: 60 }, + 1500, + 'CSMS_TO_CS', + ), + makeEvent('e3', 'm2', 'Call', 'Authorize', { idTag: 'BAD-TAG' }, 2000), + makeEvent( + 'e4', + 'm2', + 'CallResult', + null, + { idTagInfo: { status: 'Invalid' } }, + 2500, + 'CSMS_TO_CS', + ), + ]; + const ctx = makeContext(events); + const assertion: ScenarioAssertion = { + type: 'failure_severity', + params: { code: 'FAILED_AUTHORIZATION', severity: 'warning' }, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(true); + }); + + it('fails when failure not detected', () => { + const ctx = makeContext(normalSessionEvents()); + const assertion: ScenarioAssertion = { + type: 'failure_severity', + params: { code: 'FAILED_AUTHORIZATION', severity: 'warning' }, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(false); + }); + }); + + describe('no_failures', () => { + it('passes when no failures detected', () => { + const ctx = makeContext(normalSessionEvents()); + const assertion: ScenarioAssertion = { + type: 'no_failures', + params: {}, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(true); + }); + + it('fails when failures are detected', () => { + const events = [ + makeEvent( + 'e1', + 'm1', + 'Call', + 'BootNotification', + { chargePointSerialNumber: 'CS-001' }, + 1000, + ), + makeEvent( + 'e2', + 'm1', + 'CallResult', + null, + { status: 'Accepted', interval: 60 }, + 1500, + 'CSMS_TO_CS', + ), + makeEvent('e3', 'm2', 'Call', 'Authorize', { idTag: 'BAD-TAG' }, 2000), + makeEvent( + 'e4', + 'm2', + 'CallResult', + null, + { idTagInfo: { status: 'Invalid' } }, + 2500, + 'CSMS_TO_CS', + ), + ]; + const ctx = makeContext(events); + const assertion: ScenarioAssertion = { + type: 'no_failures', + params: {}, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(false); + }); + }); + + describe('failure_count', () => { + it('passes when failure count is within range', () => { + const events = [ + makeEvent( + 'e1', + 'm1', + 'Call', + 'BootNotification', + { chargePointSerialNumber: 'CS-001' }, + 1000, + ), + makeEvent( + 'e2', + 'm1', + 'CallResult', + null, + { status: 'Accepted', interval: 60 }, + 1500, + 'CSMS_TO_CS', + ), + makeEvent('e3', 'm2', 'Call', 'Authorize', { idTag: 'BAD-TAG' }, 2000), + makeEvent( + 'e4', + 'm2', + 'CallResult', + null, + { idTagInfo: { status: 'Invalid' } }, + 2500, + 'CSMS_TO_CS', + ), + ]; + const ctx = makeContext(events); + const assertion: ScenarioAssertion = { + type: 'failure_count', + params: { min: 1, max: 3 }, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(true); + }); + + it('passes when counting specific failure code', () => { + const events = [ + makeEvent( + 'e1', + 'm1', + 'Call', + 'BootNotification', + { chargePointSerialNumber: 'CS-001' }, + 1000, + ), + makeEvent( + 'e2', + 'm1', + 'CallResult', + null, + { status: 'Accepted', interval: 60 }, + 1500, + 'CSMS_TO_CS', + ), + makeEvent('e3', 'm2', 'Call', 'Authorize', { idTag: 'BAD-TAG' }, 2000), + makeEvent( + 'e4', + 'm2', + 'CallResult', + null, + { idTagInfo: { status: 'Invalid' } }, + 2500, + 'CSMS_TO_CS', + ), + ]; + const ctx = makeContext(events); + const assertion: ScenarioAssertion = { + type: 'failure_count', + params: { code: 'FAILED_AUTHORIZATION', min: 1 }, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(true); + }); + + it('fails when count is below min', () => { + const ctx = makeContext(normalSessionEvents()); + const assertion: ScenarioAssertion = { + type: 'failure_count', + params: { min: 5 }, + }; + const [result] = runAssertions(ctx, [assertion]); + expect(result?.passed).toBe(false); + }); + }); + + describe('runAssertions (batch)', () => { + it('evaluates multiple assertions', () => { + const ctx = makeContext(normalSessionEvents()); + const assertions: ScenarioAssertion[] = [ + { type: 'event_order', params: { actions: ['BootNotification', 'Authorize'] } }, + { type: 'event_count', params: { min: 5 } }, + { type: 'no_failures', params: {} }, + ]; + const results = runAssertions(ctx, assertions); + expect(results).toHaveLength(3); + expect(results.every((r) => r.passed)).toBe(true); + }); + }); + + describe('evaluateScenario', () => { + it('evaluates a scenario with assertions', () => { + const scenario: Scenario = { + name: 'test-scenario', + description: 'Test scenario with assertions', + trace: { + events: [ + { + timestamp: '2024-01-15T10:00:00.000Z', + message: [2, 'm1', 'BootNotification', { chargePointSerialNumber: 'CS-001' }], + }, + { + timestamp: '2024-01-15T10:00:00.500Z', + message: [3, 'm1', { status: 'Accepted', interval: 60 }], + }, + { + timestamp: '2024-01-15T10:00:01.000Z', + message: [2, 'm2', 'Authorize', { idTag: 'TAG-001' }], + }, + { + timestamp: '2024-01-15T10:00:01.500Z', + message: [3, 'm2', { idTagInfo: { status: 'Accepted' } }], + }, + { + timestamp: '2024-01-15T10:00:02.000Z', + message: [2, 'm3', 'StartTransaction', { connectorId: 1, idTag: 'TAG-001' }], + }, + { + timestamp: '2024-01-15T10:00:02.500Z', + message: [3, 'm3', { idTagInfo: { status: 'Accepted' }, transactionId: 42 }], + }, + { + timestamp: '2024-01-15T10:00:03.000Z', + message: [2, 'mh', 'Heartbeat', {}], + }, + { + timestamp: '2024-01-15T10:00:03.500Z', + message: [3, 'mh', { currentTime: '2024-01-15T10:00:03Z' }], + }, + { + timestamp: '2024-01-15T10:00:04.000Z', + message: [ + 2, + 'mmv', + 'MeterValues', + { + connectorId: 1, + transactionId: 42, + meterValue: [{ sampledValue: [{ value: '100' }] }], + }, + ], + }, + { + timestamp: '2024-01-15T10:00:04.500Z', + message: [3, 'mmv', {}], + }, + { + timestamp: '2024-01-15T10:30:00.000Z', + message: [2, 'm4', 'StopTransaction', { transactionId: 42, reason: 'Local' }], + }, + { + timestamp: '2024-01-15T10:30:00.500Z', + message: [3, 'm4', { idTagInfo: { status: 'Accepted' } }], + }, + ], + }, + expectedFailures: [], + assertions: [ + { + type: 'event_order', + params: { + actions: ['BootNotification', 'Authorize', 'StartTransaction', 'StopTransaction'], + }, + }, + { type: 'no_failures', params: {} }, + { type: 'session_state', params: { expected: 'completed' } }, + ], + }; + + const result = evaluateScenario(scenario); + expect(result.allPassed).toBe(true); + expect(result.expectedFailuresPassed).toBe(true); + expect(result.assertions).toHaveLength(3); + expect(result.assertions.every((a) => a.passed)).toBe(true); + }); + + it('evaluates scenario with assertions that fail', () => { + const scenario: Scenario = { + name: 'test-scenario-fail', + description: 'Test scenario with failing assertions', + trace: { + events: [ + { + timestamp: '2024-01-15T10:00:00.000Z', + message: [2, 'm1', 'BootNotification', { chargePointSerialNumber: 'CS-001' }], + }, + { + timestamp: '2024-01-15T10:00:00.500Z', + message: [3, 'm1', { status: 'Accepted', interval: 60 }], + }, + ], + }, + expectedFailures: [], + assertions: [{ type: 'event_count', params: { min: 100 } }], + }; + + const result = evaluateScenario(scenario); + expect(result.allPassed).toBe(false); + expect(result.assertions[0]?.passed).toBe(false); + }); + + it('works backward compatibly with expectedFailures only (no assertions)', () => { + const scenario: Scenario = { + name: 'test-no-assertions', + description: 'Backward compat — no assertions field', + trace: { + events: [ + { + timestamp: '2024-01-15T10:00:00.000Z', + message: [2, 'm1', 'BootNotification', { chargePointSerialNumber: 'CS-001' }], + }, + { + timestamp: '2024-01-15T10:00:00.500Z', + message: [3, 'm1', { status: 'Accepted', interval: 60 }], + }, + ], + }, + expectedFailures: [], + }; + + const result = evaluateScenario(scenario); + expect(result.assertions).toHaveLength(0); + expect(result.allPassed).toBe(true); + }); + }); +}); diff --git a/packages/toolkit/src/core/assertions.ts b/packages/toolkit/src/core/assertions.ts new file mode 100644 index 0000000..c16c245 --- /dev/null +++ b/packages/toolkit/src/core/assertions.ts @@ -0,0 +1,438 @@ +/** + * Scenario assertions — declarative assertions for scenario evaluation. + * + * Supports 8 assertion types: + * - event_order: verify events appear in a specific order + * - event_count: verify the count of events (optionally filtered by action) + * - payload_field: verify a payload field value on a specific action + * - timing: verify timing gaps between two actions + * - session_state: verify session status + * - failure_severity: verify a specific failure has the expected severity + * - no_failures: verify no failures are detected + * - failure_count: verify the count of detected failures + * + * @module @ocpp-debugkit/toolkit/core + */ + +import type { + AssertionResult, + Event, + Failure, + FailureCode, + FailureSeverity, + Scenario, + ScenarioAssertion, + ScenarioEvalResult, + Session, +} from './types.js'; +import { parseTrace } from './parser.js'; +import { buildSessionTimeline } from './timeline.js'; +import { detectFailures } from './detection.js'; +import { summarizeSessions } from './summarizer.js'; + +// --------------------------------------------------------------------------- +// Deep equality (same limited implementation as diff.ts) +// --------------------------------------------------------------------------- + +function deepEqual(a: unknown, b: unknown): boolean { + if (a === b) return true; + if (a === null || b === null) return false; + if (typeof a !== typeof b) return false; + + if (Array.isArray(a) && Array.isArray(b)) { + if (a.length !== b.length) return false; + return a.every((item, i) => deepEqual(item, b[i])); + } + + if (typeof a === 'object' && typeof b === 'object') { + const objA = a as Record; + const objB = b as Record; + const keysA = Object.keys(objA); + const keysB = Object.keys(objB); + if (keysA.length !== keysB.length) return false; + return keysA.every((key) => deepEqual(objA[key], objB[key])); + } + + return false; +} + +/** Safely access a nested field via dot notation (e.g., "idTagInfo.status"). */ +function getNestedValue(obj: unknown, path: string): unknown { + const parts = path.split('.'); + let current: unknown = obj; + for (const part of parts) { + if (current === null || typeof current !== 'object') return undefined; + current = (current as Record)[part]; + } + return current; +} + +// --------------------------------------------------------------------------- +// Assertion evaluators +// --------------------------------------------------------------------------- + +/** Input for assertion evaluation. */ +export interface AssertionContext { + events: Event[]; + sessions: Session[]; + failures: Failure[]; +} + +/** + * Evaluate a single assertion against the analysis context. + */ +export function runAssertion(assertion: ScenarioAssertion, ctx: AssertionContext): AssertionResult { + switch (assertion.type) { + case 'event_order': + return evalEventOrder(assertion, ctx); + case 'event_count': + return evalEventCount(assertion, ctx); + case 'payload_field': + return evalPayloadField(assertion, ctx); + case 'timing': + return evalTiming(assertion, ctx); + case 'session_state': + return evalSessionState(assertion, ctx); + case 'failure_severity': + return evalFailureSeverity(assertion, ctx); + case 'no_failures': + return evalNoFailures(assertion, ctx); + case 'failure_count': + return evalFailureCount(assertion, ctx); + default: + return { + assertion, + passed: false, + message: `Unknown assertion type: ${(assertion as { type: string }).type}`, + }; + } +} + +function evalEventOrder( + assertion: { type: 'event_order'; params: { actions: string[] } }, + ctx: AssertionContext, +): AssertionResult { + const { actions } = assertion.params; + const callActions = ctx.events + .filter((e) => e.messageType === 'Call' && e.action !== null) + .map((e) => e.action as string); + + // Check if actions appear in order (not necessarily consecutive) + let searchFrom = 0; + for (const action of actions) { + const foundIndex = callActions.indexOf(action, searchFrom); + if (foundIndex === -1) { + return { + assertion, + passed: false, + message: `Expected action "${action}" in order after previous actions, but not found`, + }; + } + searchFrom = foundIndex + 1; + } + + return { + assertion, + passed: true, + message: `Actions [${actions.join(', ')}] appear in the correct order`, + }; +} + +function evalEventCount( + assertion: { type: 'event_count'; params: { min?: number; max?: number; action?: string } }, + ctx: AssertionContext, +): AssertionResult { + const { min, max, action } = assertion.params; + const count = action + ? ctx.events.filter((e) => e.messageType === 'Call' && e.action === action).length + : ctx.events.length; + + if (min !== undefined && count < min) { + return { + assertion, + passed: false, + message: `Expected at least ${min} ${action ? `"${action}" ` : ''}events, got ${count}`, + }; + } + + if (max !== undefined && count > max) { + return { + assertion, + passed: false, + message: `Expected at most ${max} ${action ? `"${action}" ` : ''}events, got ${count}`, + }; + } + + return { + assertion, + passed: true, + message: `Event count ${count} is within range [${min ?? 0}, ${max ?? '∞'}]${action ? ` for "${action}"` : ''}`, + }; +} + +function evalPayloadField( + assertion: { + type: 'payload_field'; + params: { action: string; field: string; equals?: unknown; contains?: unknown }; + }, + ctx: AssertionContext, +): AssertionResult { + const { action, field, equals, contains } = assertion.params; + const matchingEvents = ctx.events.filter((e) => e.messageType === 'Call' && e.action === action); + + if (matchingEvents.length === 0) { + return { + assertion, + passed: false, + message: `No events with action "${action}" found`, + }; + } + + for (const event of matchingEvents) { + const value = getNestedValue(event.payload, field); + if (value === undefined) { + return { + assertion, + passed: false, + message: `Field "${field}" not found in payload of "${action}" event`, + }; + } + + if (equals !== undefined && !deepEqual(value, equals)) { + return { + assertion, + passed: false, + message: `Field "${field}" in "${action}" expected to equal ${JSON.stringify(equals)}, got ${JSON.stringify(value)}`, + }; + } + + if (contains !== undefined) { + if (Array.isArray(value) && !value.some((v) => deepEqual(v, contains))) { + return { + assertion, + passed: false, + message: `Field "${field}" in "${action}" expected to contain ${JSON.stringify(contains)}, got ${JSON.stringify(value)}`, + }; + } + if (typeof value === 'string' && !value.includes(String(contains))) { + return { + assertion, + passed: false, + message: `Field "${field}" in "${action}" expected to contain "${contains}", got "${value}"`, + }; + } + } + } + + return { + assertion, + passed: true, + message: `Payload field "${field}" in "${action}" matches expectations`, + }; +} + +function evalTiming( + assertion: { + type: 'timing'; + params: { actionA: string; actionB: string; maxGapMs?: number; minGapMs?: number }; + }, + ctx: AssertionContext, +): AssertionResult { + const { actionA, actionB, maxGapMs, minGapMs } = assertion.params; + + const eventA = ctx.events.find( + (e) => e.messageType === 'Call' && e.action === actionA && e.timestamp !== null, + ); + const eventB = ctx.events.find( + (e) => e.messageType === 'Call' && e.action === actionB && e.timestamp !== null, + ); + + if (!eventA || eventA.timestamp === null) { + return { + assertion, + passed: false, + message: `Action "${actionA}" not found or has no timestamp`, + }; + } + if (!eventB || eventB.timestamp === null) { + return { + assertion, + passed: false, + message: `Action "${actionB}" not found or has no timestamp`, + }; + } + + const gap = eventB.timestamp - eventA.timestamp; + + if (maxGapMs !== undefined && gap > maxGapMs) { + return { + assertion, + passed: false, + message: `Gap between "${actionA}" and "${actionB}" is ${gap}ms, exceeds max ${maxGapMs}ms`, + }; + } + + if (minGapMs !== undefined && gap < minGapMs) { + return { + assertion, + passed: false, + message: `Gap between "${actionA}" and "${actionB}" is ${gap}ms, below min ${minGapMs}ms`, + }; + } + + return { + assertion, + passed: true, + message: `Gap between "${actionA}" and "${actionB}" is ${gap}ms (within range)`, + }; +} + +function evalSessionState( + assertion: { type: 'session_state'; params: { expected: 'active' | 'completed' | 'aborted' } }, + ctx: AssertionContext, +): AssertionResult { + const { expected } = assertion.params; + + if (ctx.sessions.length === 0) { + return { assertion, passed: false, message: 'No sessions found' }; + } + + for (const session of ctx.sessions) { + if (session.status !== expected) { + return { + assertion, + passed: false, + message: `Session ${session.sessionId} has status "${session.status}", expected "${expected}"`, + }; + } + } + + return { + assertion, + passed: true, + message: `All sessions have status "${expected}"`, + }; +} + +function evalFailureSeverity( + assertion: { type: 'failure_severity'; params: { code: FailureCode; severity: FailureSeverity } }, + ctx: AssertionContext, +): AssertionResult { + const { code, severity } = assertion.params; + const matching = ctx.failures.find((f) => f.code === code); + + if (!matching) { + return { + assertion, + passed: false, + message: `Failure with code "${code}" not detected`, + }; + } + + if (matching.severity !== severity) { + return { + assertion, + passed: false, + message: `Failure "${code}" has severity "${matching.severity}", expected "${severity}"`, + }; + } + + return { + assertion, + passed: true, + message: `Failure "${code}" has severity "${severity}" as expected`, + }; +} + +function evalNoFailures( + assertion: { type: 'no_failures'; params: Record }, + ctx: AssertionContext, +): AssertionResult { + if (ctx.failures.length > 0) { + return { + assertion, + passed: false, + message: `Expected no failures, but ${ctx.failures.length} detected: ${ctx.failures.map((f) => f.code).join(', ')}`, + }; + } + + return { assertion, passed: true, message: 'No failures detected as expected' }; +} + +function evalFailureCount( + assertion: { type: 'failure_count'; params: { min?: number; max?: number; code?: FailureCode } }, + ctx: AssertionContext, +): AssertionResult { + const { min, max, code } = assertion.params; + const count = code ? ctx.failures.filter((f) => f.code === code).length : ctx.failures.length; + + if (min !== undefined && count < min) { + return { + assertion, + passed: false, + message: `Expected at least ${min} ${code ? `"${code}" ` : ''}failures, got ${count}`, + }; + } + + if (max !== undefined && count > max) { + return { + assertion, + passed: false, + message: `Expected at most ${max} ${code ? `"${code}" ` : ''}failures, got ${count}`, + }; + } + + return { + assertion, + passed: true, + message: `Failure count ${count} ${code ? `for "${code}" ` : ''}is within range [${min ?? 0}, ${max ?? '∞'}]`, + }; +} + +// --------------------------------------------------------------------------- +// Batch evaluation +// --------------------------------------------------------------------------- + +/** + * Evaluate multiple assertions against the analysis context. + */ +export function runAssertions( + ctx: AssertionContext, + assertions: ScenarioAssertion[], +): AssertionResult[] { + return assertions.map((assertion) => runAssertion(assertion, ctx)); +} + +/** + * Evaluate a scenario: parse the trace, run detection, evaluate assertions, + * and compare detected vs expected failures. + */ +export function evaluateScenario(scenario: Scenario): ScenarioEvalResult { + const { events } = parseTrace(JSON.stringify(scenario.trace)); + const sessions = buildSessionTimeline(events); + const failures = detectFailures(events, sessions); + const detectedFailureCodes = failures.map((f) => f.code); + + // Compare detected vs expected failures + const sortedDetected = [...detectedFailureCodes].sort(); + const sortedExpected = [...scenario.expectedFailures].sort(); + const expectedFailuresPassed = JSON.stringify(sortedDetected) === JSON.stringify(sortedExpected); + + // Evaluate assertions + const ctx: AssertionContext = { events, sessions, failures }; + const assertions = scenario.assertions ? runAssertions(ctx, scenario.assertions) : []; + + // Use summaries to ensure sessions are properly evaluated + summarizeSessions(sessions, failures); + + const allPassed = + expectedFailuresPassed && (assertions.length === 0 || assertions.every((a) => a.passed)); + + return { + assertions, + allPassed, + failures, + detectedFailureCodes, + expectedFailuresPassed, + }; +} diff --git a/packages/toolkit/src/core/index.ts b/packages/toolkit/src/core/index.ts index d63ebb8..8544a39 100644 --- a/packages/toolkit/src/core/index.ts +++ b/packages/toolkit/src/core/index.ts @@ -41,5 +41,9 @@ export { validateMessage, validateMessages } from './validator.js'; export { diffTraces } from './diff.js'; export type { TraceDiff, EventDiff, SummaryDiff } from './diff.js'; +// Assertions +export { runAssertion, runAssertions, evaluateScenario } from './assertions.js'; +export type { AssertionContext } from './assertions.js'; + // Fixtures export * from './fixtures/index.js'; diff --git a/packages/toolkit/src/core/types.ts b/packages/toolkit/src/core/types.ts index 0e7625c..f504ee5 100644 --- a/packages/toolkit/src/core/types.ts +++ b/packages/toolkit/src/core/types.ts @@ -225,4 +225,51 @@ export interface Scenario { trace: Trace; /** Failure codes expected to be detected. */ expectedFailures: FailureCode[]; + /** Optional rich assertions (v0.3). When present, evaluated alongside expectedFailures. */ + assertions?: ScenarioAssertion[]; +} + +// --------------------------------------------------------------------------- +// Scenario Assertions (v0.3) +// --------------------------------------------------------------------------- + +/** A declarative assertion for scenario evaluation. */ +export type ScenarioAssertion = + | { type: 'event_order'; params: { actions: string[] } } + | { type: 'event_count'; params: { min?: number; max?: number; action?: string } } + | { + type: 'payload_field'; + params: { action: string; field: string; equals?: unknown; contains?: unknown }; + } + | { + type: 'timing'; + params: { actionA: string; actionB: string; maxGapMs?: number; minGapMs?: number }; + } + | { type: 'session_state'; params: { expected: 'active' | 'completed' | 'aborted' } } + | { type: 'failure_severity'; params: { code: FailureCode; severity: FailureSeverity } } + | { type: 'no_failures'; params: Record } + | { type: 'failure_count'; params: { min?: number; max?: number; code?: FailureCode } }; + +/** Result of evaluating a single assertion. */ +export interface AssertionResult { + /** The assertion that was evaluated. */ + assertion: ScenarioAssertion; + /** Whether the assertion passed. */ + passed: boolean; + /** Human-readable description of the result. */ + message: string; +} + +/** Result of evaluating all assertions for a scenario. */ +export interface ScenarioEvalResult { + /** Results for each assertion. */ + assertions: AssertionResult[]; + /** Whether all assertions passed. */ + allPassed: boolean; + /** Detected failures. */ + failures: Failure[]; + /** Detected failure codes. */ + detectedFailureCodes: FailureCode[]; + /** Whether expectedFailures matches detectedFailures. */ + expectedFailuresPassed: boolean; } diff --git a/packages/toolkit/src/index.ts b/packages/toolkit/src/index.ts index 3e8cbda..41788fb 100644 --- a/packages/toolkit/src/index.ts +++ b/packages/toolkit/src/index.ts @@ -39,6 +39,7 @@ export { detectFailures } from './core/index.js'; export { summarizeSession, summarizeSessions } from './core/index.js'; export { validateMessage, validateMessages } from './core/index.js'; export { diffTraces } from './core/index.js'; +export { runAssertions, evaluateScenario } from './core/index.js'; // Re-export core types and schemas export type { @@ -61,6 +62,9 @@ export type { TraceDiff, EventDiff, SummaryDiff, + ScenarioAssertion, + AssertionResult, + ScenarioEvalResult, } from './core/index.js'; // Re-export schemas