From 62b4be1480ffda083a741eafc9d3dd33a907e907 Mon Sep 17 00:00:00 2001 From: sepehr-safari Date: Thu, 9 Jul 2026 09:51:32 +0300 Subject: [PATCH] feat(scenarios): assert-based scenario format + 5 new scenarios + report comparison Closes #87 --- .changeset/v03-scenarios-assertions.md | 12 ++ CURRENT_STATE.md | 1 + .../__scenarios__/heartbeat-irregular.ts | 81 ++++++++++ .../scenarios/__scenarios__/meter-anomaly.ts | 153 ++++++++++++++++++ .../scenarios/__scenarios__/short-session.ts | 129 +++++++++++++++ .../__scenarios__/slow-csms-response.ts | 61 +++++++ .../__scenarios__/unresponsive-csms.ts | 53 ++++++ .../toolkit/src/scenarios/compare.test.ts | 89 ++++++++++ packages/toolkit/src/scenarios/compare.ts | 116 +++++++++++++ packages/toolkit/src/scenarios/index.test.ts | 24 ++- packages/toolkit/src/scenarios/index.ts | 28 ++++ tests/external-fixture/test.mjs | 4 +- 12 files changed, 747 insertions(+), 4 deletions(-) create mode 100644 .changeset/v03-scenarios-assertions.md create mode 100644 packages/toolkit/src/scenarios/__scenarios__/heartbeat-irregular.ts create mode 100644 packages/toolkit/src/scenarios/__scenarios__/meter-anomaly.ts create mode 100644 packages/toolkit/src/scenarios/__scenarios__/short-session.ts create mode 100644 packages/toolkit/src/scenarios/__scenarios__/slow-csms-response.ts create mode 100644 packages/toolkit/src/scenarios/__scenarios__/unresponsive-csms.ts create mode 100644 packages/toolkit/src/scenarios/compare.test.ts create mode 100644 packages/toolkit/src/scenarios/compare.ts diff --git a/.changeset/v03-scenarios-assertions.md b/.changeset/v03-scenarios-assertions.md new file mode 100644 index 0000000..8e7e2c7 --- /dev/null +++ b/.changeset/v03-scenarios-assertions.md @@ -0,0 +1,12 @@ +--- +'@ocpp-debugkit/toolkit': minor +--- + +Add 5 new scenarios using rich assertions (15 total): +- `slow-csms-response` — SLOW_RESPONSE + timing assertion +- `meter-anomaly` — METER_VALUE_ANOMALY + payload_field/event_count assertions +- `short-session` — SUSPICIOUS_SESSION_DURATION + session_state/event_count assertions +- `heartbeat-irregular` — HEARTBEAT_INTERVAL_VIOLATION + event_count assertion +- `unresponsive-csms` — UNRESPONSIVE_CSMS + failure_severity/failure_count assertions + +Add `compareScenarioReports()` — compare two scenario evaluation results for regression testing. diff --git a/CURRENT_STATE.md b/CURRENT_STATE.md index 84258f5..988bbe1 100644 --- a/CURRENT_STATE.md +++ b/CURRENT_STATE.md @@ -202,6 +202,7 @@ integration examples, and contributor onboarding. `METER_VALUE_ANOMALY`, `UNRESPONSIVE_CSMS` (15 total rules) - ✅ Trace diffing — `diffTraces()` API (Issue #85) - ✅ Rich scenario assertions — 8 assertion types, `evaluateScenario()` (Issue #86) +- ✅ Assert-based scenarios — 5 new scenarios (15 total) + `compareScenarioReports()` (Issue #87) ## What's Next diff --git a/packages/toolkit/src/scenarios/__scenarios__/heartbeat-irregular.ts b/packages/toolkit/src/scenarios/__scenarios__/heartbeat-irregular.ts new file mode 100644 index 0000000..40785ac --- /dev/null +++ b/packages/toolkit/src/scenarios/__scenarios__/heartbeat-irregular.ts @@ -0,0 +1,81 @@ +export default { + name: 'heartbeat-irregular', + description: + 'Heartbeats sent at irregular intervals — 300s expected, but 600s gap observed. Expects HEARTBEAT_INTERVAL_VIOLATION failure. Uses event_count assertion for heartbeats.', + trace: { + traceId: 'scenario-heartbeat-irregular', + metadata: { + stationId: 'CS-SYNTHETIC-014', + ocppVersion: '1.6', + source: 'synthetic-scenario', + description: 'Heartbeats sent at irregular intervals — 300s expected, 600s gap observed.', + }, + events: [ + { + timestamp: '2024-01-15T12:00:00.000Z', + direction: 'CS_TO_CSMS', + message: [ + 2, + 'msg-001', + 'BootNotification', + { + chargePointVendor: 'SyntheticVendor', + chargePointModel: 'SM-100', + chargePointSerialNumber: 'CS-SYNTHETIC-014', + firmwareVersion: '1.0.0', + }, + ], + }, + { + timestamp: '2024-01-15T12:00:00.500Z', + direction: 'CSMS_TO_CS', + message: [ + 3, + 'msg-001', + { + currentTime: '2024-01-15T12:00:00.500Z', + interval: 300, + status: 'Accepted', + }, + ], + }, + { + timestamp: '2024-01-15T12:05:00.000Z', + direction: 'CS_TO_CSMS', + message: [2, 'msg-hb-1', 'Heartbeat', {}], + }, + { + timestamp: '2024-01-15T12:05:00.500Z', + direction: 'CSMS_TO_CS', + message: [3, 'msg-hb-1', { currentTime: '2024-01-15T12:05:00.500Z' }], + }, + { + timestamp: '2024-01-15T12:15:00.000Z', + direction: 'CS_TO_CSMS', + message: [2, 'msg-hb-2', 'Heartbeat', {}], + }, + { + timestamp: '2024-01-15T12:15:00.500Z', + direction: 'CSMS_TO_CS', + message: [3, 'msg-hb-2', { currentTime: '2024-01-15T12:15:00.500Z' }], + }, + { + timestamp: '2024-01-15T12:25:00.000Z', + direction: 'CS_TO_CSMS', + message: [2, 'msg-hb-3', 'Heartbeat', {}], + }, + { + timestamp: '2024-01-15T12:25:00.500Z', + direction: 'CSMS_TO_CS', + message: [3, 'msg-hb-3', { currentTime: '2024-01-15T12:25:00.500Z' }], + }, + ], + }, + expectedFailures: ['HEARTBEAT_INTERVAL_VIOLATION'], + assertions: [ + { + type: 'event_count', + params: { min: 3, action: 'Heartbeat' }, + }, + ], +}; diff --git a/packages/toolkit/src/scenarios/__scenarios__/meter-anomaly.ts b/packages/toolkit/src/scenarios/__scenarios__/meter-anomaly.ts new file mode 100644 index 0000000..f88d672 --- /dev/null +++ b/packages/toolkit/src/scenarios/__scenarios__/meter-anomaly.ts @@ -0,0 +1,153 @@ +export default { + name: 'meter-anomaly', + description: + 'Meter values decrease during an active transaction (non-monotonic). Expects METER_VALUE_ANOMALY failure. Uses payload_field assertion to verify meter values.', + trace: { + traceId: 'scenario-meter-anomaly', + metadata: { + stationId: 'CS-SYNTHETIC-012', + ocppVersion: '1.6', + source: 'synthetic-scenario', + description: + 'Meter reading decreases from 5000 to 3000 during charging — non-monotonic anomaly.', + }, + events: [ + { + timestamp: '2024-01-15T14:00:00.000Z', + direction: 'CS_TO_CSMS', + message: [ + 2, + 'msg-001', + 'BootNotification', + { + chargePointVendor: 'SyntheticVendor', + chargePointModel: 'SM-100', + chargePointSerialNumber: 'CS-SYNTHETIC-012', + firmwareVersion: '1.0.0', + }, + ], + }, + { + timestamp: '2024-01-15T14:00:00.500Z', + direction: 'CSMS_TO_CS', + message: [ + 3, + 'msg-001', + { + currentTime: '2024-01-15T14:00:00.500Z', + interval: 300, + status: 'Accepted', + }, + ], + }, + { + timestamp: '2024-01-15T14:01:00.000Z', + direction: 'CS_TO_CSMS', + message: [2, 'msg-002', 'Authorize', { idTag: 'SYNTHETIC-TAG-012' }], + }, + { + timestamp: '2024-01-15T14:01:00.500Z', + direction: 'CSMS_TO_CS', + message: [3, 'msg-002', { idTagInfo: { status: 'Accepted' } }], + }, + { + timestamp: '2024-01-15T14:02:00.000Z', + direction: 'CS_TO_CSMS', + message: [ + 2, + 'msg-003', + 'StartTransaction', + { connectorId: 1, idTag: 'SYNTHETIC-TAG-012', meterStart: 0 }, + ], + }, + { + timestamp: '2024-01-15T14:02:00.500Z', + direction: 'CSMS_TO_CS', + message: [3, 'msg-003', { idTagInfo: { status: 'Accepted' }, transactionId: 100012 }], + }, + { + timestamp: '2024-01-15T14:05:00.000Z', + direction: 'CS_TO_CSMS', + message: [ + 2, + 'msg-004', + 'MeterValues', + { + connectorId: 1, + transactionId: 100012, + meterValue: [{ sampledValue: [{ value: '5000' }] }], + }, + ], + }, + { + timestamp: '2024-01-15T14:05:00.500Z', + direction: 'CSMS_TO_CS', + message: [3, 'msg-004', {}], + }, + { + timestamp: '2024-01-15T14:06:00.000Z', + direction: 'CS_TO_CSMS', + message: [ + 2, + 'msg-005', + 'MeterValues', + { + connectorId: 1, + transactionId: 100012, + meterValue: [{ sampledValue: [{ value: '3000' }] }], + }, + ], + }, + { + timestamp: '2024-01-15T14:06:00.500Z', + direction: 'CSMS_TO_CS', + message: [3, 'msg-005', {}], + }, + { + timestamp: '2024-01-15T14:10:00.000Z', + direction: 'CS_TO_CSMS', + message: [2, 'msg-006', 'Heartbeat', {}], + }, + { + timestamp: '2024-01-15T14:10:00.500Z', + direction: 'CSMS_TO_CS', + message: [3, 'msg-006', { currentTime: '2024-01-15T14:10:00.500Z' }], + }, + { + timestamp: '2024-01-15T14:30:00.000Z', + direction: 'CS_TO_CSMS', + message: [ + 2, + 'msg-007', + 'StopTransaction', + { + transactionId: 100012, + idTag: 'SYNTHETIC-TAG-012', + meterStop: 3000, + reason: 'EVDisconnected', + }, + ], + }, + { + timestamp: '2024-01-15T14:30:00.500Z', + direction: 'CSMS_TO_CS', + message: [3, 'msg-007', { idTagInfo: { status: 'Accepted' } }], + }, + ], + }, + expectedFailures: ['METER_VALUE_ANOMALY'], + assertions: [ + { + type: 'payload_field', + params: { + action: 'StartTransaction', + field: 'connectorId', + equals: 1, + }, + }, + { + type: 'event_count', + params: { min: 2, max: 2, action: 'MeterValues' }, + }, + ], +}; diff --git a/packages/toolkit/src/scenarios/__scenarios__/short-session.ts b/packages/toolkit/src/scenarios/__scenarios__/short-session.ts new file mode 100644 index 0000000..69714a7 --- /dev/null +++ b/packages/toolkit/src/scenarios/__scenarios__/short-session.ts @@ -0,0 +1,129 @@ +export default { + name: 'short-session', + description: + 'Session lasts only 10 seconds — suspiciously short. Expects SUSPICIOUS_SESSION_DURATION failure. Uses session_state and event_count assertions.', + trace: { + traceId: 'scenario-short-session', + metadata: { + stationId: 'CS-SYNTHETIC-013', + ocppVersion: '1.6', + source: 'synthetic-scenario', + description: 'Transaction starts and stops within 10 seconds — suspiciously short.', + }, + events: [ + { + timestamp: '2024-01-15T09:00:00.000Z', + direction: 'CS_TO_CSMS', + message: [ + 2, + 'msg-001', + 'BootNotification', + { + chargePointVendor: 'SyntheticVendor', + chargePointModel: 'SM-100', + chargePointSerialNumber: 'CS-SYNTHETIC-013', + firmwareVersion: '1.0.0', + }, + ], + }, + { + timestamp: '2024-01-15T09:00:00.500Z', + direction: 'CSMS_TO_CS', + message: [ + 3, + 'msg-001', + { + currentTime: '2024-01-15T09:00:00.500Z', + interval: 300, + status: 'Accepted', + }, + ], + }, + { + timestamp: '2024-01-15T09:00:01.000Z', + direction: 'CS_TO_CSMS', + message: [2, 'msg-002', 'Authorize', { idTag: 'SYNTHETIC-TAG-013' }], + }, + { + timestamp: '2024-01-15T09:00:01.500Z', + direction: 'CSMS_TO_CS', + message: [3, 'msg-002', { idTagInfo: { status: 'Accepted' } }], + }, + { + timestamp: '2024-01-15T09:00:02.000Z', + direction: 'CS_TO_CSMS', + message: [ + 2, + 'msg-003', + 'StartTransaction', + { connectorId: 1, idTag: 'SYNTHETIC-TAG-013', meterStart: 0 }, + ], + }, + { + timestamp: '2024-01-15T09:00:02.500Z', + direction: 'CSMS_TO_CS', + message: [3, 'msg-003', { idTagInfo: { status: 'Accepted' }, transactionId: 100013 }], + }, + { + timestamp: '2024-01-15T09:00:03.000Z', + direction: 'CS_TO_CSMS', + message: [ + 2, + 'msg-004', + 'MeterValues', + { + connectorId: 1, + transactionId: 100013, + meterValue: [{ sampledValue: [{ value: '100' }] }], + }, + ], + }, + { + timestamp: '2024-01-15T09:00:03.500Z', + direction: 'CSMS_TO_CS', + message: [3, 'msg-004', {}], + }, + { + timestamp: '2024-01-15T09:00:04.000Z', + direction: 'CS_TO_CSMS', + message: [2, 'msg-005', 'Heartbeat', {}], + }, + { + timestamp: '2024-01-15T09:00:04.500Z', + direction: 'CSMS_TO_CS', + message: [3, 'msg-005', { currentTime: '2024-01-15T09:00:04.500Z' }], + }, + { + timestamp: '2024-01-15T09:00:10.000Z', + direction: 'CS_TO_CSMS', + message: [ + 2, + 'msg-006', + 'StopTransaction', + { + transactionId: 100013, + idTag: 'SYNTHETIC-TAG-013', + meterStop: 100, + reason: 'EVDisconnected', + }, + ], + }, + { + timestamp: '2024-01-15T09:00:10.500Z', + direction: 'CSMS_TO_CS', + message: [3, 'msg-006', { idTagInfo: { status: 'Accepted' } }], + }, + ], + }, + expectedFailures: ['SUSPICIOUS_SESSION_DURATION'], + assertions: [ + { + type: 'session_state', + params: { expected: 'completed' }, + }, + { + type: 'event_count', + params: { min: 1, action: 'StartTransaction' }, + }, + ], +}; diff --git a/packages/toolkit/src/scenarios/__scenarios__/slow-csms-response.ts b/packages/toolkit/src/scenarios/__scenarios__/slow-csms-response.ts new file mode 100644 index 0000000..0e07c2a --- /dev/null +++ b/packages/toolkit/src/scenarios/__scenarios__/slow-csms-response.ts @@ -0,0 +1,61 @@ +export default { + name: 'slow-csms-response', + description: + 'CSMS takes 15 seconds to respond to a BootNotification Call. Expects SLOW_RESPONSE failure. Uses timing assertion to verify response gap.', + trace: { + traceId: 'scenario-slow-csms-response', + metadata: { + stationId: 'CS-SYNTHETIC-011', + ocppVersion: '1.6', + source: 'synthetic-scenario', + description: 'CSMS responds slowly to BootNotification — 15s response gap.', + }, + events: [ + { + timestamp: '2024-01-15T06:00:00.000Z', + direction: 'CS_TO_CSMS', + message: [ + 2, + 'msg-001', + 'BootNotification', + { + chargePointVendor: 'SyntheticVendor', + chargePointModel: 'SM-100', + chargePointSerialNumber: 'CS-SYNTHETIC-011', + firmwareVersion: '1.0.0', + }, + ], + }, + { + timestamp: '2024-01-15T06:00:15.000Z', + direction: 'CSMS_TO_CS', + message: [ + 3, + 'msg-001', + { + currentTime: '2024-01-15T06:00:15.000Z', + interval: 300, + status: 'Accepted', + }, + ], + }, + { + timestamp: '2024-01-15T06:00:16.000Z', + direction: 'CS_TO_CSMS', + message: [2, 'msg-002', 'Heartbeat', {}], + }, + { + timestamp: '2024-01-15T06:00:16.500Z', + direction: 'CSMS_TO_CS', + message: [3, 'msg-002', { currentTime: '2024-01-15T06:00:16.500Z' }], + }, + ], + }, + expectedFailures: ['SLOW_RESPONSE'], + assertions: [ + { + type: 'timing', + params: { actionA: 'BootNotification', actionB: 'Heartbeat', maxGapMs: 5000 }, + }, + ], +}; diff --git a/packages/toolkit/src/scenarios/__scenarios__/unresponsive-csms.ts b/packages/toolkit/src/scenarios/__scenarios__/unresponsive-csms.ts new file mode 100644 index 0000000..c94c731 --- /dev/null +++ b/packages/toolkit/src/scenarios/__scenarios__/unresponsive-csms.ts @@ -0,0 +1,53 @@ +export default { + name: 'unresponsive-csms', + description: + 'BootNotification Call sent but CSMS never responds — no CallResult or CallError. Expects UNRESPONSIVE_CSMS failure. Uses failure_severity and failure_count assertions.', + trace: { + traceId: 'scenario-unresponsive-csms', + metadata: { + stationId: 'CS-SYNTHETIC-015', + ocppVersion: '1.6', + source: 'synthetic-scenario', + description: 'CSMS never responds to BootNotification — unresponsive CSMS.', + }, + events: [ + { + timestamp: '2024-01-15T08:00:00.000Z', + direction: 'CS_TO_CSMS', + message: [ + 2, + 'msg-001', + 'BootNotification', + { + chargePointVendor: 'SyntheticVendor', + chargePointModel: 'SM-100', + chargePointSerialNumber: 'CS-SYNTHETIC-015', + firmwareVersion: '1.0.0', + }, + ], + }, + // No CallResult for msg-001 — CSMS unresponsive + { + timestamp: '2024-01-15T08:00:05.000Z', + direction: 'CS_TO_CSMS', + message: [2, 'msg-002', 'Heartbeat', {}], + }, + { + timestamp: '2024-01-15T08:00:05.500Z', + direction: 'CSMS_TO_CS', + message: [3, 'msg-002', { currentTime: '2024-01-15T08:00:05.500Z' }], + }, + ], + }, + expectedFailures: ['UNRESPONSIVE_CSMS'], + assertions: [ + { + type: 'failure_severity', + params: { code: 'UNRESPONSIVE_CSMS', severity: 'critical' }, + }, + { + type: 'failure_count', + params: { code: 'UNRESPONSIVE_CSMS', min: 1, max: 1 }, + }, + ], +}; diff --git a/packages/toolkit/src/scenarios/compare.test.ts b/packages/toolkit/src/scenarios/compare.test.ts new file mode 100644 index 0000000..e259e50 --- /dev/null +++ b/packages/toolkit/src/scenarios/compare.test.ts @@ -0,0 +1,89 @@ +import { describe, it, expect } from 'vitest'; +import { compareScenarioReports } from './compare.js'; +import type { ScenarioEvalResult } from '../core/index.js'; + +function makeResult( + allPassed: boolean, + detectedFailureCodes: string[], + assertionsPassed: boolean[] = [], +): ScenarioEvalResult { + return { + assertions: assertionsPassed.map((passed, i) => ({ + assertion: { type: 'event_order' as const, params: { actions: [`action-${i}`] } }, + passed, + message: `assertion ${i}`, + })), + allPassed, + failures: detectedFailureCodes.map((code) => ({ + code: code as 'FAILED_AUTHORIZATION', + description: `${code} failure`, + severity: 'warning' as const, + eventIds: [], + suggestedSteps: [], + })), + detectedFailureCodes: detectedFailureCodes as 'FAILED_AUTHORIZATION'[], + expectedFailuresPassed: allPassed, + }; +} + +describe('compareScenarioReports', () => { + it('reports both passed', () => { + const a = makeResult(true, []); + const b = makeResult(true, []); + const result = compareScenarioReports(a, b); + expect(result.bothPassed).toBe(true); + expect(result.aPassed).toBe(true); + expect(result.bPassed).toBe(true); + expect(result.failuresOnlyInA).toHaveLength(0); + expect(result.failuresOnlyInB).toHaveLength(0); + }); + + it('detects regression (A passed, B failed)', () => { + const a = makeResult(true, []); + const b = makeResult(false, ['FAILED_AUTHORIZATION']); + const result = compareScenarioReports(a, b); + expect(result.aPassed).toBe(true); + expect(result.bPassed).toBe(false); + expect(result.failuresOnlyInA).toHaveLength(0); + expect(result.failuresOnlyInB).toContain('FAILED_AUTHORIZATION'); + expect(result.summary).toContain('regression'); + }); + + it('detects improvement (A failed, B passed)', () => { + const a = makeResult(false, ['FAILED_AUTHORIZATION']); + const b = makeResult(true, []); + const result = compareScenarioReports(a, b); + expect(result.aPassed).toBe(false); + expect(result.bPassed).toBe(true); + expect(result.failuresOnlyInA).toContain('FAILED_AUTHORIZATION'); + expect(result.summary).toContain('improvement'); + }); + + it('detects failure code changes', () => { + const a = makeResult(false, ['FAILED_AUTHORIZATION', 'CONNECTOR_FAULT']); + const b = makeResult(false, ['FAILED_AUTHORIZATION', 'METER_VALUE_GAP']); + const result = compareScenarioReports(a, b); + expect(result.failuresOnlyInA).toContain('CONNECTOR_FAULT'); + expect(result.failuresOnlyInB).toContain('METER_VALUE_GAP'); + }); + + it('detects assertion result changes', () => { + const a = makeResult(false, [], [true, false, true]); + const b = makeResult(false, [], [true, true, false]); + const result = compareScenarioReports(a, b); + expect(result.changedAssertions).toHaveLength(2); + expect(result.changedAssertions[0]?.assertionType).toBe('event_order'); + expect(result.changedAssertions[0]?.passedInA).toBe(false); + expect(result.changedAssertions[0]?.passedInB).toBe(true); + }); + + it('reports both failed with same failures', () => { + const a = makeResult(false, ['FAILED_AUTHORIZATION']); + const b = makeResult(false, ['FAILED_AUTHORIZATION']); + const result = compareScenarioReports(a, b); + expect(result.bothPassed).toBe(false); + expect(result.failuresOnlyInA).toHaveLength(0); + expect(result.failuresOnlyInB).toHaveLength(0); + expect(result.summary).toContain('Both scenarios failed'); + }); +}); diff --git a/packages/toolkit/src/scenarios/compare.ts b/packages/toolkit/src/scenarios/compare.ts new file mode 100644 index 0000000..e941a3c --- /dev/null +++ b/packages/toolkit/src/scenarios/compare.ts @@ -0,0 +1,116 @@ +/** + * Scenario comparison — compare two scenario evaluation results. + * + * Useful for regression testing: did the fix change the detected failures? + * Compare a "before" run against an "after" run to see what changed. + * + * @module @ocpp-debugkit/toolkit/scenarios + */ + +import type { FailureCode, ScenarioEvalResult } from '../core/index.js'; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +/** Result of comparing two scenario evaluation reports. */ +export interface ScenarioComparison { + /** Whether both scenarios passed all assertions. */ + bothPassed: boolean; + /** Whether the first scenario (a) passed. */ + aPassed: boolean; + /** Whether the second scenario (b) passed. */ + bPassed: boolean; + /** Failure codes detected in A but not B. */ + failuresOnlyInA: FailureCode[]; + /** Failure codes detected in B but not A. */ + failuresOnlyInB: FailureCode[]; + /** Assertion results that changed (passed→failed or failed→passed). */ + changedAssertions: { + assertionType: string; + passedInA: boolean; + passedInB: boolean; + }[]; + /** Human-readable summary of changes. */ + summary: string; +} + +// --------------------------------------------------------------------------- +// Comparison +// --------------------------------------------------------------------------- + +/** + * Compare two scenario evaluation results. + * + * @param a - First scenario evaluation result + * @param b - Second scenario evaluation result + * @returns A {@link ScenarioComparison} describing the differences + */ +export function compareScenarioReports( + a: ScenarioEvalResult, + b: ScenarioEvalResult, +): ScenarioComparison { + const aPassed = a.allPassed; + const bPassed = b.allPassed; + const bothPassed = aPassed && bPassed; + + // Failure code diff + const aCodes = new Set(a.detectedFailureCodes); + const bCodes = new Set(b.detectedFailureCodes); + const failuresOnlyInA = a.detectedFailureCodes.filter((c) => !bCodes.has(c)); + const failuresOnlyInB = b.detectedFailureCodes.filter((c) => !aCodes.has(c)); + + // Assertion diff — compare assertions at the same index + const maxLen = Math.max(a.assertions.length, b.assertions.length); + const changedAssertions: { + assertionType: string; + passedInA: boolean; + passedInB: boolean; + }[] = []; + + for (let i = 0; i < maxLen; i++) { + const aResult = a.assertions[i]; + const bResult = b.assertions[i]; + if (!aResult || !bResult) continue; + + if (aResult.passed !== bResult.passed) { + changedAssertions.push({ + assertionType: aResult.assertion.type, + passedInA: aResult.passed, + passedInB: bResult.passed, + }); + } + } + + // Summary + const parts: string[] = []; + if (aPassed && !bPassed) { + parts.push('Scenario A passed but B failed — regression detected'); + } else if (!aPassed && bPassed) { + parts.push('Scenario A failed but B passed — improvement detected'); + } else if (bothPassed) { + parts.push('Both scenarios passed all assertions'); + } else { + parts.push('Both scenarios failed'); + } + + if (failuresOnlyInA.length > 0) { + parts.push(`Failures only in A: ${failuresOnlyInA.join(', ')}`); + } + if (failuresOnlyInB.length > 0) { + parts.push(`Failures only in B: ${failuresOnlyInB.join(', ')}`); + } + if (changedAssertions.length > 0) { + parts.push(`${changedAssertions.length} assertion(s) changed result`); + } + + return { + bothPassed, + aPassed, + bPassed, + failuresOnlyInA, + failuresOnlyInB, + changedAssertions, + summary: parts.join('. '), + }; +} diff --git a/packages/toolkit/src/scenarios/index.test.ts b/packages/toolkit/src/scenarios/index.test.ts index d8531e1..28b1493 100644 --- a/packages/toolkit/src/scenarios/index.test.ts +++ b/packages/toolkit/src/scenarios/index.test.ts @@ -13,6 +13,11 @@ import { unexpectedStartScenario, statusTransitionViolationScenario, diagnosticsFailureScenario, + slowCsmsResponseScenario, + meterAnomalyScenario, + shortSessionScenario, + heartbeatIrregularScenario, + unresponsiveCsmsScenario, } from './index.js'; import { parseTrace, buildSessionTimeline, detectFailures } from '../core/index.js'; @@ -21,8 +26,8 @@ import { parseTrace, buildSessionTimeline, detectFailures } from '../core/index. // --------------------------------------------------------------------------- describe('scenario registry', () => { - it('exports exactly 10 scenarios', () => { - expect(scenarios).toHaveLength(10); + it('exports exactly 15 scenarios', () => { + expect(scenarios).toHaveLength(15); }); it('exports scenario names in order', () => { @@ -37,6 +42,11 @@ describe('scenario registry', () => { 'unexpected-start', 'status-transition-violation', 'diagnostics-failure', + 'slow-csms-response', + 'meter-anomaly', + 'short-session', + 'heartbeat-irregular', + 'unresponsive-csms', ]); }); @@ -60,6 +70,11 @@ describe('scenario registry', () => { expect(getScenario('unexpected-start')).toBe(unexpectedStartScenario); expect(getScenario('status-transition-violation')).toBe(statusTransitionViolationScenario); expect(getScenario('diagnostics-failure')).toBe(diagnosticsFailureScenario); + expect(getScenario('slow-csms-response')).toBe(slowCsmsResponseScenario); + expect(getScenario('meter-anomaly')).toBe(meterAnomalyScenario); + expect(getScenario('short-session')).toBe(shortSessionScenario); + expect(getScenario('heartbeat-irregular')).toBe(heartbeatIrregularScenario); + expect(getScenario('unresponsive-csms')).toBe(unresponsiveCsmsScenario); }); it('getScenario returns undefined for unknown name', () => { @@ -83,6 +98,11 @@ describe('expectedFailures alignment with detection rules', () => { 'STATUS_TRANSITION_VIOLATION', 'DIAGNOSTICS_FAILURE', 'FIRMWARE_UPDATE_FAILURE', + 'SUSPICIOUS_SESSION_DURATION', + 'SLOW_RESPONSE', + 'HEARTBEAT_INTERVAL_VIOLATION', + 'METER_VALUE_ANOMALY', + 'UNRESPONSIVE_CSMS', ]); it('all expectedFailures reference valid failure codes', () => { diff --git a/packages/toolkit/src/scenarios/index.ts b/packages/toolkit/src/scenarios/index.ts index 3cd18bb..38aad78 100644 --- a/packages/toolkit/src/scenarios/index.ts +++ b/packages/toolkit/src/scenarios/index.ts @@ -15,6 +15,11 @@ import invalidStopReason from './__scenarios__/invalid-stop-reason.js'; import unexpectedStart from './__scenarios__/unexpected-start.js'; import statusTransitionViolation from './__scenarios__/status-transition-violation.js'; import diagnosticsFailure from './__scenarios__/diagnostics-failure.js'; +import slowCsmsResponse from './__scenarios__/slow-csms-response.js'; +import meterAnomaly from './__scenarios__/meter-anomaly.js'; +import shortSession from './__scenarios__/short-session.js'; +import heartbeatIrregular from './__scenarios__/heartbeat-irregular.js'; +import unresponsiveCsms from './__scenarios__/unresponsive-csms.js'; // --------------------------------------------------------------------------- // Scenarios derived from core fixtures @@ -56,6 +61,11 @@ const unexpectedStartScenario: Scenario = unexpectedStart as unknown as Scenario const statusTransitionViolationScenario: Scenario = statusTransitionViolation as unknown as Scenario; const diagnosticsFailureScenario: Scenario = diagnosticsFailure as unknown as Scenario; +const slowCsmsResponseScenario: Scenario = slowCsmsResponse as unknown as Scenario; +const meterAnomalyScenario: Scenario = meterAnomaly as unknown as Scenario; +const shortSessionScenario: Scenario = shortSession as unknown as Scenario; +const heartbeatIrregularScenario: Scenario = heartbeatIrregular as unknown as Scenario; +const unresponsiveCsmsScenario: Scenario = unresponsiveCsms as unknown as Scenario; // --------------------------------------------------------------------------- // Registry @@ -72,6 +82,11 @@ export const scenarios = [ unexpectedStartScenario, statusTransitionViolationScenario, diagnosticsFailureScenario, + slowCsmsResponseScenario, + meterAnomalyScenario, + shortSessionScenario, + heartbeatIrregularScenario, + unresponsiveCsmsScenario, ] as const; export const scenarioNames = [ @@ -85,6 +100,11 @@ export const scenarioNames = [ 'unexpected-start', 'status-transition-violation', 'diagnostics-failure', + 'slow-csms-response', + 'meter-anomaly', + 'short-session', + 'heartbeat-irregular', + 'unresponsive-csms', ] as const; export { @@ -98,8 +118,16 @@ export { unexpectedStartScenario, statusTransitionViolationScenario, diagnosticsFailureScenario, + slowCsmsResponseScenario, + meterAnomalyScenario, + shortSessionScenario, + heartbeatIrregularScenario, + unresponsiveCsmsScenario, }; +export { compareScenarioReports } from './compare.js'; +export type { ScenarioComparison } from './compare.js'; + /** * Get a scenario by name. * @returns The scenario, or undefined if not found. diff --git a/tests/external-fixture/test.mjs b/tests/external-fixture/test.mjs index 3ced7af..efeec34 100644 --- a/tests/external-fixture/test.mjs +++ b/tests/external-fixture/test.mjs @@ -110,8 +110,8 @@ const scenarios = await import('@ocpp-debugkit/toolkit/scenarios'); assert(Array.isArray(scenarios.scenarios), 'scenarios is an array'); assert( - scenarios.scenarios.length === 10, - `10 scenarios exported (got ${scenarios.scenarios.length})`, + scenarios.scenarios.length === 15, + `15 scenarios exported (got ${scenarios.scenarios.length})`, ); assert(typeof scenarios.getScenario === 'function', 'getScenario is a function'); assert(scenarios.getScenario('normal-session') !== undefined, 'normal-session scenario exists');