From 5a0c2aa5db1a1dc8226b15c84b09e8aa0c48b137 Mon Sep 17 00:00:00 2001 From: kevinccbsg Date: Sat, 4 Jul 2026 12:11:13 +0200 Subject: [PATCH 1/5] docs: add describe-aware --test filter design spec Co-Authored-By: Claude Fable 5 --- ...07-04-describe-aware-test-filter-design.md | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-04-describe-aware-test-filter-design.md diff --git a/docs/superpowers/specs/2026-07-04-describe-aware-test-filter-design.md b/docs/superpowers/specs/2026-07-04-describe-aware-test-filter-design.md new file mode 100644 index 0000000..d32f9dc --- /dev/null +++ b/docs/superpowers/specs/2026-07-04-describe-aware-test-filter-design.md @@ -0,0 +1,67 @@ +# Describe-Aware `--test` Filtering — Design + +**Date:** 2026-07-04 +**Status:** Approved + +## Problem + +`--test "name"` (and the `testNames` field of the `run` command) only substring-matches +`it(...)` test names. Filtering by a `describe(...)` name silently matches nothing. +This creates friction for developers and especially for AI agents, since most models +assume describe-name matching is the default behavior (as in Vitest/Jest). twd-cli +already implements describe-aware matching; twd-relay should match its semantics. + +## Approach (chosen) + +Full-path matching, identical to twd-cli: for each test, build the path +`"Describe > nested describe > test name"` by walking `parent` links in +`window.__TWD_STATE__.handlers`, then substring-match each lowercased filter against +the lowercased path. A filter equal to a describe name selects all tests under it; +cross-boundary filters like `"login > error"` also work. + +Rejected alternative: matching test name OR any ancestor name individually — slightly +stricter, but diverges from twd-cli and blocks cross-boundary filters. + +## Scope + +Only the browser client's matching logic changes. Relay server, message protocol, and +CLI flag parsing are untouched — `testNames: string[]` already flows through as-is. + +## Changes + +### 1. New helper `buildTestPath` (`src/browser/buildTestPath.ts`) + +Port of twd-cli's `buildTestPath`, adapted to the `Map` the browser +client already reads from `__TWD_STATE__`. Walks `handler.parent` up through suites, +joins names with `" > "`, returns `null` if the starting id is missing. Pure function. + +### 2. Matching in `handleRunCommand` (`src/browser/createBrowserClient.ts`) + +Replace the current name-only loop (~lines 149–177): for each handler with +`type === 'test'`, build its full path and match each filter as a lowercased substring +of the lowercased path. + +### 3. NO_MATCH message lists full paths + +The existing `NO_MATCH` error lists available tests by bare name; change it to list +full paths (`"Login flow > shows error"`), so an agent can construct a working retry. + +### 4. Docs + +Update README's `--test` description (relay and `run` subcommand sections): matches +against the full describe-path, with an example showing filtering by describe name. + +## Testing + +- Unit tests for `buildTestPath`: root-level test, nested describes, missing parent id. +- Browser-client tests via the existing pattern (unique port, `TrackedWs` wrapper): + - filter by describe name selects all tests under it + - filter by test name still works + - `run` message with `testNames` containing a `" > "` cross-boundary filter matches + - no-match case emits `NO_MATCH` listing full paths + +## Not included (YAGNI) + +twd-cli's per-filter "matched nothing (others did)" warning — the relay protocol has +no warning channel and the all-or-nothing `NO_MATCH` covers the agent-facing failure +mode. Revisit if friction shows up. From 6a4308134bebe811089b91153ddff05ccd62391b Mon Sep 17 00:00:00 2001 From: kevinccbsg Date: Sat, 4 Jul 2026 12:19:08 +0200 Subject: [PATCH 2/5] feat: add buildTestPath helper for describe-path construction --- src/browser/buildTestPath.ts | 25 ++++++++++++++++ src/tests/browser/buildTestPath.spec.ts | 40 +++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 src/browser/buildTestPath.ts create mode 100644 src/tests/browser/buildTestPath.spec.ts diff --git a/src/browser/buildTestPath.ts b/src/browser/buildTestPath.ts new file mode 100644 index 0000000..dac94b3 --- /dev/null +++ b/src/browser/buildTestPath.ts @@ -0,0 +1,25 @@ +export interface PathHandler { + id: string; + name: string; + parent?: string; + type: 'suite' | 'test'; +} + +/** + * Builds the full describe-path for a test, e.g. + * "Login flow > validation > shows error on bad password". + * Returns null if the starting id is not in the map. + */ +export function buildTestPath( + testId: string, + handlers: Map, +): string | null { + let current = handlers.get(testId); + if (!current) return null; + const parts: string[] = []; + while (current) { + parts.unshift(current.name); + current = current.parent ? handlers.get(current.parent) : undefined; + } + return parts.join(' > '); +} diff --git a/src/tests/browser/buildTestPath.spec.ts b/src/tests/browser/buildTestPath.spec.ts new file mode 100644 index 0000000..8d4b898 --- /dev/null +++ b/src/tests/browser/buildTestPath.spec.ts @@ -0,0 +1,40 @@ +import { describe, it, expect } from 'vitest'; +import { buildTestPath, type PathHandler } from '../../browser/buildTestPath'; + +function makeHandlers(list: PathHandler[]): Map { + return new Map(list.map((h) => [h.id, h])); +} + +describe('buildTestPath', () => { + it('returns the bare name for a root-level test', () => { + const handlers = makeHandlers([ + { id: 't1', name: 'adds numbers', type: 'test' }, + ]); + expect(buildTestPath('t1', handlers)).toBe('adds numbers'); + }); + + it('joins nested describe names with " > "', () => { + const handlers = makeHandlers([ + { id: 's1', name: 'Login flow', type: 'suite' }, + { id: 's2', name: 'validation', type: 'suite', parent: 's1' }, + { id: 't1', name: 'shows error on bad password', type: 'test', parent: 's2' }, + ]); + expect(buildTestPath('t1', handlers)).toBe( + 'Login flow > validation > shows error on bad password' + ); + }); + + it('returns null for an unknown id', () => { + const handlers = makeHandlers([ + { id: 't1', name: 'adds numbers', type: 'test' }, + ]); + expect(buildTestPath('missing', handlers)).toBeNull(); + }); + + it('stops walking when a parent id is missing from the map', () => { + const handlers = makeHandlers([ + { id: 't1', name: 'orphan test', type: 'test', parent: 'gone' }, + ]); + expect(buildTestPath('t1', handlers)).toBe('orphan test'); + }); +}); From ddf435aac3c596952028acfca1eb3789d59b682e Mon Sep 17 00:00:00 2001 From: kevinccbsg Date: Sat, 4 Jul 2026 12:20:43 +0200 Subject: [PATCH 3/5] feat: add describe-aware selectTestIds and listTestPaths --- src/browser/filterTests.ts | 36 ++++++++++++++++ src/tests/browser/filterTests.spec.ts | 60 +++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 src/browser/filterTests.ts create mode 100644 src/tests/browser/filterTests.spec.ts diff --git a/src/browser/filterTests.ts b/src/browser/filterTests.ts new file mode 100644 index 0000000..13a2780 --- /dev/null +++ b/src/browser/filterTests.ts @@ -0,0 +1,36 @@ +import { buildTestPath, type PathHandler } from './buildTestPath'; + +/** + * Returns ids of tests whose full describe-path contains any filter + * as a case-insensitive substring. Mirrors twd-cli's selectTestIds. + */ +export function selectTestIds( + handlers: Map, + filters: string[], +): string[] { + const needles = filters.map((f) => f.toLowerCase()); + const ids: string[] = []; + + for (const [, handler] of handlers) { + if (handler.type !== 'test') continue; + const path = buildTestPath(handler.id, handlers); + if (!path) continue; + const haystack = path.toLowerCase(); + if (needles.some((n) => haystack.includes(n))) { + ids.push(handler.id); + } + } + + return ids; +} + +/** Full describe-paths of all tests — used in the NO_MATCH error message. */ +export function listTestPaths(handlers: Map): string[] { + const paths: string[] = []; + for (const [, handler] of handlers) { + if (handler.type !== 'test') continue; + const path = buildTestPath(handler.id, handlers); + if (path) paths.push(path); + } + return paths; +} diff --git a/src/tests/browser/filterTests.spec.ts b/src/tests/browser/filterTests.spec.ts new file mode 100644 index 0000000..27b9085 --- /dev/null +++ b/src/tests/browser/filterTests.spec.ts @@ -0,0 +1,60 @@ +import { describe, it, expect } from 'vitest'; +import { selectTestIds, listTestPaths } from '../../browser/filterTests'; +import type { PathHandler } from '../../browser/buildTestPath'; + +function makeHandlers(list: PathHandler[]): Map { + return new Map(list.map((h) => [h.id, h])); +} + +const handlers = makeHandlers([ + { id: 's1', name: 'Login flow', type: 'suite' }, + { id: 't1', name: 'shows error on bad password', type: 'test', parent: 's1' }, + { id: 't2', name: 'redirects on success', type: 'test', parent: 's1' }, + { id: 's2', name: 'Signup', type: 'suite' }, + { id: 't3', name: 'shows error on taken email', type: 'test', parent: 's2' }, + { id: 't4', name: 'root-level smoke test', type: 'test' }, +]); + +describe('selectTestIds', () => { + it('matches by test name substring (existing behavior)', () => { + expect(selectTestIds(handlers, ['redirects'])).toEqual(['t2']); + }); + + it('matches all tests under a describe by its name', () => { + expect(selectTestIds(handlers, ['login flow'])).toEqual(['t1', 't2']); + }); + + it('is case-insensitive', () => { + expect(selectTestIds(handlers, ['LOGIN FLOW'])).toEqual(['t1', 't2']); + }); + + it('supports cross-boundary filters spanning describe and test', () => { + expect(selectTestIds(handlers, ['login flow > shows error'])).toEqual(['t1']); + }); + + it('unions multiple filters without duplicating ids', () => { + expect(selectTestIds(handlers, ['login flow', 'shows error'])).toEqual([ + 't1', 't2', 't3', + ]); + }); + + it('returns an empty array when nothing matches', () => { + expect(selectTestIds(handlers, ['nonexistent'])).toEqual([]); + }); + + it('never returns suite ids', () => { + const ids = selectTestIds(handlers, ['login flow']); + expect(ids).not.toContain('s1'); + }); +}); + +describe('listTestPaths', () => { + it('lists full paths of all tests, skipping suites', () => { + expect(listTestPaths(handlers)).toEqual([ + 'Login flow > shows error on bad password', + 'Login flow > redirects on success', + 'Signup > shows error on taken email', + 'root-level smoke test', + ]); + }); +}); From 70193b8177384676e0d717a657f1f15d7b543d01 Mon Sep 17 00:00:00 2001 From: kevinccbsg Date: Sat, 4 Jul 2026 12:22:27 +0200 Subject: [PATCH 4/5] feat: match --test filters against full describe-path --- src/browser/createBrowserClient.ts | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/src/browser/createBrowserClient.ts b/src/browser/createBrowserClient.ts index d2980af..a5b9db8 100644 --- a/src/browser/createBrowserClient.ts +++ b/src/browser/createBrowserClient.ts @@ -1,6 +1,7 @@ import type { BrowserClient, BrowserClientOptions } from './types'; import { createFaviconManager } from './faviconManager'; import { createRunMonitor } from './runMonitor'; +import { selectTestIds, listTestPaths } from './filterTests'; declare global { interface Window { @@ -147,21 +148,10 @@ export function createBrowserClient(options?: BrowserClientOptions): BrowserClie let testIds: string[] | undefined; if (testNames && testNames.length > 0) { - const lowerNames = testNames.map(n => n.toLowerCase()); - const matched: string[] = []; - for (const [, handler] of handlers) { - if (handler.type === 'test') { - const lowerName = handler.name.toLowerCase(); - if (lowerNames.some(n => lowerName.includes(n))) { - matched.push(handler.id); - } - } - } + const matched = selectTestIds(handlers, testNames); if (matched.length === 0) { - const available = Array.from(handlers.values()) - .filter(h => h.type === 'test') - .map(h => h.name); + const available = listTestPaths(handlers); send({ type: 'run:start', testCount: 0 }); send({ type: 'error', From 2f5ebfb7b334b2b0cc95c9327d4dc03750370d79 Mon Sep 17 00:00:00 2001 From: kevinccbsg Date: Sat, 4 Jul 2026 12:24:09 +0200 Subject: [PATCH 5/5] docs: document describe-aware --test filtering --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 65efdaa..2cbb88b 100644 --- a/README.md +++ b/README.md @@ -165,8 +165,11 @@ twd-relay run # Different port (e.g. standalone relay) twd-relay run --port 9876 -# Filter tests by name (substring match, case-insensitive, repeatable) +# Filter tests by name or describe block (substring match against the full +# path "Describe > nested > test name", case-insensitive, repeatable) twd-relay run --test "should show error" +twd-relay run --test "Login flow" # runs every test in that describe +twd-relay run --test "login flow > shows error" # path match across describe/test twd-relay run --test "login" --test "signup" ``` @@ -177,9 +180,9 @@ twd-relay run --test "login" --test "signup" | `--path ` | WebSocket path | `/__twd/ws` | | `--timeout ` | Run timeout | `180000` | | `--max-test-duration ` | Per-test wall-clock abort threshold | `10000` | -| `--test ` | Filter tests by name substring (repeatable) | — | +| `--test ` | Filter tests by substring of the full describe-path (repeatable) | — | -When `--test` is used and no tests match, the CLI prints the available test names so you can correct the filter. +When `--test` is used and no tests match, the CLI prints the available tests as full describe-paths so you can correct the filter. ---