Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -879,16 +879,21 @@ jobs:
- name: UI typecheck
if: ${{ !cancelled() && (github.event_name == 'push' || needs.changes.outputs.ui == 'true') }}
run: npx turbo run typecheck --filter=@loopover/ui --filter=@loopover/ui-miner
# Split into 3 independent steps (was one `&&`-chained step) -- turbo.json deliberately has no `test`
# Split into 4 independent steps (was one `&&`-chained step) -- turbo.json deliberately has no `test`
# task for any package (per the plan doc: test orchestration stays out of Turborepo entirely, so
# Codecov's codecov/patch keeps one centralized source of truth), so this can't use the same
# union-`--filter` fix already applied to "UI lint"/"UI typecheck" two steps above. Plain `&&` meant a
# failing @loopover/ui test silently prevented @loopover/ui-miner's tests from ever running in that
# invocation -- same class of failure-masking bug those two steps'
# own comment already describes, just still present here. `!cancelled()` on all 3 (not `always()`)
# own comment already describes, just still present here. `!cancelled()` on all 4 (not `always()`)
# matches the pattern already used by "Save Turborepo cache"/"Save TypeScript incremental build
# cache" elsewhere in this file: still run after an earlier step's failure, just not after the job
# was cancelled outright. The job as a whole still fails if any of the three fails.
# was cancelled outright. The job as a whole still fails if any of the four fails.
# #10049: ui-kit's package-local vitest suite was declared and documented as "this suite runs" but
# never invoked from CI or ui:test — same silently-partial shape as #9860's typecheck gap.
- name: UI tests (ui-kit)
if: ${{ !cancelled() && (github.event_name == 'push' || needs.changes.outputs.ui == 'true') }}
run: npm --workspace @loopover/ui-kit run test
- name: UI tests (ui)
if: ${{ !cancelled() && (github.event_name == 'push' || needs.changes.outputs.ui == 'true') }}
run: npm --workspace @loopover/ui run test
Expand Down
5 changes: 3 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
"db:migrations:immutable:check": "tsx scripts/check-released-migrations-immutable.ts",
"turbo-inputs:check": "tsx scripts/check-turbo-typecheck-inputs.ts",
"typecheck-coverage:check": "tsx scripts/check-typecheck-coverage.ts",
"test-wiring:check": "tsx scripts/check-test-coverage-wiring.ts",
"fixture-clock-races:check": "tsx scripts/check-fixture-clock-races.ts",
"workspace-dep-ranges:check": "tsx scripts/check-workspace-dep-ranges.ts",
"db:schema-drift:check": "tsx scripts/check-schema-drift.ts",
Expand All @@ -73,7 +74,7 @@
"ui:lint": "npm run ui:kit:build && npm --workspace @loopover/ui-kit run format:check && npm --workspace @loopover/ui run format:check && npm --workspace @loopover/ui run lint && npm --workspace @loopover/ui-miner run format:check && npm --workspace @loopover/ui-miner run lint",
"ui:typecheck": "npm run ui:kit:build && npm --workspace @loopover/ui-kit run typecheck && npm --workspace @loopover/ui run typecheck && npm --workspace @loopover/ui-miner run typecheck",
"preui:test": "npm run check-node-version",
"ui:test": "npm run ui:kit:build && npm --workspace @loopover/ui run test && npm --workspace @loopover/ui-miner run test",
"ui:test": "npm run ui:kit:build && npm --workspace @loopover/ui-kit run test && npm --workspace @loopover/ui run test && npm --workspace @loopover/ui-miner run test",
"ui:openapi": "tsx scripts/write-ui-openapi.ts",
"ui:openapi:check": "tsx scripts/write-ui-openapi.ts --check",
"control-plane:contract": "tsx scripts/gen-control-plane-contract.ts",
Expand Down Expand Up @@ -148,7 +149,7 @@
"test:smoke:browser:install": "playwright install chromium",
"test:smoke:browser": "node --experimental-strip-types scripts/smoke-ui-browser.ts",
"pretest:ci": "npm run check-node-version",
"test:ci": "git diff --check && npm run actionlint && npm run lint:composite-actions && npm run db:migrations:check && npm run db:migrations:immutable:check && npm run turbo-inputs:check && npm run workspace-dep-ranges:check && npm run db:schema-drift:check && npm run selfhost:env-reference:check && npm run miner:env-reference:check && npm run selfhost:validate-observability && npm run cf-typegen:check && npm run build --workspace @loopover/engine && npm run build --workspace @loopover/discovery-index && npm run build:mcp && npm run build:miner && npm run build --workspace @loopover/ui-kit && npm run typecheck && npm run test:coverage && npm run test:engine-parity && npm run test:live-gate-parity && npm run test:driver-parity && npm run validate:mcp && npm run test --workspace @loopover/engine && npm run test:workers && npm run test:mcp-pack && npm run test:contract-pack && npm run test:miner-pack && npm run test:engine-pack && npm run test:ui-kit-pack && npm run test:miner-deployment-docs-audit && npm run rees:test && npm run ui:openapi:check && npm run control-plane:contract:check && npm run control-plane:openapi:check && npm run ui:version-audit && npm run docs:drift-check && npm run coverage-boltons:check && npm run import-specifiers:check && npm run ui-derived-types:check && npm run server-manifest:check && npm run dead-source-files:check && npm run dead-exports:check && npm run publishable-deps:check && npm run fixture-clock-races:check && npm run typecheck-coverage:check && npm run checkers-wired:check && npm run focus-manifest-fields:check && npm run release-commit-parsing:check && npm run releasable-commit-types:check && npm run regate-sort-key:check && npm run maintainer-associations:check && npm run command-redelivery-guards:check && npm run dispatch-gate-reasons:check && npm run validate:no-hand-written-js && npm run replay-runner-manifest:check && npm run coco-dev-versions:check && npm run branding-drift:check && npm run manifest:drift-check && npm run engine-parity:drift-check && npm run engines-nvmrc:check && npm run release-manifest:sync:check && npm run release-linked-versions:check && npm run command-reference:check && npm run mcp:tool-reference:check && npm run mcp:client-config:check && npm run contract:api-schemas:check && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build",
"test:ci": "git diff --check && npm run actionlint && npm run lint:composite-actions && npm run db:migrations:check && npm run db:migrations:immutable:check && npm run turbo-inputs:check && npm run workspace-dep-ranges:check && npm run db:schema-drift:check && npm run selfhost:env-reference:check && npm run miner:env-reference:check && npm run selfhost:validate-observability && npm run cf-typegen:check && npm run build --workspace @loopover/engine && npm run build --workspace @loopover/discovery-index && npm run build:mcp && npm run build:miner && npm run build --workspace @loopover/ui-kit && npm run typecheck && npm run test:coverage && npm run test:engine-parity && npm run test:live-gate-parity && npm run test:driver-parity && npm run validate:mcp && npm run test --workspace @loopover/engine && npm run test:workers && npm run test:mcp-pack && npm run test:contract-pack && npm run test:miner-pack && npm run test:engine-pack && npm run test:ui-kit-pack && npm run test:miner-deployment-docs-audit && npm run rees:test && npm run ui:openapi:check && npm run control-plane:contract:check && npm run control-plane:openapi:check && npm run ui:version-audit && npm run docs:drift-check && npm run coverage-boltons:check && npm run import-specifiers:check && npm run ui-derived-types:check && npm run server-manifest:check && npm run dead-source-files:check && npm run dead-exports:check && npm run publishable-deps:check && npm run fixture-clock-races:check && npm run typecheck-coverage:check && npm run test-wiring:check && npm run checkers-wired:check && npm run focus-manifest-fields:check && npm run release-commit-parsing:check && npm run releasable-commit-types:check && npm run regate-sort-key:check && npm run maintainer-associations:check && npm run command-redelivery-guards:check && npm run dispatch-gate-reasons:check && npm run validate:no-hand-written-js && npm run replay-runner-manifest:check && npm run coco-dev-versions:check && npm run branding-drift:check && npm run manifest:drift-check && npm run engine-parity:drift-check && npm run engines-nvmrc:check && npm run release-manifest:sync:check && npm run release-linked-versions:check && npm run command-reference:check && npm run mcp:tool-reference:check && npm run mcp:client-config:check && npm run contract:api-schemas:check && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build",
"test:release": "npm run test:ci && npm run changelog:check",
"test:release:mcp": "npm run test:ci",
"test:watch": "vitest",
Expand Down
126 changes: 126 additions & 0 deletions scripts/check-test-coverage-wiring.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
#!/usr/bin/env node
// `npm run test:ci` must actually run every workspace suite that declares a `test` script (#10049).
//
// THE INCIDENT. packages/loopover-ui-kit ships a 12-file vitest suite whose package.json declares
// `"test": "vitest run"`, and its own vitest.config.ts documents the acceptance signal as "this suite
// runs". Nothing invoked it: root `ui:test` built ui-kit but only ran @loopover/ui and @loopover/ui-miner;
// ci.yml had "UI tests (ui)" / "UI tests (ui-miner)" and no ui-kit step; turbo.json deliberately has no
// `test` task. The tests existed, could not fail, and were trusted precisely because everything around
// them was green -- the same SILENTLY PARTIAL shape #9860 named for typecheck.
//
// WHAT THIS COMPUTES. Every workspace under apps/* / packages/* that declares its own `test` script, and
// whether the root `test:ci` script reaches it -- following `npm run <script>` references transitively,
// so a workspace covered through an intermediate script (`ui:test` -> `npm --workspace @loopover/ui-kit
// run test`) counts as covered. Anything declaring a test nobody runs is reported.
//
// Mirror of scripts/check-typecheck-coverage.ts, for the `test` script / `test:ci` entry.
import { readFileSync, readdirSync } from "node:fs";
import { join } from "node:path";
import { fileURLToPath, URL } from "node:url";

export type TestWiringGap = { workspace: string; script: string };

export type WorkspaceWithTest = { name: string; dir: string };

/** Every `npm run <name>` this script body invokes (the root package's own scripts). */
function referencedRootScripts(body: string): string[] {
// `npm run x`, `npm run x --silent`, `npm --silent run x` -- all forms used in this package.json.
return [...body.matchAll(/npm\s+(?:--\S+\s+)*run\s+([\w:.-]+)/g)].map((match) => match[1]).filter((name): name is string => Boolean(name));
}

/** Every workspace whose OWN `test` this script body invokes directly. */
function referencedWorkspaces(body: string): string[] {
// `npm --workspace @scope/name run test` and `npm run test --workspace @scope/name`.
const names = [
...body.matchAll(/npm\s+--workspace[= ]\s*(\S+)\s+run\s+([\w:.-]+)/g),
...body.matchAll(/npm\s+run\s+([\w:.-]+)\s+--workspace[= ]\s*(\S+)/g),
];
const out: string[] = [];
for (const match of names) {
// The two patterns capture (workspace, script) and (script, workspace) respectively; the workspace is
// whichever capture looks like a package name.
const [a, b] = [match[1], match[2]];
const workspace = a?.startsWith("@") || a?.includes("/") ? a : b;
const script = workspace === a ? b : a;
if (workspace && script === "test") out.push(workspace);
}
return out;
}

/**
* PURE: workspaces that declare a `test` script the root `test:ci` never reaches.
*
* `scripts` is the root package's script map; `workspacesWithTest` is every workspace that declares one
* (package name + directory). Reachability follows `npm run` references transitively from `entry`.
*/
export function findTestWiringGaps(
scripts: Readonly<Record<string, string>>,
workspacesWithTest: readonly WorkspaceWithTest[],
entry = "test:ci",
): TestWiringGap[] {
const covered = new Set<string>();
const seen = new Set<string>();
const queue: string[] = [entry];
while (queue.length > 0) {
const name = queue.shift();
if (!name || seen.has(name)) continue;
seen.add(name);
const body = scripts[name];
if (body === undefined) continue;
for (const workspace of referencedWorkspaces(body)) covered.add(workspace);
queue.push(...referencedRootScripts(body));
}
return workspacesWithTest
.filter(
(workspace) =>
!covered.has(workspace.name) &&
!covered.has(workspace.name.replace(/^@[\w-]+\//, "")) &&
!covered.has(workspace.dir),
)
.map((workspace) => ({ workspace: workspace.name, script: "test" }));
}

/** Workspace package names (and their directories) that declare their own `test` script. */
export function workspacesDeclaringTest(root: string): WorkspaceWithTest[] {
const out: WorkspaceWithTest[] = [];
for (const group of ["apps", "packages"]) {
let dirs: string[];
try {
dirs = readdirSync(join(root, group), { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name);
} catch {
continue;
}
for (const dir of dirs) {
try {
const manifest = JSON.parse(readFileSync(join(root, group, dir, "package.json"), "utf8")) as { name?: string; scripts?: Record<string, string> };
if (manifest.name && manifest.scripts?.test) out.push({ name: manifest.name, dir: `${group}/${dir}` });
} catch {
// not a workspace package
}
}
}
return out;
}

function main(): void {
const root = join(fileURLToPath(new URL(".", import.meta.url)), "..");
const rootManifest = JSON.parse(readFileSync(join(root, "package.json"), "utf8")) as { scripts?: Record<string, string> };
const declared = workspacesDeclaringTest(root);
const gaps = findTestWiringGaps(rootManifest.scripts ?? {}, declared);

if (gaps.length > 0) {
console.error("`npm run test:ci` does not reach every workspace that declares a test script:\n");
for (const gap of gaps) console.error(` ${gap.workspace} (declares "${gap.script}", never invoked)`);
console.error(
"\n A test suite that exists but never runs is worse than no suite: it is trusted BECAUSE everything\n" +
" around it is green. #10049's ui-kit suite sat that way -- declared, documented as \"this suite\n" +
" runs\", never invoked from ui:test or CI.\n\n" +
" Fix: chain the workspace into `test:ci` (directly, or through a script it already calls such as\n" +
" `ui:test`), so declaring a test means CI actually runs it.",
);
process.exit(1);
}
console.log(`test-wiring: OK — all ${declared.length} workspace test script(s) are reachable from \`npm run test:ci\`.`);
}

if (process.argv[1]?.endsWith("check-test-coverage-wiring.ts")) main();
74 changes: 74 additions & 0 deletions test/unit/check-test-coverage-wiring-script.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
// The test-coverage-wiring checker must catch a workspace the root test:ci chain never reaches (#10049).
//
// The failure it guards is specific: a workspace declares `"test"` and nothing in test:ci invokes it, so
// the suite can never fail CI. That is worse than having no suite, because green results around it are
// trusted. Mirror of check-typecheck-coverage-script.test.ts for the test / test:ci pair.
import { describe, expect, it } from "vitest";

import { findTestWiringGaps } from "../../scripts/check-test-coverage-wiring";

describe("findTestWiringGaps (#10049)", () => {
it("reports a workspace reached by nobody — the ui-kit shape", () => {
const scripts = {
"test:ci": "npm run test --workspace @loopover/engine && npm run ui:test",
"ui:test": "npm run ui:kit:build && npm --workspace @loopover/ui run test && npm --workspace @loopover/ui-miner run test",
"ui:kit:build": "npm --workspace @loopover/ui-kit run build",
};
expect(
findTestWiringGaps(scripts, [
{ name: "@loopover/ui", dir: "apps/loopover-ui" },
{ name: "@loopover/ui-miner", dir: "apps/loopover-miner-ui" },
{ name: "@loopover/engine", dir: "packages/loopover-engine" },
{ name: "@loopover/ui-kit", dir: "packages/loopover-ui-kit" },
]),
).toEqual([{ workspace: "@loopover/ui-kit", script: "test" }]);
});

it("counts a workspace reached directly as covered", () => {
const scripts = {
"test:ci": "npm run test --workspace @loopover/engine",
};
expect(findTestWiringGaps(scripts, [{ name: "@loopover/engine", dir: "packages/loopover-engine" }])).toEqual([]);
});

it("counts a workspace reached THROUGH an intermediate script as covered", () => {
const scripts = {
"test:ci": "npm run ui:test",
"ui:test": "npm run ui:kit:build && npm --workspace @loopover/ui-kit run test",
"ui:kit:build": "npm --workspace @loopover/ui-kit run build",
};
expect(findTestWiringGaps(scripts, [{ name: "@loopover/ui-kit", dir: "packages/loopover-ui-kit" }])).toEqual([]);
});

it("does NOT count a workspace whose BUILD is run but whose test is not", () => {
const scripts = {
"test:ci": "npm run ui:kit:build",
"ui:kit:build": "npm --workspace @loopover/ui-kit run build",
};
expect(findTestWiringGaps(scripts, [{ name: "@loopover/ui-kit", dir: "packages/loopover-ui-kit" }])).toEqual([
{ workspace: "@loopover/ui-kit", script: "test" },
]);
});

it("handles the reversed flag order, since both spellings appear in this package.json", () => {
const scripts = { "test:ci": "npm run test --workspace @loopover/ui" };
expect(findTestWiringGaps(scripts, [{ name: "@loopover/ui", dir: "apps/loopover-ui" }])).toEqual([]);
});

it("terminates on a cyclic script graph instead of looping forever", () => {
const scripts = { "test:ci": "npm run a", a: "npm run b", b: "npm run a" };
expect(findTestWiringGaps(scripts, [{ name: "@loopover/ui", dir: "apps/loopover-ui" }])).toEqual([
{ workspace: "@loopover/ui", script: "test" },
]);
});

it("tolerates a missing entry script rather than throwing", () => {
expect(findTestWiringGaps({}, [{ name: "@loopover/ui", dir: "apps/loopover-ui" }])).toEqual([
{ workspace: "@loopover/ui", script: "test" },
]);
});

it("reports nothing when no workspace declares a test at all", () => {
expect(findTestWiringGaps({ "test:ci": "echo ok" }, [])).toEqual([]);
});
});
Loading