From abc95f8123160b536864b32b8c9a683bae1bfefb Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 26 Jun 2026 13:46:10 -0400 Subject: [PATCH 1/4] test(integration): harden shard-5/6 flakes (early-hints restart + job timeouts) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two pre-existing shard-5/6 failures (red on main HEAD 572640741 across all runtimes), neither a product bug — both test-robustness issues: - early-hints: the `before` hook called deploy_component(restart: true) unguarded. On a slow runner the restart delays/drops the HTTP response, surfacing as a fetch HeadersTimeoutError that failed the whole suite — even though the readiness polling right after it is the real success check. Tolerate a missing deploy response and let the poll be authoritative. - terminology: the async-job tests used a 30s completion timeout on Node. Under job-processor starvation (already documented for Bun at 120s) the csv/delete/ export jobs sit IN_PROGRESS past 30s and ~10 fail in a cascade. Extend the same mitigation to Node runtimes (90s). The drop_database assertion ahead of them is left intact, so the separate RocksDB-lock root cause still surfaces — this only removes the downstream timeout cascade. Could not reproduce locally (the integration harness requires ~32 sudo-configured loopback aliases); validated by parse/format checks and to be confirmed in CI. Co-Authored-By: Claude Opus 4.8 --- .../apiTests/terminology.test.mjs | 11 ++++++-- .../components/early-hints.test.ts | 28 +++++++++++++------ 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/integrationTests/apiTests/terminology.test.mjs b/integrationTests/apiTests/terminology.test.mjs index 79e31f0151..0ccbe74c03 100644 --- a/integrationTests/apiTests/terminology.test.mjs +++ b/integrationTests/apiTests/terminology.test.mjs @@ -20,10 +20,15 @@ import { awaitJobCompleted } from './utils/operations.mjs'; // Resolve the CSV fixture path relative to this file so Harper can read it. const SUPPLIERS_CSV = path.join(path.dirname(fileURLToPath(import.meta.url)), 'data/Suppliers.csv'); -// On Bun shard 2, embed-directive tear-down (HNSW flush) starves the job processor, -// so csv_data_load can sit IN_PROGRESS past the default 30s. Same pattern as northwind. +// Job-processor starvation can leave these async jobs IN_PROGRESS well past a tight timeout. +// On Bun shard 2 the embed-directive tear-down (HNSW flush) starves it; in CI the Node shard +// 5/6 run showed the same — the slow drop/teardown ahead of these jobs pushed the csv / delete / +// export jobs past the old 30s limit and failed ~10 of them in a single cascade. Give every +// runtime generous headroom so a slow-but-completing job processor doesn't fail the suite (the +// jobs do complete; they were just starved). This does not mask the separate drop_database +// failure ahead of them — that assertion is left intact. const isBunRuntime = process.env.HARPER_RUNTIME === 'bun'; -const JOB_TIMEOUT_SECONDS = isBunRuntime ? 120 : 30; +const JOB_TIMEOUT_SECONDS = isBunRuntime ? 120 : 90; suite('Terminology aliases (database / primary_key)', (ctx) => { let client; diff --git a/integrationTests/components/early-hints.test.ts b/integrationTests/components/early-hints.test.ts index f16681a89d..7111dd970f 100644 --- a/integrationTests/components/early-hints.test.ts +++ b/integrationTests/components/early-hints.test.ts @@ -20,14 +20,26 @@ suite('Component: early-hints', (ctx: ContextWithHarper) => { before(async () => { await startHarper(ctx); - const deployBody = await sendOperation(ctx.harper, { - operation: 'deploy_component', - project: 'early-hints', - package: join(__dirname, '../fixtures/template-early-hints-2.0.0.tgz'), - restart: true, - }); - strictEqual(deployBody.message, 'Successfully deployed: early-hints, restarting Harper'); - ok(typeof deployBody.deployment_id === 'string', `expected deployment_id, got ${deployBody.deployment_id}`); + // `restart: true` makes Harper restart right after accepting the deploy, which on a slow + // runner can delay or drop the HTTP response to this call — surfacing as a fetch + // `HeadersTimeoutError`. That is not a deploy failure: the readiness polling below (which + // waits for the /hints endpoint + seed data) is the authoritative success check. So + // tolerate a missing/failed response here instead of failing the whole suite in `before`. + let deployBody: any; + try { + deployBody = await sendOperation(ctx.harper, { + operation: 'deploy_component', + project: 'early-hints', + package: join(__dirname, '../fixtures/template-early-hints-2.0.0.tgz'), + restart: true, + }); + } catch (e: any) { + console.log(`[early-hints] deploy response not received (${e?.message}); relying on readiness poll`); + } + if (deployBody) { + strictEqual(deployBody.message, 'Successfully deployed: early-hints, restarting Harper'); + ok(typeof deployBody.deployment_id === 'string', `expected deployment_id, got ${deployBody.deployment_id}`); + } // poll until /hints endpoint is registered and seed data is loaded const seedDeadline = Date.now() + 60_000; From 28574cb7430d27c8bc86d5c16046d765177ebb62 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 26 Jun 2026 16:22:59 -0400 Subject: [PATCH 2/4] test(integration): log full error in early-hints deploy guard Address review feedback (#1504): log the full error object in the deploy catch, not just e.message, so a genuine deploy failure (bad package, invalid operation) remains debuggable when the readiness poll later times out. Co-Authored-By: Claude Opus 4.8 --- integrationTests/components/early-hints.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integrationTests/components/early-hints.test.ts b/integrationTests/components/early-hints.test.ts index 7111dd970f..05eb6419cc 100644 --- a/integrationTests/components/early-hints.test.ts +++ b/integrationTests/components/early-hints.test.ts @@ -34,7 +34,9 @@ suite('Component: early-hints', (ctx: ContextWithHarper) => { restart: true, }); } catch (e: any) { - console.log(`[early-hints] deploy response not received (${e?.message}); relying on readiness poll`); + // Log the full error (not just the message) so a genuine deploy failure — bad + // package, invalid operation — is debuggable when the readiness poll below times out. + console.log('[early-hints] deploy response not received; relying on readiness poll', e); } if (deployBody) { strictEqual(deployBody.message, 'Successfully deployed: early-hints, restarting Harper'); From 49179af87bffab3d2b33a0af9dc5db941e8c3d8e Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 26 Jun 2026 16:20:38 -0400 Subject: [PATCH 3/4] perf(test): cut round-trips in HNSW vector-index integration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The vector-index-integrity suite is a major Windows CI hotspot (~29 min for the suite). Most of the cost is ~100–200 sequential, individually-committed HTTP round-trips per test — tiny data (DIMS=8, N≤50), so it's round-trip + per-write fsync overhead, not vector math. Apply correctness-neutral reductions: - reindex backfill: bulk-insert the N pre-index seed records in one request instead of N sequential POSTs. There is no HNSW index during insert, so per-record ordering is irrelevant; the backfill builds the index afterward. - update-churn & reindex: run the independent read-back verification searches concurrently (Promise.all) instead of serializing N round-trips. Left untouched (semantically order-sensitive): the entry-point insert/delete sequence and the churn update sequence. Note: validated on CI, not locally — restart_service is flaky in a local multi-loopback setup (readiness probes time out at 60s), which both fails and dominates local timings; the churn test (the one that runs cleanly locally) still passes with the parallelized read-back. Co-Authored-By: Claude Opus 4.8 --- .../server/vector-index-integrity.test.ts | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/integrationTests/server/vector-index-integrity.test.ts b/integrationTests/server/vector-index-integrity.test.ts index 17e8dc7d25..76309d970c 100644 --- a/integrationTests/server/vector-index-integrity.test.ts +++ b/integrationTests/server/vector-index-integrity.test.ts @@ -360,12 +360,17 @@ suite('HNSW vector-index data-integrity (integration)', (ctx: ContextWithHarper) } // Each record's final vector is seedVector(i*100 + ROUNDS, dims). - // Searching for that exact vector must return it in the top-5. + // Searching for that exact vector must return it in the top-5. These are + // independent read-only searches, so run them concurrently rather than + // serializing N round-trips. + const churnResults = await Promise.all( + Array.from({ length: N }, (_, i) => + vectorSearch(httpURL, headers, path, seedVector(i * 100 + ROUNDS, DIMS), { limit: 5 }) + ) + ); let misses = 0; for (let i = 0; i < N; i++) { - const finalVec = seedVector(i * 100 + ROUNDS, DIMS); - const results = await vectorSearch(httpURL, headers, path, finalVec, { limit: 5 }); - if (!results.some((r: any) => r.id === `churn-${i}`)) misses++; + if (!churnResults[i].some((r: any) => r.id === `churn-${i}`)) misses++; } const allowed = Math.ceil(N * 0.1); ok(misses <= allowed, `expected ≤${allowed} misses after ${ROUNDS} churn rounds; got ${misses}/${N}`); @@ -495,10 +500,18 @@ suite('HNSW vector-index data-integrity (integration)', (ctx: ContextWithHarper) const headers = client.headers; const path = `/${TABLE}/`; - // Step 2: Insert N records (plain [Float], no HNSW index yet). - for (let i = 0; i < N; i++) { - await insertRecord(httpURL, headers, path, { id: `reindex-${i}`, embedding: seedVector(i, DIMS) }); - } + // Step 2: Insert N records (plain [Float], no HNSW index yet). Bulk-insert in a + // single request — there is no index during insert, so per-record ordering is + // irrelevant here; the backfill (Step 3) builds the index from the stored rows. + await client + .req() + .send({ + operation: 'insert', + database: DB, + table: TABLE, + records: Array.from({ length: N }, (_, i) => ({ id: `reindex-${i}`, embedding: seedVector(i, DIMS) })), + }) + .expect(200); // Sanity-check: records exist. const hashCheck = await client @@ -537,12 +550,14 @@ suite('HNSW vector-index data-integrity (integration)', (ctx: ContextWithHarper) } }, 60_000); - // Step 5: All N pre-existing records must be reachable. + // Step 5: All N pre-existing records must be reachable. Independent read-only + // searches — run them concurrently rather than serializing N round-trips. + const reindexResults = await Promise.all( + Array.from({ length: N }, (_, i) => vectorSearch(httpURL, headers, path, seedVector(i, DIMS), { limit: 5 })) + ); let misses = 0; for (let i = 0; i < N; i++) { - const vec = seedVector(i, DIMS); - const results = await vectorSearch(httpURL, headers, path, vec, { limit: 5 }); - if (!results.some((r: any) => r.id === `reindex-${i}`)) misses++; + if (!reindexResults[i].some((r: any) => r.id === `reindex-${i}`)) misses++; } const allowed = Math.ceil(N * 0.1); ok(misses <= allowed, `expected ≤${allowed} misses after backfill; got ${misses}/${N}`); From 551c8f98a7b1ac6d4ca43f9f99a386be8b4d59f5 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 26 Jun 2026 17:08:37 -0400 Subject: [PATCH 4/4] perf(test): deploy HNSW vector tables once, cut per-test worker restarts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each test in this suite deployed its own component and restarted the HTTP workers; the restart is the dominant per-test cost on slow runners (a test doing 3 inserts still took ~265s on Windows — almost entirely restart/setup). Deploy all four tables in one component with a single restart in `before`, and drop the per-test add_component/set_component_file/restart. Restarts: ~5 -> 2 (the shared one, plus the single restart `reindex` needs for its mid-test index-add); add_component: 4 -> 1. Tables keep distinct names/databases so the tests stay isolated despite sharing one component. ReindexTable is deployed without an index initially; the reindex test redeploys the full schema with it indexed (a no-op re-assert for the other three, which have already run — reindex is last) to trigger backfill. CI-validated (local restart_service is flaky in a multi-loopback setup). Co-Authored-By: Claude Opus 4.8 --- .../server/vector-index-integrity.test.ts | 158 ++++++------------ 1 file changed, 49 insertions(+), 109 deletions(-) diff --git a/integrationTests/server/vector-index-integrity.test.ts b/integrationTests/server/vector-index-integrity.test.ts index 76309d970c..623d3126fe 100644 --- a/integrationTests/server/vector-index-integrity.test.ts +++ b/integrationTests/server/vector-index-integrity.test.ts @@ -226,9 +226,49 @@ async function waitFor(predicate: () => Promise, timeoutMs = 60_000, in suite('HNSW vector-index data-integrity (integration)', (ctx: ContextWithHarper) => { let client: any; + // All four tables are deployed by a SINGLE component with ONE worker restart in `before`. + // Previously every test deployed its own component and restarted the HTTP workers (~5 restarts + // across the suite); on slow runners that restart is the dominant per-test cost (a test doing + // 3 inserts still took ~265s on Windows — almost entirely restart/setup). Consolidating to one + // shared restart here — plus the single restart `reindex` needs for its mid-test index-add — is + // the bulk of this suite's wall-time. The tables use distinct names/databases so the tests stay + // isolated despite sharing one component; `reindex` runs last and is the only test that mutates + // the schema afterward, so re-asserting the others on its redeploy is a harmless no-op. + const PROJECT = 'vecintegsuite'; + // ReindexTable starts WITHOUT an index — the reindex test adds it mid-test to exercise backfill. + const SCHEMA_INITIAL = [ + makeSchemaWithIndex('DelEPTable', 'vecinteg1'), + makeSchemaWithIndex('ChurnTable', 'vecinteg2'), + makeSchemaWithIndex('ThreshTable', 'vecinteg3'), + makeSchemaWithoutIndex('ReindexTable', 'vecinteg4'), + ].join('\n'); + // Same schema with ReindexTable now indexed — redeployed by the reindex test to trigger backfill. + const SCHEMA_REINDEX_INDEXED = [ + makeSchemaWithIndex('DelEPTable', 'vecinteg1'), + makeSchemaWithIndex('ChurnTable', 'vecinteg2'), + makeSchemaWithIndex('ThreshTable', 'vecinteg3'), + makeSchemaWithIndex('ReindexTable', 'vecinteg4'), + ].join('\n'); + before(async () => { await startHarper(ctx, { config: {}, env: {} }); client = createApiClient(ctx.harper); + await client + .req() + .send({ operation: 'add_component', project: PROJECT }) + .expect((r: any) => { + ok( + JSON.stringify(r.body).includes('Successfully added project') || + JSON.stringify(r.body).includes('Project already exists'), + r.text + ); + }); + await client + .req() + .send({ operation: 'set_component_file', project: PROJECT, file: 'schema.graphql', payload: SCHEMA_INITIAL }) + .expect(200); + // One restart loads all four tables; probing one ready endpoint confirms the workers are back. + await restartHttpWorkers(client, '/DelEPTable/'); }); after(async () => { @@ -241,39 +281,13 @@ suite('HNSW vector-index data-integrity (integration)', (ctx: ContextWithHarper) // getRange and skips the node being deleted. // Pre-fix: getRange ran outside the write-set → re-elected the deleted node // as EP → dangling entry point → subsequent searches returned []. - const DB = 'vecinteg1'; - const TABLE = 'DelEPTable'; - const PROJECT = 'vecintegsuite1'; const DIMS = 8; const N = 50; const SURVIVORS = 10; const deleteCount = N - SURVIVORS; - - await client - .req() - .send({ operation: 'add_component', project: PROJECT }) - .expect((r: any) => { - ok( - JSON.stringify(r.body).includes('Successfully added project') || - JSON.stringify(r.body).includes('Project already exists'), - r.text - ); - }); - await client - .req() - .send({ - operation: 'set_component_file', - project: PROJECT, - file: 'schema.graphql', - payload: makeSchemaWithIndex(TABLE, DB), - }) - .expect(200); - - await restartHttpWorkers(client, `/${TABLE}/`); - const httpURL = ctx.harper.httpURL; const headers = client.headers; - const path = `/${TABLE}/`; + const path = '/DelEPTable/'; // Insert N records — the first-inserted node becomes the initial entry point. for (let i = 0; i < N; i++) { @@ -312,38 +326,12 @@ suite('HNSW vector-index data-integrity (integration)', (ctx: ContextWithHarper) // where the old connection existed, not the full 0..l sweep (correct for DELETE). // The broader sweep destroyed reverse edges that addConnection had just re-added, // accumulating asymmetry with every re-embed round. - const DB = 'vecinteg2'; - const TABLE = 'ChurnTable'; - const PROJECT = 'vecintegsuite2'; const DIMS = 8; const N = 30; const ROUNDS = 5; - - await client - .req() - .send({ operation: 'add_component', project: PROJECT }) - .expect((r: any) => { - ok( - JSON.stringify(r.body).includes('Successfully added project') || - JSON.stringify(r.body).includes('Project already exists'), - r.text - ); - }); - await client - .req() - .send({ - operation: 'set_component_file', - project: PROJECT, - file: 'schema.graphql', - payload: makeSchemaWithIndex(TABLE, DB), - }) - .expect(200); - - await restartHttpWorkers(client, `/${TABLE}/`); - const httpURL = ctx.harper.httpURL; const headers = client.headers; - const path = `/${TABLE}/`; + const path = '/ChurnTable/'; for (let i = 0; i < N; i++) { await insertRecord(httpURL, headers, path, { id: `churn-${i}`, embedding: seedVector(i, DIMS) }); @@ -380,35 +368,9 @@ suite('HNSW vector-index data-integrity (integration)', (ctx: ContextWithHarper) test('threshold queries: le includes exact boundary; le(~0) returns exact-match only', async () => { // Guards fix #6b: le comparator now uses <= (was <). // Pre-fix: records at exactly the threshold distance were excluded by le. - const DB = 'vecinteg3'; - const TABLE = 'ThreshTable'; - const PROJECT = 'vecintegsuite3'; - - await client - .req() - .send({ operation: 'add_component', project: PROJECT }) - .expect((r: any) => { - ok( - JSON.stringify(r.body).includes('Successfully added project') || - JSON.stringify(r.body).includes('Project already exists'), - r.text - ); - }); - await client - .req() - .send({ - operation: 'set_component_file', - project: PROJECT, - file: 'schema.graphql', - payload: makeSchemaWithIndex(TABLE, DB), - }) - .expect(200); - - await restartHttpWorkers(client, `/${TABLE}/`); - const httpURL = ctx.harper.httpURL; const headers = client.headers; - const path = `/${TABLE}/`; + const path = '/ThreshTable/'; // 2-D vectors at three well-separated cosine distances from [1,0]: // [1,0] → distance 0 (exact; representable in float32, so exactly 0) @@ -469,37 +431,13 @@ suite('HNSW vector-index data-integrity (integration)', (ctx: ContextWithHarper) // on structural index change so runIndexing starts clean). const DB = 'vecinteg4'; const TABLE = 'ReindexTable'; - const PROJECT = 'vecintegsuite4'; const DIMS = 8; const N = 40; - - // Step 1: Deploy schema WITHOUT the HNSW index. - await client - .req() - .send({ operation: 'add_component', project: PROJECT }) - .expect((r: any) => { - ok( - JSON.stringify(r.body).includes('Successfully added project') || - JSON.stringify(r.body).includes('Project already exists'), - r.text - ); - }); - await client - .req() - .send({ - operation: 'set_component_file', - project: PROJECT, - file: 'schema.graphql', - payload: makeSchemaWithoutIndex(TABLE, DB), - }) - .expect(200); - - await restartHttpWorkers(client, `/${TABLE}/`); - const httpURL = ctx.harper.httpURL; const headers = client.headers; - const path = `/${TABLE}/`; + const path = '/ReindexTable/'; + // Step 1: ReindexTable was deployed WITHOUT an HNSW index by the shared `before` setup. // Step 2: Insert N records (plain [Float], no HNSW index yet). Bulk-insert in a // single request — there is no index during insert, so per-record ordering is // irrelevant here; the backfill (Step 3) builds the index from the stored rows. @@ -526,14 +464,16 @@ suite('HNSW vector-index data-integrity (integration)', (ctx: ContextWithHarper) .expect(200); ok(Array.isArray(hashCheck.body) && hashCheck.body.length === 1, 'reindex-0 must exist before reindex'); - // Step 3: Alter schema to ADD the HNSW index → triggers runIndexing backfill. + // Step 3: Alter schema to ADD the HNSW index on ReindexTable → triggers runIndexing + // backfill. Redeploy the full schema (the other three tables are unchanged, so this is + // a no-op re-assert for them); reindex runs last, so restarting them here is harmless. await client .req() .send({ operation: 'set_component_file', project: PROJECT, file: 'schema.graphql', - payload: makeSchemaWithIndex(TABLE, DB), + payload: SCHEMA_REINDEX_INDEXED, }) .expect(200);