diff --git a/integrationTests/database/large-blob-streaming.test.ts b/integrationTests/database/large-blob-streaming.test.ts new file mode 100644 index 0000000000..8c03a4c81d --- /dev/null +++ b/integrationTests/database/large-blob-streaming.test.ts @@ -0,0 +1,406 @@ +/** + * Large-blob storage limits + streaming upload/download correctness. + * + * BACKGROUND (prior waves): + * - Blobs round-trip byte-exact via CBOR/msgpack PUT. + * - Single values to 128MB round-trip. + * - Large result sets stream at constant worker memory (heapUsed via + * system_information{threads} is the streaming-vs-buffering signal). + * - 1MB blobs round-trip byte-exact, no aliasing — but using payload.bytes() + * (full buffer), so it cannot speak to large-blob memory. + * + * THIS TEST pushes the blob path past 128MB and asks the streaming question for + * a SINGLE huge blob on BOTH directions: + * - Store a genuinely large blob (64 → 256MB, then 512MB if memory stays + * bounded) and verify byte-exact round-trip via a STREAMING hash (the server + * reads it back through blob.stream() chunk-by-chunk; the wire sub-attribute + * GET is also hashed streaming). No full copy is ever held in any JS heap. + * - Measure worker heapUsed + rss DURING upload and DURING download. Bounded + * (flat as size grows) => true streaming. Linear in size => full buffering => + * OOM lever. + * - Overwrite + delete a large blob — does blob-path disk space get reclaimed? + * - The blob sub-attribute GET path (GET /Big//payload) at large size. + * + * HOW WE STAY SAFE (don't OOM the box): + * - Bytes are generated in 4MB chunks server-side (async generator -> createBlob + * streaming source) and read back in stream chunks — the worker never holds a + * whole blob. The client only ever sends/receives small JSON, except the + * wire sub-attribute GET which is consumed as a streaming running-hash (never + * buffered in the test process either). + * - Sweep starts at 64/128/256MB. We only attempt 512MB if peak worker rss + * stayed well under a guard threshold at 256MB (true streaming confirmed). + */ +import { suite, test, before, after } from 'node:test'; +import { ok } from 'node:assert/strict'; +import { resolve, join } from 'node:path'; +import { mkdtemp } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { createHash } from 'node:crypto'; +import { setTimeout as sleep } from 'node:timers/promises'; +import http from 'node:http'; +import https from 'node:https'; +import request from 'supertest'; +import { setupHarperWithFixture, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing'; +// @ts-expect-error utils/client.mjs has no type declarations; runtime resolves fine +import { createApiClient } from '../apiTests/utils/client.mjs'; + +const FIXTURE_PATH = resolve(import.meta.dirname, 'large-blob-streaming'); +const skipSuite = process.platform === 'win32'; +const MB = 1024 * 1024; +const ENGINE = process.env.HARPER_STORAGE_ENGINE ?? 'rocksdb(default)'; + +// rss guard: if a 256MB store/verify pushes peak worker rss above this much +// ABOVE its idle baseline, we treat that as "not bounded" and DO NOT escalate +// to 512MB (avoid OOMing the box). 256MB blob + true streaming should add only +// tens of MB; full buffering would add ~256MB+. +const RSS_ESCALATE_GUARD_MB = 350; + +type Client = ReturnType; + +interface WorkerMem { + maxHeapMB: number; + maxRssMB: number; +} + +async function sampleWorkerMem(client: Client): Promise { + const r = await client.req().send({ operation: 'system_information', attributes: ['threads'] }).timeout(10_000); + const threads: any[] = Array.isArray(r.body?.threads) ? r.body.threads : []; + let maxHeap = 0; + let maxRss = 0; + for (const t of threads) { + maxHeap = Math.max(maxHeap, Number(t.heapUsed) || 0); + // rss field naming varies; try common keys. + maxRss = Math.max(maxRss, Number(t.rss) || Number(t.residentSetSize) || 0); + } + return { maxHeapMB: maxHeap / MB, maxRssMB: maxRss / MB }; +} + +/** Run `fn` while polling worker mem ~every 50ms; return fn result + peak heap/rss. */ +async function withMemWatch(client: Client, fn: () => Promise): Promise<{ result: T; peakHeapMB: number; peakRssMB: number }> { + let peakHeap = 0; + let peakRss = 0; + let watching = true; + const watcher = (async () => { + while (watching) { + try { + const m = await sampleWorkerMem(client); + peakHeap = Math.max(peakHeap, m.maxHeapMB); + peakRss = Math.max(peakRss, m.maxRssMB); + } catch { + /* ignore sampling blips */ + } + await sleep(50); + } + })(); + const result = await fn(); + try { + const m = await sampleWorkerMem(client); + peakHeap = Math.max(peakHeap, m.maxHeapMB); + peakRss = Math.max(peakRss, m.maxRssMB); + } catch { + /* ignore */ + } + watching = false; + await watcher; + return { result, peakHeapMB: peakHeap, peakRssMB: peakRss }; +} + +/** Streaming sub-attribute GET: GET /Big//payload, hash chunks without buffering. */ +function streamHashGet( + baseURL: string, + path: string, + authHeader: string +): Promise<{ status: number; transferEncoding?: string; contentLength?: string; bytes: number; sha: string; ttfbMs: number; totalMs: number }> { + const url = new URL(path, baseURL); + const lib = url.protocol === 'https:' ? https : http; + const start = Date.now(); + const h = createHash('sha256'); + return new Promise((resolvePromise, reject) => { + const req = lib.request( + url, + { + method: 'GET', + headers: { Authorization: authHeader, Connection: 'close' }, + ...(url.protocol === 'https:' ? { rejectUnauthorized: false } : {}), + }, + (res) => { + let bytes = 0; + let ttfb = -1; + res.on('data', (d: Buffer) => { + if (ttfb < 0) ttfb = Date.now() - start; + bytes += d.length; + h.update(d); + }); + res.on('end', () => + resolvePromise({ + status: res.statusCode ?? 0, + transferEncoding: res.headers['transfer-encoding'], + contentLength: res.headers['content-length'], + bytes, + sha: h.digest('hex'), + ttfbMs: ttfb < 0 ? Date.now() - start : ttfb, + totalMs: Date.now() - start, + }) + ); + } + ); + req.on('error', reject); + req.setTimeout(300_000, () => req.destroy(new Error('streamHashGet timeout'))); + req.end(); + }); +} + +async function dirBytes(dir: string): Promise { + const fs = await import('node:fs/promises'); + let total = 0; + async function walk(d: string) { + let entries: any[]; + try { + entries = await fs.readdir(d, { withFileTypes: true }); + } catch { + return; + } + for (const e of entries) { + const p = join(d, e.name); + if (e.isDirectory()) await walk(p); + else { + try { + total += (await fs.stat(p)).size; + } catch { + /* race */ + } + } + } + } + await walk(dir); + return total; +} + +suite(`large-blob streaming (engine=${ENGINE})`, { skip: skipSuite }, (ctx: ContextWithHarper) => { + let client: Client; + let httpURL: string; + let authHeader: string; + let blobPath: string; + let idleRssMB = 0; + const findings: string[] = []; + + async function op(body: Record, timeoutMs = 300_000) { + return request(httpURL).post('/BlobOps/').set(client.headers).send(body).timeout(timeoutMs); + } + + before(async () => { + blobPath = await mkdtemp(join(tmpdir(), 'large-blob-streaming-')); + await setupHarperWithFixture(ctx, FIXTURE_PATH, { + config: { storage: { blobPaths: [blobPath] } }, + env: {}, + }); + client = createApiClient(ctx.harper); + httpURL = ctx.harper.httpURL; + authHeader = client.headers.Authorization; + + const deadline = Date.now() + 60_000; + while (Date.now() < deadline) { + try { + const probe = await client.reqRest('/Big/').timeout(3_000); + if (probe.status !== 404) break; + } catch { + /* not ready */ + } + await sleep(250); + } + await sleep(1500); + const m = await sampleWorkerMem(client); + idleRssMB = m.maxRssMB; + findings.push(`baseline: idle worker maxHeap=${m.maxHeapMB.toFixed(1)}MB maxRss=${m.maxRssMB.toFixed(1)}MB (engine=${ENGINE})`); + }); + + after(async () => { + await teardownHarper(ctx); + console.log(`\n[large-blob-streaming] SUMMARY (engine=${ENGINE})`); + for (const f of findings) console.log(' ' + f); + }); + + // Sweep sizes; characterize round-trip + memory trend on BOTH directions. + test('Q1: size sweep — byte-exact round-trip + bounded upload/download memory', async () => { + const baseSizes = [64, 128, 256]; + const results: { mb: number; klass: string; upHeapMB: number; upRssMB: number; dlHeapMB: number; dlRssMB: number; match: boolean }[] = []; + + const runSize = async (mb: number) => { + const size = mb * MB; + const key = `blob-${mb}`; + // ---- UPLOAD ---- + let klass = 'round-trip-exact'; + let match = false; + let upHeapMB = 0; + let upRssMB = 0; + let dlHeapMB = 0; + let dlRssMB = 0; + const up = await withMemWatch(client, () => op({ action: 'store', key, seed: key, size })); + upHeapMB = up.peakHeapMB; + upRssMB = up.peakRssMB; + if (up.result.status !== 200 || up.result.body?.ok !== true) { + klass = up.result.status === 413 ? 'clean-413' : up.result.status >= 500 ? `${up.result.status}-cliff` : `status-${up.result.status}`; + findings.push(`Q1 ${mb}MB UPLOAD ${klass}: status=${up.result.status} body=${JSON.stringify(up.result.body).slice(0, 200)}`); + results.push({ mb, klass, upHeapMB, upRssMB, dlHeapMB, dlRssMB, match }); + return klass; + } + const expectedSha = up.result.body.expectedSha; + + // ---- DOWNLOAD/VERIFY (server-side streaming hash) ---- + const dl = await withMemWatch(client, () => op({ action: 'verify', key })); + dlHeapMB = dl.peakHeapMB; + dlRssMB = dl.peakRssMB; + if (dl.result.status !== 200 || dl.result.body?.present !== true) { + klass = dl.result.status >= 500 ? `${dl.result.status}-cliff-on-read` : `verify-status-${dl.result.status}`; + findings.push(`Q1 ${mb}MB VERIFY ${klass}: status=${dl.result.status} body=${JSON.stringify(dl.result.body).slice(0, 200)}`); + results.push({ mb, klass, upHeapMB, upRssMB, dlHeapMB, dlRssMB, match }); + return klass; + } + match = dl.result.body.match === true && dl.result.body.readSha === expectedSha; + if (!match) klass = dl.result.body.readSize !== size ? 'truncated' : 'corrupt-checksum'; + + findings.push( + `Q1 ${mb}MB → ${klass}; readSha==expected=${dl.result.body.readSha === expectedSha} ` + + `readSize=${dl.result.body.readSize}/${size} ` + + `UP[heap+${(upHeapMB - idleRssMB > 0 ? upHeapMB : upHeapMB).toFixed(0)} rss=${upRssMB.toFixed(0)}MB Δ${(upRssMB - idleRssMB).toFixed(0)}] ` + + `DL[heap=${dlHeapMB.toFixed(0)} rss=${dlRssMB.toFixed(0)}MB Δ${(dlRssMB - idleRssMB).toFixed(0)}] ` + + `up=${up.result.body.uploadMs}ms dl=${dl.result.body.verifyMs}ms` + ); + results.push({ mb, klass, upHeapMB, upRssMB, dlHeapMB, dlRssMB, match }); + return klass; + }; + + for (const mb of baseSizes) { + await sleep(1500); + await runSize(mb); + } + + // Escalation: only attempt 512MB if 256MB stayed bounded (true streaming). + const at256 = results.find((r) => r.mb === 256); + const bounded256 = + at256 && at256.match && at256.upRssMB - idleRssMB < RSS_ESCALATE_GUARD_MB && at256.dlRssMB - idleRssMB < RSS_ESCALATE_GUARD_MB; + if (bounded256) { + findings.push(`Q1 escalate: 256MB bounded (upΔrss=${(at256!.upRssMB - idleRssMB).toFixed(0)} dlΔrss=${(at256!.dlRssMB - idleRssMB).toFixed(0)}MB < ${RSS_ESCALATE_GUARD_MB}) → attempting 512MB`); + await sleep(2000); + await runSize(512); + } else { + findings.push( + `Q1 escalate: NOT attempting 512MB — 256MB ` + + (at256 ? `Δrss up=${(at256.upRssMB - idleRssMB).toFixed(0)}/dl=${(at256.dlRssMB - idleRssMB).toFixed(0)}MB match=${at256.match}` : 'missing') + + ` (guard ${RSS_ESCALATE_GUARD_MB}MB)` + ); + } + + // Memory trend across the bounded sizes that succeeded. + const okPts = results.filter((r) => r.match); + if (okPts.length >= 2) { + const a = okPts[0]; + const b = okPts[okPts.length - 1]; + const upTrend = b.upRssMB - a.upRssMB; + const dlTrend = b.dlRssMB - a.dlRssMB; + findings.push( + `Q1 MEMORY TREND ${a.mb}→${b.mb}MB (${(b.mb / a.mb).toFixed(1)}x): ` + + `upload Δrss=${upTrend.toFixed(0)}MB, download Δrss=${dlTrend.toFixed(0)}MB → ` + + (Math.abs(upTrend) < (b.mb - a.mb) * 0.5 && Math.abs(dlTrend) < (b.mb - a.mb) * 0.5 + ? 'BOUNDED (rss does NOT scale ~1:1 with blob size => true streaming both directions)' + : 'rss scales with blob size => BUFFERING (OOM lever)') + ); + } + + ok( + results.some((r) => r.match), + 'no size round-tripped byte-exact' + ); + }); + + // Sub-attribute wire GET at the largest succeeded size — streaming hash on the wire. + test('Q2: blob sub-attribute GET path (small + large; record + sub-attr routes)', async () => { + // First a SMALL (1MB) blob to characterize the read-path routes cheaply, so a + // 404 at large size can be distinguished from "this route is unsupported". + const small = 1 * MB; + const smallKey = 'wire-small'; + const sSt = await op({ action: 'store', key: smallKey, seed: smallKey, size: small }); + ok(sSt.status === 200, `small store failed: ${sSt.status} ${JSON.stringify(sSt.body).slice(0, 150)}`); + const smallExpected = sSt.body.expectedSha; + + // Probe candidate routes for reading a blob attribute over REST. + const routes = (k: string) => [`/Big/${k}/payload`, `/Big/${k}.payload`, `/Big/${k}?select(payload)`, `/Big/${k}`]; + for (const path of routes(smallKey)) { + const r = await streamHashGet(httpURL, path, authHeader); + const exact = r.status === 200 && r.bytes === small && r.sha === smallExpected; + findings.push( + `Q2 route GET ${path} → status=${r.status} bytes=${r.bytes}/${small} blobShaMatch=${r.sha === smallExpected} ` + + `ct-len=${r.contentLength ?? '-'} te=${r.transferEncoding ?? '-'}${exact ? ' [BLOB BYTE-EXACT]' : ''}` + ); + } + + // Now the large sub-attribute GET (256MB), streaming wire hash. + const mb = 256; + const size = mb * MB; + const key = 'wire-256'; + const st = await op({ action: 'store', key, seed: key, size }); + if (st.status !== 200) { + findings.push(`Q2 setup store ${mb}MB failed status=${st.status}; skipping large wire GET`); + } else { + const expectedSha = st.body.expectedSha; + const { result: r } = await withMemWatch(client, () => streamHashGet(httpURL, `/Big/${key}/payload`, authHeader)); + const exact = r.status === 200 && r.bytes === size && r.sha === expectedSha; + findings.push( + `Q2 LARGE GET /Big/${key}/payload → status=${r.status} bytes=${r.bytes}/${size} ` + + `sha==expected=${r.sha === expectedSha} te=${r.transferEncoding ?? '-'} ct-len=${r.contentLength ?? '-'} ` + + `ttfb=${r.ttfbMs}ms total=${r.totalMs}ms → ${exact ? 'WIRE BYTE-EXACT (streaming)' : 'NOT-EXACT/ERR'}` + ); + await op({ action: 'delete', key }); + } + await op({ action: 'delete', key: smallKey }); + ok(true); // diagnostic test: record route behaviors, don't hard-fail on route shape + }); + + // Overwrite + delete reclaim: write a large blob, overwrite it, delete it, + // confirm blob-path disk usage drops back (space reclaimed, not leaked). + test('Q3: overwrite + delete reclaims blob-path disk space', async () => { + const mb = 128; + const size = mb * MB; + const key = 'reclaim-1'; + + const before = await dirBytes(blobPath); + await op({ action: 'store', key, seed: 'v1', size }); + await sleep(2000); + const afterStore = await dirBytes(blobPath); + + // Overwrite with new content (same key) — old blob file should be reclaimed. + await op({ action: 'store', key, seed: 'v2', size }); + await sleep(3000); + const afterOverwrite = await dirBytes(blobPath); + // Confirm the overwrite content is what's stored now. + const v2exp = (await op({ action: 'expected', key, seed: 'v2', size })).body.expectedSha; + const v = await op({ action: 'verify', key }); + const overwriteCorrect = v.body?.readSha === v2exp; + + // Delete — space should drop back toward baseline. + await op({ action: 'delete', key }); + await sleep(4000); + const afterDelete = await dirBytes(blobPath); + + const toMB = (b: number) => (b / MB).toFixed(1); + findings.push( + `Q3 reclaim (${mb}MB): blobDir before=${toMB(before)} afterStore=${toMB(afterStore)} ` + + `afterOverwrite=${toMB(afterOverwrite)} afterDelete=${toMB(afterDelete)}MB; ` + + `overwrite content correct=${overwriteCorrect}` + ); + const grewOnStore = afterStore - before > size * 0.8; + const overwriteReclaimed = afterOverwrite - before < size * 1.5; // not ~2x (old file lingering forever) + const deleteReclaimed = afterDelete - before < size * 0.5; + findings.push( + `Q3 verdict: store-grew=${grewOnStore} overwrite-bounded(~1x not 2x)=${overwriteReclaimed} ` + + `delete-reclaimed=${deleteReclaimed} → ` + + (overwriteReclaimed && deleteReclaimed ? 'SPACE RECLAIMED (no large-blob leak)' : 'POSSIBLE LEAK — disk did not drop back') + ); + ok(overwriteCorrect, 'overwrite did not store the new content'); + }); + + test('instance stayed alive throughout', async () => { + const r = await client.req().send({ operation: 'system_information', attributes: ['threads'] }).expect(200); + ok(Array.isArray(r.body.threads), 'system_information should report threads (instance alive)'); + }); +}); diff --git a/integrationTests/database/large-blob-streaming/config.yaml b/integrationTests/database/large-blob-streaming/config.yaml new file mode 100644 index 0000000000..0d7108edf3 --- /dev/null +++ b/integrationTests/database/large-blob-streaming/config.yaml @@ -0,0 +1,12 @@ +# QA-144 — Large-blob storage limits + streaming upload/download correctness. +# +# Big: a durable blob table holding genuinely large (256MB+) blobs. The +# BlobOps resource generates blob bytes via an async generator (chunked, never +# the whole blob in the worker heap), saves them through createBlob's streaming +# source path, and reads them back through blob.stream() with a running hash — +# so neither upload nor download server-side requires buffering the full blob. +graphqlSchema: + files: '*.graphql' +jsResource: + files: resources.js +rest: true diff --git a/integrationTests/database/large-blob-streaming/resources.js b/integrationTests/database/large-blob-streaming/resources.js new file mode 100644 index 0000000000..1d22e5dbae --- /dev/null +++ b/integrationTests/database/large-blob-streaming/resources.js @@ -0,0 +1,147 @@ +// QA-144 — Large-blob storage limits + streaming upload/download correctness. +// +// EVERYTHING is done in bounded chunks so we never hold a whole 256MB+ blob in +// the worker JS heap: +// - UPLOAD: createBlob() streams +// the bytes straight to the blob file (writeBlobWithStream). The generator +// produces a deterministic HMAC-SHA256 keystream so the content is +// non-compressible and unique per (seed,size), and we can recompute the +// expected SHA-256 independently — also chunk-by-chunk, never buffered. +// - DOWNLOAD/VERIFY: read the stored blob via blob.stream() and feed each +// chunk into a running createHash('sha256'), so the read-back hash is +// computed without ever materializing the full blob in memory. +// +// If Harper truly streams, the worker heap should stay BOUNDED (a few chunk +// buffers) during both upload and download regardless of blob size. If it +// buffers (e.g. payload.bytes()), heap/rss would balloon ~proportional to size. + +import { createHash, createHmac } from 'node:crypto'; + +const { Big } = tables; + +const CHUNK = 4 * 1024 * 1024; // 4MB working chunk — the only large allocation we hold + +// Deterministic keystream: block i = HMAC-SHA256(seed, i). Yields CHUNK-sized +// Buffers totalling exactly `size` bytes, generating on the fly (no full buffer). +function* keystreamChunks(seed, size) { + let produced = 0; + let counter = 0; + let pending = Buffer.alloc(0); + while (produced < size) { + // Fill up a CHUNK from 32-byte HMAC blocks. + const want = Math.min(CHUNK, size - produced); + const parts = [pending]; + let have = pending.length; + while (have < want) { + const block = createHmac('sha256', String(seed)).update(String(counter++)).digest(); + parts.push(block); + have += block.length; + } + const joined = Buffer.concat(parts, have); + const out = joined.subarray(0, want); + pending = joined.subarray(want); // carry leftover into next chunk + produced += out.length; + yield out; + } +} + +// Expected SHA-256 of the (seed,size) pattern, computed chunk-by-chunk. +function expectedSha(seed, size) { + const h = createHash('sha256'); + for (const c of keystreamChunks(seed, size)) h.update(c); + return h.digest('hex'); +} + +export class BlobOps extends Resource { + static loadAsInstance = false; + + async post(query, body) { + try { + switch (body && body.action) { + case 'store': + return await this.store(body); + case 'verify': + return await this.verify(body); + case 'delete': + return await this.del(body); + case 'expected': + // Pure server-side expected SHA (chunked) for cross-check. + return { ok: true, expectedSha: expectedSha(body.seed ?? body.key, Number(body.size)) }; + default: { + const ctx = this.getContext(); + if (ctx && ctx.response) ctx.response.status = 400; + return { ok: false, reason: 'unknown-action', action: body && body.action }; + } + } + } catch (e) { + const ctx = this.getContext(); + if (ctx && ctx.response) ctx.response.status = 500; + return { ok: false, error: String(e && e.message), stack: String(e && e.stack).split('\n').slice(0, 4) }; + } + } + + // STREAMING UPLOAD: create the blob from an async generator (no full buffer). + async store(body) { + const key = String(body.key); + const seed = body.seed == null ? key : String(body.seed); + const size = Number(body.size); + const t0 = Date.now(); + + // Async generator so createBlob takes the streaming (Readable.from) path. + async function* gen() { + for (const c of keystreamChunks(seed, size)) { + yield c; + // yield to the event loop so heap sampling can observe us mid-flight + await new Promise((r) => setImmediate(r)); + } + } + + const want = expectedSha(seed, size); + await Big.put({ + key, + payload: createBlob(gen(), { type: 'application/octet-stream', size }), + sha256: want, + size, + }); + + return { ok: true, key, expectedSha: want, size, uploadMs: Date.now() - t0 }; + } + + // STREAMING VERIFY: read back via blob.stream(), running-hash, never buffered. + async verify(body) { + const key = String(body.key); + const t0 = Date.now(); + const rec = await Big.get(key); + if (!rec) return { ok: true, present: false, key }; + + const blob = rec.payload; + const h = createHash('sha256'); + let bytesRead = 0; + // blob.stream() returns a web ReadableStream of Uint8Array chunks. + const reader = blob.stream().getReader(); + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + bytesRead += value.length; + h.update(value); + } + const readSha = h.digest('hex'); + return { + ok: true, + present: true, + key, + storedSha: rec.sha256, + storedSize: Number(rec.size), + readSize: bytesRead, + readSha, + match: readSha === rec.sha256 && bytesRead === Number(rec.size), + verifyMs: Date.now() - t0, + }; + } + + async del(body) { + const key = String(body.key); + await Big.delete(key); + return { ok: true, key, deleted: true }; + } +} diff --git a/integrationTests/database/large-blob-streaming/schema.graphql b/integrationTests/database/large-blob-streaming/schema.graphql new file mode 100644 index 0000000000..f73ab506da --- /dev/null +++ b/integrationTests/database/large-blob-streaming/schema.graphql @@ -0,0 +1,11 @@ +# QA-144 — Large-blob storage limits + streaming upload/download correctness. +# +# Big: a durable blob table (no expiration). Holds genuinely LARGE blobs +# (256MB+). @sealed with a declared Blob field plus checksum + size so the +# server-side resource can verify integrity against the actual stored content. +type Big @table @sealed @export { + key: ID! @primaryKey + payload: Blob! + sha256: String + size: Float +} diff --git a/integrationTests/mqtt/mqtt-at-scale.test.ts b/integrationTests/mqtt/mqtt-at-scale.test.ts new file mode 100644 index 0000000000..3daf7af5d3 --- /dev/null +++ b/integrationTests/mqtt/mqtt-at-scale.test.ts @@ -0,0 +1,529 @@ +/** + * MQTT broker AT SCALE: many topics x many subscribers, wildcard + * fan-out, QoS-1 reconnect-backlog, per-subscription isolation, broker memory. + * + * Prior MQTT waves were single-topic. This pushes to a table-backed topic + * space with MANY device topics (Device/) and MANY concurrent subscribers — + * some WILDCARD (Device/#, Device/+), some specific-topic — then a burst of + * publishes across all topics, plus a QoS-1 subscriber that disconnects and + * reconnects to drain a backlog. + * + * Wiring recap: + * - MQTT transport: WebSocket on Harper's HTTP port at /mqtt, MQTT v5, admin + * basic-auth — same loopback the REST client uses. + * - Topics are table-backed: Device/ <-> a Device record keyed by . + * A publish to Device/7 is a write to record id "7"; a subscriber on + * Device/7 / Device/+ / Device/# sees it as a change message. + * - Durable QoS-1 backlog: clean:false + sessionExpiryInterval; on reconnect + * (no re-subscribe) the broker replays the audit gap. + * + * QUESTIONS + * Q1 WILDCARD FAN-OUT CORRECTNESS (no cross-topic bleed): with subscribers on + * Device/#, Device/+, and several SPECIFIC Device/, burst one publish to + * every Device/. Assert: # sub gets EXACTLY the full id set; + sub gets + * exactly the full id set (single level); each specific sub gets ONLY its id + * and NOTHING else (a Device/7 sub receiving Device/2 is cross-topic BLEED — + * a correctness/security defect). No drop, no dup. + * Q2 QoS-1 DURABLE BACKLOG ON RECONNECT: a clean:false QoS-1 sub on Device/# + * attaches, confirms it is armed, disconnects; we then publish a backlog + * across ALL topics while it is gone; it reconnects (no re-subscribe) and + * must drain the backlog (every gap publish delivered, no loss). + * Q3 PER-SUBSCRIPTION ISOLATION (no head-of-line blocking): a STALLED subscriber + * (paused WS stream, never drains) on Device/# coexists with HEALTHY + * subscribers during a burst. The healthy subs must receive their full set + * promptly — the stalled peer must not starve/delay them. Also sample + * per-worker heap to characterize broker memory under fan-out (bounded vs + * growing). + * + * Run BOTH single-worker and multi-worker (threads.count=4) by parameterizing + * the suite over a config matrix. + * + * SANDBOX CAVEAT: MQTT-over-loopback-WebSocket sometimes fails to connect here + * ("socket hang up"). `before` does a connect probe; if THAT fails we report a + * HARNESS limitation rather than a Harper defect (skip the probes). + */ +import { suite, test, before, after } from 'node:test'; +import { ok, strictEqual, deepStrictEqual } from 'node:assert/strict'; +import { resolve } from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; + +import mqtt, { type IClientOptions, type MqttClient } from 'mqtt'; +import request from 'supertest'; + +import { setupHarperWithFixture, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing'; +// @ts-expect-error utils/client.mjs has no type declarations; runtime resolves fine +import { createApiClient } from '../apiTests/utils/client.mjs'; + +const FIXTURE_PATH = resolve(import.meta.dirname, 'mqtt-at-scale'); +const TABLE = 'Device'; + +const skipSuite = process.env.HARPER_RUNTIME === 'bun' || process.platform === 'win32'; + +const ADMIN_USER = 'admin'; +const ADMIN_PASS = 'Abc1234!'; + +// ── MQTT helpers ────────────────────────────────────────────────────────────── +function baseOpts(overrides: Partial = {}): IClientOptions { + return { + protocolVersion: 5, + reconnectPeriod: 0, + connectTimeout: 8000, + clean: true, + username: ADMIN_USER, + password: ADMIN_PASS, + ...overrides, + }; +} + +function connect(url: string, opts: IClientOptions): Promise { + return new Promise((resolve, reject) => { + const client = mqtt.connect(url, opts); + const onError = (err: Error) => { + client.removeListener('connect', onConnect); + client.end(true); + reject(err); + }; + const onConnect = () => { + client.removeListener('error', onError); + resolve(client); + }; + client.once('error', onError); + client.once('connect', onConnect); + }); +} + +function subscribe(client: MqttClient, topic: string, qos: 0 | 1 | 2 = 1): Promise { + return new Promise((resolve, reject) => { + client.subscribe(topic, { qos }, (err, granted) => (err ? reject(err) : resolve(granted ?? []))); + }); +} + +function endQuiet(client: MqttClient | undefined): Promise { + return new Promise((resolve) => { + if (!client) return resolve(); + client.end(true, {}, () => resolve()); + }); +} + +interface MqttMsg { + topic: string; + id?: string; + value?: number; + tag?: string; + at: number; +} + +/** Collect+parse Device change messages for one MQTT consumer. */ +function collect(client: MqttClient) { + const events: MqttMsg[] = []; + const handler = (topic: string, payload: Buffer) => { + let parsed: any; + try { + parsed = JSON.parse(payload.toString()); + } catch { + parsed = undefined; + } + const value = + typeof parsed?.value === 'number' + ? parsed.value + : typeof parsed?.value?.value === 'number' + ? parsed.value.value + : undefined; + events.push({ topic, id: parsed?.id != null ? String(parsed.id) : undefined, value, tag: parsed?.tag, at: Date.now() }); + }; + client.on('message', handler); + return { events, stop: () => client.removeListener('message', handler) }; +} + +async function waitFor(predicate: () => boolean, timeoutMs = 15_000, intervalMs = 25): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate()) return true; + await sleep(intervalMs); + } + return predicate(); +} + +/** Sample per-worker heap (system_information threads). Returns max heapUsed across workers, in MB. */ +async function maxWorkerHeapMB(client: ReturnType): Promise { + try { + const r = await client.req().send({ operation: 'system_information', attributes: ['threads'] }).timeout(10_000); + const threads = (r.body as any)?.threads; + if (Array.isArray(threads)) { + const heaps = threads.map((t: any) => (typeof t.heapUsed === 'number' ? t.heapUsed : 0)); + return Math.max(0, ...heaps) / (1024 * 1024); + } + } catch { + /* ignore */ + } + return -1; +} + +const j = (x: any) => JSON.stringify(x); + +// The id of the topic the SPECIFIC subscribers target (must never bleed). +const SPECIFIC_IDS = ['7', '13', '42']; + +/** Build one parameterized suite per worker-count config. */ +function buildSuite(label: string, config: Record) { + suite(`MQTT at scale (${label})`, { skip: skipSuite }, (ctx: ContextWithHarper) => { + let client: ReturnType; + let restBase = ''; + let mqttURL = ''; + let mqttUsable = false; + let mqttSkipReason = ''; + + const restPut = (id: string, body: object) => + request(restBase).put(`/${TABLE}/${id}`).set(client.headers).send(body); + + before(async () => { + await setupHarperWithFixture(ctx, FIXTURE_PATH, { config, env: {} }); + client = createApiClient(ctx.harper); + restBase = client.restURL; + + const httpURL = ctx.harper.httpURL; + const wsScheme = httpURL.startsWith('https') ? 'wss' : 'ws'; + mqttURL = `${httpURL.replace(/^https?/, wsScheme)}/mqtt`; + + // Readiness poll. + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + try { + const probe = await client.reqRest(`/${TABLE}/`).timeout(3_000); + if (probe.status !== 404) break; + } catch { + /* not ready */ + } + await sleep(250); + } + + // MQTT connect probe — separates a Harper defect from sandbox WS hang-ups. + try { + const probe = await connect(mqttURL, baseOpts({ clientId: `mqtt-scale-${label}-probe` })); + mqttUsable = probe.connected === true; + await endQuiet(probe); + } catch (err) { + mqttUsable = false; + mqttSkipReason = `MQTT connect probe failed on ${mqttURL}: ${(err as Error)?.message}`; + console.log(`\n[mqtt-at-scale][${label}] HARNESS: ${mqttSkipReason}`); + } + }); + + after(async () => { + await teardownHarper(ctx); + }); + + // ── Q1: WILDCARD FAN-OUT — exact #/+ matching, NO cross-topic bleed ──────── + test('Q1: wildcard fan-out correctness — no cross-topic bleed', async () => { + if (!mqttUsable) { + console.log(`[mqtt-at-scale][${label}][Q1] SKIPPED (harness): ${mqttSkipReason}`); + return; + } + const N = 60; // 60 device topics Device/0..Device/59 + const runTag = `q1-${label}-${Date.now()}`; + const allIds = [...Array(N)].map((_, i) => String(i)); + + const pub = await connect(mqttURL, baseOpts({ clientId: `mqtt-scale-${label}-q1-pub` })); + const hashSub = await connect(mqttURL, baseOpts({ clientId: `mqtt-scale-${label}-q1-hash` })); + const plusSub = await connect(mqttURL, baseOpts({ clientId: `mqtt-scale-${label}-q1-plus` })); + const specSubs: { id: string; c: MqttClient; obs: ReturnType }[] = []; + try { + const hashObs = collect(hashSub); + const plusObs = collect(plusSub); + await subscribe(hashSub, `${TABLE}/#`, 1); + await subscribe(plusSub, `${TABLE}/+`, 1); + for (const id of SPECIFIC_IDS) { + const c = await connect(mqttURL, baseOpts({ clientId: `mqtt-scale-${label}-q1-spec-${id}` })); + const obs = collect(c); + await subscribe(c, `${TABLE}/${id}`, 1); + specSubs.push({ id, c, obs }); + } + await sleep(800); // let all subscriptions attach + + // Burst: one publish to EVERY Device/. value == numeric id. + const t0 = Date.now(); + for (const id of allIds) { + await restPut(id, { id, value: Number(id), tag: runTag }).expect(204); + } + const writeMs = Date.now() - t0; + + const mine = (e: MqttMsg) => e.tag === runTag; + await waitFor(() => hashObs.events.filter(mine).length >= N, 20_000); + await waitFor(() => plusObs.events.filter(mine).length >= N, 20_000); + for (const s of specSubs) await waitFor(() => s.obs.events.filter(mine).length >= 1, 20_000); + await sleep(1500); // settle window to catch any trailing bleed + + // ── Wildcard completeness + dup analysis ── + const analyze = (obs: ReturnType) => { + const ids = obs.events.filter(mine).map((e) => e.id).filter((x): x is string => x != null); + const seen = new Map(); + for (const k of ids) seen.set(k, (seen.get(k) ?? 0) + 1); + const uniq = seen.size; + return { recv: ids.length, uniq, dup: ids.length - uniq, dropped: N - uniq, set: seen }; + }; + const h = analyze(hashObs); + const p = analyze(plusObs); + + // ── Cross-topic BLEED analysis on each specific subscriber ── + const bleedReports: string[] = []; + let totalBleed = 0; + let totalSpecMissing = 0; + for (const s of specSubs) { + const ids = s.obs.events.filter(mine).map((e) => e.id).filter((x): x is string => x != null); + const got = new Set(ids); + const bleed = [...got].filter((g) => g !== s.id); // ANY id != its own == BLEED + const gotOwn = got.has(s.id); + totalBleed += bleed.length; + if (!gotOwn) totalSpecMissing++; + bleedReports.push( + ` Device/${s.id}: gotOwn=${gotOwn} recv=${ids.length} ` + + `bleed=${bleed.length ? j(bleed.slice(0, 8)) : 'none'}` + ); + } + + hashObs.stop(); + plusObs.stop(); + for (const s of specSubs) s.obs.stop(); + + console.log( + `\n[mqtt-at-scale][${label}][Q1] WILDCARD FAN-OUT (${N} topics, burst in ${writeMs}ms):\n` + + ` Device/# : recv=${h.recv} uniq=${h.uniq} dup=${h.dup} dropped=${h.dropped} (expect ${N}/${N}/0/0)\n` + + ` Device/+ : recv=${p.recv} uniq=${p.uniq} dup=${p.dup} dropped=${p.dropped} (expect ${N}/${N}/0/0)\n` + + ` SPECIFIC subscribers (each must get ONLY its own id):\n` + + bleedReports.join('\n') + + `\n => totalCrossTopicBleed=${totalBleed} specMissingOwn=${totalSpecMissing}` + ); + + // Hard invariants — wildcard completeness: + strictEqual(h.dropped, 0, `Device/# must receive every topic; dropped=${h.dropped}`); + strictEqual(h.dup, 0, `Device/# must not duplicate; dup=${h.dup}`); + strictEqual(p.dropped, 0, `Device/+ must receive every single-level topic; dropped=${p.dropped}`); + strictEqual(p.dup, 0, `Device/+ must not duplicate; dup=${p.dup}`); + // THE SECURITY/CORRECTNESS INVARIANT — no cross-topic bleed: + strictEqual( + totalBleed, + 0, + `CROSS-TOPIC BLEED: a specific subscriber received a non-matching topic. ${bleedReports.join(' | ')}` + ); + strictEqual(totalSpecMissing, 0, 'each specific subscriber must receive its OWN topic'); + } finally { + await endQuiet(pub); + await endQuiet(hashSub); + await endQuiet(plusSub); + for (const s of specSubs) await endQuiet(s.c); + } + }); + + // ── Q2: QoS-1 DURABLE BACKLOG drained on reconnect ───────────────────────── + test('Q2: QoS-1 durable backlog drained on reconnect across all topics', async () => { + if (!mqttUsable) { + console.log(`[mqtt-at-scale][${label}][Q2] SKIPPED (harness): ${mqttSkipReason}`); + return; + } + const N = 40; + const clientId = `mqtt-scale-${label}-q2-durable`; + const topic = `${TABLE}/#`; + const runTag = `q2-${label}-${Date.now()}`; + const pub = await connect(mqttURL, baseOpts({ clientId: `${clientId}-pub` })); + let durable: MqttClient | undefined; + try { + // (1) Durable session, QoS-1, wildcard subscribe. + durable = await connect( + mqttURL, + baseOpts({ clientId, clean: false, properties: { sessionExpiryInterval: 300 } }) + ); + let obs = collect(durable); + await subscribe(durable, topic, 1); + await sleep(400); + + // Pre-gap live message confirms the sub is armed. + await restPut('arm', { id: 'arm', value: -1, tag: `${runTag}-arm` }).expect(204); + await waitFor(() => obs.events.some((e) => e.tag === `${runTag}-arm`), 8000); + obs.stop(); + + // (2) Disconnect — broker retains the durable session. + await endQuiet(durable); + durable = undefined; + await sleep(200); + + // (3) BACKLOG while disconnected: one publish per topic across N topics. + const backlogIds = [...Array(N)].map((_, i) => `bk-${i}`); + for (let i = 0; i < N; i++) { + await restPut(backlogIds[i], { id: backlogIds[i], value: i, tag: runTag }).expect(204); + } + + // (4) Reconnect SAME durable session (no re-subscribe); drain backlog. + durable = await connect( + mqttURL, + baseOpts({ clientId, clean: false, properties: { sessionExpiryInterval: 300 } }) + ); + obs = collect(durable); + await waitFor(() => obs.events.filter((e) => e.tag === runTag).length >= N, 20_000); + await sleep(1500); + + const got = obs.events.filter((e) => e.tag === runTag).map((e) => e.id).filter((x): x is string => x != null); + const seen = new Map(); + for (const k of got) seen.set(k, (seen.get(k) ?? 0) + 1); + const missing = backlogIds.filter((id) => !seen.has(id)); + const dups = backlogIds.filter((id) => (seen.get(id) ?? 0) > 1); + obs.stop(); + + console.log( + `\n[mqtt-at-scale][${label}][Q2] QoS-1 DURABLE BACKLOG (${N} topics published while offline):\n` + + ` drained=${seen.size}/${N} missing=${missing.length ? j(missing.slice(0, 10)) : 'none'} ` + + `dup=${dups.length ? j(dups.slice(0, 10)) : 'none'}\n` + + ` => ${missing.length === 0 ? 'CORRECT: full backlog drained' : 'LOST BACKLOG'}` + ); + + deepStrictEqual(missing, [], `QoS-1 durable backlog must be fully drained; missing=${j(missing)}`); + deepStrictEqual(dups, [], `QoS-1 durable backlog must not duplicate; dup=${j(dups)}`); + } finally { + await endQuiet(pub); + if (durable) await endQuiet(durable); + // purge the durable session so a re-run starts clean + try { + const cleaner = await connect(mqttURL, baseOpts({ clientId, clean: true })); + await endQuiet(cleaner); + } catch { + /* best effort */ + } + } + }); + + // ── Q3: ISOLATION (no HOL blocking) + broker memory under fan-out ────────── + test('Q3: stalled subscriber does not starve healthy subs; memory bounded', async () => { + if (!mqttUsable) { + console.log(`[mqtt-at-scale][${label}][Q3] SKIPPED (harness): ${mqttSkipReason}`); + return; + } + const N = 50; + const ROUNDS = 5; + const big = 'x'.repeat(8192); // ~8KB payload, fan-out across many subs + const runTag = `q3-${label}-${Date.now()}`; + const baselineHeap = await maxWorkerHeapMB(client); + + // STALLED subscriber on Device/# — paused WS stream, never drains. + const stalled = await connect(mqttURL, baseOpts({ clientId: `mqtt-scale-${label}-q3-stalled` })); + await subscribe(stalled, `${TABLE}/#`, 1); + try { + (stalled.stream as any)?.pause?.(); + } catch { + /* ignore */ + } + + // HEALTHY subscribers: one Device/# and several specific. + const healthyHash = await connect(mqttURL, baseOpts({ clientId: `mqtt-scale-${label}-q3-healthy` })); + const healthyObs = collect(healthyHash); + await subscribe(healthyHash, `${TABLE}/#`, 1); + const specSubs: { id: string; c: MqttClient; obs: ReturnType }[] = []; + for (const id of SPECIFIC_IDS) { + const c = await connect(mqttURL, baseOpts({ clientId: `mqtt-scale-${label}-q3-spec-${id}` })); + const obs = collect(c); + await subscribe(c, `${TABLE}/${id}`, 1); + specSubs.push({ id, c, obs }); + } + + try { + await sleep(800); + const heapTrend: number[] = [baselineHeap]; + let writeErrors = 0; + const roundLatency: number[] = []; + + for (let r = 0; r < ROUNDS; r++) { + const roundTag = `${runTag}-r${r}`; + const ids = [...Array(N)].map((_, i) => String(i)); + const t0 = Date.now(); + for (const id of ids) { + try { + await restPut(id, { id, value: r, tag: roundTag }).timeout(8000).expect(204); + } catch (e) { + writeErrors++; + if (writeErrors <= 2) console.log(`[mqtt-at-scale][${label}][Q3] write err: ${(e as Error).message}`); + } + } + + // Healthy Device/# must receive the full round set promptly despite the + // stalled peer (isolation / no head-of-line blocking). + const armed = await waitFor(() => healthyObs.events.filter((e) => e.tag === roundTag).length >= N, 12_000); + const latency = Date.now() - t0; + roundLatency.push(latency); + + await sleep(1200); // let resource-report interval update before sampling heap + heapTrend.push(await maxWorkerHeapMB(client)); + const got = new Set(healthyObs.events.filter((e) => e.tag === roundTag).map((e) => e.id)); + console.log( + `[mqtt-at-scale][${label}][Q3] round ${r}: ${N} writes, healthy#recv=${got.size}/${N} ` + + `armed=${armed} latency=${latency}ms maxHeapMB=${heapTrend[heapTrend.length - 1].toFixed(1)}` + ); + } + + await sleep(3000); + const afterIdleHeap = await maxWorkerHeapMB(client); + + // Isolation: every healthy sub received its full set across all rounds. + const healthyTotal = new Set(healthyObs.events.filter((e) => e.tag?.startsWith(runTag)).map((e) => `${e.tag}/${e.id}`)); + const expectedHealthy = N * ROUNDS; + healthyObs.stop(); + let specStarved = 0; + for (const s of specSubs) { + const own = s.obs.events.filter((e) => e.tag?.startsWith(runTag) && e.id === s.id).length; + const bleed = s.obs.events.filter((e) => e.tag?.startsWith(runTag) && e.id !== s.id).length; + if (own < ROUNDS) specStarved++; + console.log(`[mqtt-at-scale][${label}][Q3] spec Device/${s.id}: own=${own}/${ROUNDS} bleed=${bleed}`); + s.obs.stop(); + } + + const valid = heapTrend.filter((h) => h >= 0); + const peak = Math.max(...valid); + const growth = peak - baselineHeap; + const totalPayloadMB = (ROUNDS * N * big.length) / (1024 * 1024); + let strictlyRising = valid.length >= 3; + for (let i = 2; i < valid.length; i++) if (valid[i] <= valid[i - 1]) strictlyRising = false; + const maxLatency = Math.max(...roundLatency); + + let memVerdict: string; + if (strictlyRising && growth > totalPayloadMB * 0.5) + memVerdict = 'UNBOUNDED (heap tracks buffered payload per stalled sub)'; + else if (growth > totalPayloadMB) memVerdict = 'GROWING (non-monotonic but large)'; + else memVerdict = 'BOUNDED (heap did not track buffered fan-out payload)'; + + console.log( + `\n[mqtt-at-scale][${label}][Q3] ISOLATION + MEMORY:\n` + + ` healthy Device/# delivered=${healthyTotal.size}/${expectedHealthy} specStarved=${specStarved}\n` + + ` round latency ms = ${j(roundLatency)} (max ${maxLatency}ms)\n` + + ` payload (note: backing-table writes, not raw buffer) ~${totalPayloadMB.toFixed(1)}MB\n` + + ` heap MB trend = ${heapTrend.map((h) => h.toFixed(1)).join(' -> ')} afterIdle=${afterIdleHeap.toFixed(1)}\n` + + ` baseline=${baselineHeap.toFixed(1)} peak=${peak.toFixed(1)} growth=${growth.toFixed(1)}MB ` + + `strictlyRising=${strictlyRising}\n` + + ` writeErrors=${writeErrors}\n` + + ` => ISOLATION: ${specStarved === 0 && healthyTotal.size >= expectedHealthy ? 'OK (no starvation)' : 'STARVED'} | MEMORY: ${memVerdict}` + ); + + // Hard invariants: a stalled peer must not starve healthy subs. + strictEqual(specStarved, 0, 'specific healthy subs must not be starved by a stalled peer'); + strictEqual( + healthyTotal.size, + expectedHealthy, + `healthy Device/# must receive all ${expectedHealthy} events despite a stalled peer; got ${healthyTotal.size}` + ); + ok(maxLatency < 12_000, `per-round delivery must stay responsive; max latency ${maxLatency}ms`); + strictEqual(writeErrors, 0, 'writes must not error while a subscriber is stalled'); + // Memory characterized (logged verdict), not hard-asserted — unbounded + // live-tail is a known META, not a fresh defect. + ok(true, 'memory behavior characterized in log'); + } finally { + await endQuiet(stalled); + await endQuiet(healthyHash); + for (const s of specSubs) await endQuiet(s.c); + } + }); + + test('instance survived scale probes (liveness)', async () => { + const rest = await client.reqRest(`/${TABLE}/`).timeout(5_000).catch(() => ({ status: 0 })); + ok((rest as any).status !== 0, `instance should still answer REST, got status ${(rest as any).status}`); + }); + }); +} + +buildSuite('single-worker', { threads: { count: 1 } }); +buildSuite('multi-worker', { threads: { count: 4 } }); diff --git a/integrationTests/mqtt/mqtt-at-scale/config.yaml b/integrationTests/mqtt/mqtt-at-scale/config.yaml new file mode 100644 index 0000000000..13e06e293c --- /dev/null +++ b/integrationTests/mqtt/mqtt-at-scale/config.yaml @@ -0,0 +1,4 @@ +graphqlSchema: + files: '*.graphql' +rest: true +mqtt: true diff --git a/integrationTests/mqtt/mqtt-at-scale/schema.graphql b/integrationTests/mqtt/mqtt-at-scale/schema.graphql new file mode 100644 index 0000000000..b70cea8eca --- /dev/null +++ b/integrationTests/mqtt/mqtt-at-scale/schema.graphql @@ -0,0 +1,20 @@ +# QA-157 — MQTT broker at scale: many topics x many subscribers, wildcard +# fan-out, QoS-1 reconnect-backlog, per-subscription isolation. +# +# Device backs the topic space Device/. We create MANY device ids (Device/0 +# .. Device/N) and attach MANY concurrent MQTT subscribers: +# - WILDCARD multi-level Device/# (must get EVERY Device/) +# - WILDCARD single-level Device/+ (must get every Device/, one level) +# - SPECIFIC topic Device/7 (must get ONLY Device/7 — no cross-topic bleed) +# A burst publishes across all topics; a QoS-1 durable subscriber disconnects, +# the broker accumulates a backlog, and it reconnects to drain. A stalled +# subscriber probes per-subscription isolation (head-of-line blocking) and the +# broker's memory behavior under fan-out. +# +# value : per-publish counter / payload witness +# tag : free-form marker +type Device @table @export { + id: ID @primaryKey + value: Int + tag: String +} diff --git a/integrationTests/resources/custom-resource-contract.test.ts b/integrationTests/resources/custom-resource-contract.test.ts new file mode 100644 index 0000000000..bf0a862631 --- /dev/null +++ b/integrationTests/resources/custom-resource-contract.test.ts @@ -0,0 +1,233 @@ +/** + * Custom-Resource HTTP contract: verb routing, custom status/headers, + * error mapping, content negotiation, odd return types. + * + * Reads RAW response status + headers + body via fetch (sendOperation throws on non-200). + * Exploratory: failures are recorded into matrices and printed in `after`; only a small set + * of hard assertions guard the headline contract (verb routing + instance liveness). + */ +import { suite, test, before, after } from 'node:test'; +import { ok, strictEqual } from 'node:assert/strict'; +import { resolve } from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { setupHarperWithFixture, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing'; +// @ts-expect-error utils/client.mjs has no type declarations; runtime resolves fine +import { createApiClient } from '../apiTests/utils/client.mjs'; + +const FIXTURE_PATH = resolve(import.meta.dirname, 'custom-resource-contract'); +const skipSuite = process.platform === 'win32'; + +suite('custom-resource HTTP contract', { skip: skipSuite }, (ctx: ContextWithHarper) => { + let client: ReturnType; + let httpURL: string; + let auth: string; + + const verbMatrix: string[] = []; + const statusMatrix: string[] = []; + const errorMatrix: string[] = []; + const negotiateMatrix: string[] = []; + const oddMatrix: string[] = []; + + before(async () => { + await setupHarperWithFixture(ctx, FIXTURE_PATH, { config: {}, env: {} }); + client = createApiClient(ctx.harper); + httpURL = ctx.harper.httpURL; + auth = client.headers.Authorization; + + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + try { + const probe = await client.reqRest('/Verbs/').timeout(3_000); + if (probe.status !== 404) break; + } catch { + /* not ready */ + } + await sleep(250); + } + }); + + after(async () => { + await teardownHarper(ctx); + const block = (title: string, rows: string[]) => { + console.log(`\n[custom-resource-contract] ${title}`); + if (rows.length === 0) console.log(' (none)'); + for (const r of rows) console.log(' ' + r); + }; + block('VERB ROUTING (method-sent -> method-invoked / argc / hasReq)', verbMatrix); + block('CUSTOM STATUS + HEADERS (expected -> actual status / X-QA146 header)', statusMatrix); + block('ERROR MAPPING (case -> status / type / title)', errorMatrix); + block('CONTENT NEGOTIATION (Accept -> status / content-type / body-prefix)', negotiateMatrix); + block('ODD RETURN TYPES (case -> status / content-type / body-prefix)', oddMatrix); + }); + + async function raw( + method: string, + path: string, + opts: { body?: unknown; accept?: string; contentType?: string } = {} + ): Promise<{ status: number; ct: string; xqa: string; headers: Record; text: string; bytes: Buffer }> { + const headers: Record = { Authorization: auth }; + if (opts.accept) headers['Accept'] = opts.accept; + let body: any; + if (opts.body !== undefined) { + headers['Content-Type'] = opts.contentType || 'application/json'; + body = typeof opts.body === 'string' ? opts.body : JSON.stringify(opts.body); + } + const r = await fetch(`${httpURL}${path}`, { method, headers, body }); + const ab = Buffer.from(await r.arrayBuffer()); + const h: Record = {}; + r.headers.forEach((v, k) => (h[k] = v)); + return { + status: r.status, + ct: r.headers.get('content-type') || '', + xqa: r.headers.get('x-qa146') || '', + headers: h, + text: ab.toString('utf8'), + bytes: ab, + }; + } + + function prefix(s: string, n = 90): string { + s = (s || '').replace(/\s+/g, ' ').trim(); + return s.length > n ? s.slice(0, n) + '…' : s; + } + + // ------------------------------------------------------------------------- + // 1. VERB ROUTING + // ------------------------------------------------------------------------- + test('verb routing: each HTTP method invokes the matching resource method', async () => { + const cases: { m: string; body?: unknown }[] = [ + { m: 'GET' }, + { m: 'POST', body: { hello: 'post' } }, + { m: 'PUT', body: { hello: 'put' } }, + { m: 'PATCH', body: { hello: 'patch' } }, + { m: 'DELETE' }, + ]; + const misroutes: string[] = []; + for (const c of cases) { + const r = await raw(c.m, '/Verbs/', { body: c.body }); + let parsed: any = null; + try { + parsed = JSON.parse(r.text); + } catch { + /* non-json */ + } + const invoked = parsed?.method ?? '?'; + const dataSeen = parsed ? JSON.stringify(parsed.dataArg) : 'n/a'; + verbMatrix.push( + `${c.m.padEnd(7)} -> status=${r.status} invoked=${String(invoked).padEnd(7)} argc=${parsed?.argc} hasReq=${parsed?.hasReq} dataArg=${prefix(dataSeen, 40)}` + ); + const expected = c.m.toLowerCase(); + if (r.status === 200 && invoked !== expected) misroutes.push(`${c.m} invoked ${invoked}`); + // data-bearing verbs must see the body they were sent + if (c.body && r.status === 200 && (!parsed?.dataArg || parsed.dataArg.hello !== (c.body as any).hello)) { + misroutes.push(`${c.m} did not receive body (saw ${dataSeen})`); + } + } + strictEqual(misroutes.length, 0, `verb mis-routes / arg gaps: ${misroutes.join('; ')}`); + }); + + // ------------------------------------------------------------------------- + // 2. CUSTOM STATUS + HEADERS + // ------------------------------------------------------------------------- + test('custom status + headers are honored on the wire', async () => { + const probes: { path: string; code: number; label: string }[] = [ + { path: '/StatusCtx/?code=201', code: 201, label: 'ctx 201' }, + { path: '/StatusCtx/?code=202', code: 202, label: 'ctx 202' }, + { path: '/StatusCtx/?code=418', code: 418, label: 'ctx 418' }, + { path: '/StatusResponse/?code=201', code: 201, label: 'Response 201' }, + { path: '/StatusResponse/?code=418', code: 418, label: 'Response 418' }, + { path: '/StatusResponseData/?code=202', code: 202, label: 'Response.data 202' }, + { path: '/StatusResponseData/?code=418', code: 418, label: 'Response.data 418' }, + ]; + for (const p of probes) { + const r = await raw('GET', p.path); + const statusOK = r.status === p.code ? 'OK' : `IGNORED(want ${p.code})`; + const hdrOK = r.xqa ? `hdr=${r.xqa}` : 'HDR-MISSING'; + const fooHdr = r.headers['x-custom-foo'] ? ' x-custom-foo=' + r.headers['x-custom-foo'] : ''; + statusMatrix.push(`${p.label.padEnd(20)} -> ${r.status} ${statusOK} ${hdrOK}${fooHdr} body=${prefix(r.text, 50)}`); + } + ok(statusMatrix.length === probes.length, 'all status probes recorded'); + }); + + // ------------------------------------------------------------------------- + // 3. ERROR MAPPING + // ------------------------------------------------------------------------- + test('error mapping: thrown vs statusCode vs ClientError vs returned-error', async () => { + const probes: { path: string; label: string; expect: string }[] = [ + { path: '/ErrPlain/', label: 'throw plain Error', expect: '500 (no statusCode)' }, + { path: '/ErrStatusCode/?code=400', label: 'throw Error{.statusCode=400}', expect: '400' }, + { path: '/ErrStatusCode/?code=409', label: 'throw Error{.statusCode=409}', expect: '409' }, + { path: '/ErrClient/', label: 'throw ClientError (default)', expect: '400' }, + { path: '/ErrClient/?code=422', label: 'throw ClientError(422)', expect: '422' }, + { path: '/ErrReturned/', label: 'RETURN error-shaped {statusCode:400}', expect: '200 (not thrown)' }, + { path: '/ErrString/', label: 'throw bare string', expect: '500?' }, + ]; + for (const p of probes) { + const r = await raw('GET', p.path); + let type = ''; + let title = ''; + try { + const j = JSON.parse(r.text); + type = j.type || j.code || ''; + title = j.title || j.error || ''; + } catch { + title = prefix(r.text, 40); + } + errorMatrix.push(`${p.label.padEnd(34)} [want ${p.expect}] -> ${r.status} type=${type} title=${prefix(title, 50)}`); + } + ok(errorMatrix.length === probes.length, 'all error probes recorded'); + }); + + // ------------------------------------------------------------------------- + // 4. CONTENT NEGOTIATION on a custom return value + // ------------------------------------------------------------------------- + test('content negotiation applies to custom-resource responses', async () => { + const accepts: { label: string; header: string }[] = [ + { label: 'json', header: 'application/json' }, + { label: 'cbor', header: 'application/cbor' }, + { label: 'msgpack', header: 'application/x-msgpack' }, + { label: '*/*', header: '*/*' }, + ]; + for (const a of accepts) { + const r = await raw('GET', '/Negotiate/', { accept: a.header }); + // Detect whether body is actually binary (cbor/msgpack) vs JSON text. + const looksJson = r.text.trimStart().startsWith('{'); + const bodyDesc = looksJson ? `json:${prefix(r.text, 40)}` : `binary:${r.bytes.length}B hex=${r.bytes.subarray(0, 8).toString('hex')}`; + negotiateMatrix.push(`Accept ${a.label.padEnd(8)} -> ${r.status} ct=${r.ct.padEnd(26)} ${bodyDesc}`); + } + ok(negotiateMatrix.length === accepts.length, 'all negotiation probes recorded'); + }); + + // ------------------------------------------------------------------------- + // 5. ODD RETURN TYPES + // ------------------------------------------------------------------------- + test('odd return types serialize sanely (no blanket 500)', async () => { + const probes: { path: string; label: string }[] = [ + { path: '/RetString/', label: 'string' }, + { path: '/RetNumber/', label: 'number' }, + { path: '/RetBool/', label: 'boolean' }, + { path: '/RetNull/', label: 'null' }, + { path: '/RetUndefined/', label: 'undefined' }, + { path: '/RetArrayHuge/', label: 'huge array (50k)' }, + { path: '/RetAsyncIter/', label: 'async iterator' }, + ]; + const crashes: string[] = []; + for (const p of probes) { + const r = await raw('GET', p.path); + const len = r.bytes.length; + oddMatrix.push(`${p.label.padEnd(18)} -> ${r.status} ct=${r.ct.padEnd(26)} len=${len} body=${prefix(r.text, 50)}`); + if (r.status === 500) crashes.push(`${p.label}=500`); + } + // Record but don't hard-fail on individual odd types; the contract finding is in the matrix. + console.log(`\n[custom-resource-contract] odd-return 500s: ${crashes.length ? crashes.join(', ') : 'none'}`); + ok(oddMatrix.length === probes.length, 'all odd-return probes recorded'); + }); + + // ------------------------------------------------------------------------- + // 6. LIVENESS + // ------------------------------------------------------------------------- + test('instance still alive after all probes', async () => { + const r = await raw('GET', '/Verbs/'); + ok(r.status === 200, `instance should still answer, got ${r.status}`); + }); +}); diff --git a/integrationTests/resources/custom-resource-contract/config.yaml b/integrationTests/resources/custom-resource-contract/config.yaml new file mode 100644 index 0000000000..efffc0833f --- /dev/null +++ b/integrationTests/resources/custom-resource-contract/config.yaml @@ -0,0 +1,5 @@ +graphqlSchema: + files: '*.graphql' +jsResource: + files: resources.js +rest: true diff --git a/integrationTests/resources/custom-resource-contract/resources.js b/integrationTests/resources/custom-resource-contract/resources.js new file mode 100644 index 0000000000..b4b0480cfe --- /dev/null +++ b/integrationTests/resources/custom-resource-contract/resources.js @@ -0,0 +1,197 @@ +// QA-146 — Custom-Resource HTTP contract probe. +// +// Probes the contract app developers build on: +// - verb routing: does each HTTP method invoke the matching resource method, with right args? +// - custom status + headers honored on the wire? +// - error mapping: thrown plain Error vs error-with-.statusCode vs ClientError vs returned +// error-shaped value -> what HTTP status? +// - content negotiation on a CUSTOM return value (json/cbor/msgpack) +// - odd return types: primitive, huge array, null, undefined. +// +// Live REST path (server/REST.ts switch) calls handlers as: +// GET resource.get(target, request) +// POST resource.post(target, data, request) +// PUT resource.put(target, data, request) +// PATCH resource.patch(target, data, request) +// DELETE resource.delete(target, request) +// For loadAsInstance=false the first arg is the query/RequestTarget. +// +// Custom status: this.getContext().response.status = N, OR return a web Response. +// Custom headers: this.getContext().response.headers.set(name, value), OR Response headers. + +// --------------------------------------------------------------------------- +// VERB ROUTING — one resource overriding every verb; each records which method +// ran and what args it observed, so the test can assert correct routing. +// --------------------------------------------------------------------------- +export class Verbs extends Resource { + static loadAsInstance = false; + async get(query, request) { + return { method: 'get', argc: arguments.length, hasReq: !!request, dataArg: undefined }; + } + async post(query, data, request) { + return { method: 'post', argc: arguments.length, hasReq: !!request, dataArg: data }; + } + async put(query, data, request) { + return { method: 'put', argc: arguments.length, hasReq: !!request, dataArg: data }; + } + async patch(query, data, request) { + return { method: 'patch', argc: arguments.length, hasReq: !!request, dataArg: data }; + } + async delete(query, request) { + return { method: 'delete', argc: arguments.length, hasReq: !!request, dataArg: undefined }; + } +} + +// --------------------------------------------------------------------------- +// CUSTOM STATUS + HEADERS via getContext().response +// --------------------------------------------------------------------------- +export class StatusCtx extends Resource { + static loadAsInstance = false; + async get(query) { + const want = Number((query.get && query.get('code')) || 201); + const ctx = this.getContext(); + if (ctx?.response) { + ctx.response.status = want; + ctx.response.headers.set('X-QA146', 'ctx-' + want); + ctx.response.headers.set('X-Custom-Foo', 'bar'); + } + return { setVia: 'context', code: want }; + } +} + +// CUSTOM STATUS + HEADERS via a returned web Response object +export class StatusResponse extends Resource { + static loadAsInstance = false; + async get(query) { + const want = Number((query.get && query.get('code')) || 202); + return new Response(JSON.stringify({ setVia: 'Response', code: want }), { + status: want, + headers: { 'Content-Type': 'application/json', 'X-QA146': 'resp-' + want }, + }); + } +} + +// Response object with `data` (let Harper serialize/negotiate) + explicit status/headers +export class StatusResponseData extends Resource { + static loadAsInstance = false; + async get(query) { + const want = Number((query.get && query.get('code')) || 418); + return { status: want, headers: { 'X-QA146': 'respdata-' + want }, data: { setVia: 'Response.data', code: want } }; + } +} + +// --------------------------------------------------------------------------- +// ERROR MAPPING +// --------------------------------------------------------------------------- +// Throw a PLAIN Error (no statusCode). Intended-vs-actual: does it leak as 500? +export class ErrPlain extends Resource { + static loadAsInstance = false; + async get() { + throw new Error('QA146 plain error, no statusCode'); + } +} + +// Throw an Error carrying a .statusCode (the documented escape hatch). +export class ErrStatusCode extends Resource { + static loadAsInstance = false; + async get(query) { + const code = Number((query.get && query.get('code')) || 400); + const e = new Error('QA146 error with statusCode=' + code); + e.statusCode = code; + throw e; + } +} + +// Throw a Harper ClientError (defaults to 400 if no code passed). +export class ErrClient extends Resource { + static loadAsInstance = false; + async get(query) { + // ClientError is available in the resource sandbox? Try; fall back to statusCode error. + const code = query.get && query.get('code'); + if (typeof ClientError !== 'undefined') { + throw code ? new ClientError('QA146 ClientError', Number(code)) : new ClientError('QA146 ClientError default'); + } + const e = new Error('QA146 (ClientError unavailable, simulated)'); + e.statusCode = code ? Number(code) : 400; + throw e; + } +} + +// RETURN an error-shaped value (NOT thrown) — does the body just serialize with 200? +export class ErrReturned extends Resource { + static loadAsInstance = false; + async get() { + return { error: 'QA146 returned-but-not-thrown', statusCode: 400, code: 'BadRequest' }; + } +} + +// Throw a non-Error (string) — how is it mapped? +export class ErrString extends Resource { + static loadAsInstance = false; + async get() { + throw 'QA146 thrown string (not an Error)'; + } +} + +// --------------------------------------------------------------------------- +// CONTENT NEGOTIATION on a custom return value +// --------------------------------------------------------------------------- +export class Negotiate extends Resource { + static loadAsInstance = false; + async get() { + return { kind: 'negotiate', n: 42, list: [1, 2, 3], nested: { a: true, b: null }, s: 'héllo-ünïcode' }; + } +} + +// --------------------------------------------------------------------------- +// ODD RETURN TYPES +// --------------------------------------------------------------------------- +export class RetString extends Resource { + static loadAsInstance = false; + async get() { + return 'just a bare string'; + } +} +export class RetNumber extends Resource { + static loadAsInstance = false; + async get() { + return 1234.5; + } +} +export class RetBool extends Resource { + static loadAsInstance = false; + async get() { + return true; + } +} +export class RetNull extends Resource { + static loadAsInstance = false; + async get() { + return null; + } +} +export class RetUndefined extends Resource { + static loadAsInstance = false; + async get() { + return undefined; + } +} +export class RetArrayHuge extends Resource { + static loadAsInstance = false; + async get() { + const n = 50000; + const out = new Array(n); + for (let i = 0; i < n; i++) out[i] = { i, v: 'row-' + i }; + return out; + } +} +// Async iterator / streamed response +export class RetAsyncIter extends Resource { + static loadAsInstance = false; + async get() { + async function* gen() { + for (let i = 0; i < 5; i++) yield { i, v: 'stream-' + i }; + } + return gen(); + } +} diff --git a/integrationTests/resources/custom-resource-contract/schema.graphql b/integrationTests/resources/custom-resource-contract/schema.graphql new file mode 100644 index 0000000000..1efb163c1e --- /dev/null +++ b/integrationTests/resources/custom-resource-contract/schema.graphql @@ -0,0 +1,8 @@ +# QA-146 — Custom-Resource HTTP contract probe. +# A single backing table so we have a place to anchor exported custom Resources. +# All the interesting behavior lives in resources.js custom Resource subclasses. + +type Kv @table @export { + id: ID @primaryKey + v: String +} diff --git a/integrationTests/resources/etag-version-monotonicity.test.ts b/integrationTests/resources/etag-version-monotonicity.test.ts new file mode 100644 index 0000000000..f42f0aa282 --- /dev/null +++ b/integrationTests/resources/etag-version-monotonicity.test.ts @@ -0,0 +1,364 @@ +/** + * Version / ETag monotonicity under a same-record update storm. + * + * Hammer a SINGLE record with rapid CONCURRENT plain set()s (full-replace PUTs, + * NOT addTo). Lost-update under concurrent plain set() is EXPECTED and NOT what we + * test — we probe the integrity of the VERSION metadata that drives ETag caching + * and conditional-write ordering: + * + * M1 MONOTONICITY — across the storm, does the committed record's lastModified + * (== __updatedtime__) only ever ADVANCE (never regress) as + * observed by successive read-backs? + * M2 COLLISION-FREE — do two DISTINCT committed states ever share the same + * lastModified (=> same ETag for different data = a + * conditional-cache hazard: a client holding the old ETag + * gets 304 for changed data)? + * M3 REAL FINAL — is the final stored state one of the actually-submitted + * values (no torn / merged record)? + * M4 CROSS-SURFACE — for the SAME committed write, do the REST ETag (decoded to + * a Float64 ms), the record's __updatedtime__, and the audit + * log timestamp all agree? + * + * ETag derivation: etag is a packed encoding of the Float64 of request.lastModified. + * NOTE: the encoding is LOSSY in the most-significant float byte (byte7) — only its + * low nibble survives — so it is not a strict bijection in general. BUT byte7 is + * constant (0x42=66) for every timestamp in the current ~ms-epoch era, so within + * this era each distinct integer-ms lastModified maps to a UNIQUE etag string, and + * we can decode an etag back to lastModified exactly by fixing byte7=66. We + * cross-check both ways: decode the etag, and forward-encode __updatedtime__ and + * compare the etag STRING. + * + * Run BOTH engines x single/multi worker: + * npm run test:integration -- "integrationTests/resources/etag-version-monotonicity.test.ts" + * HARPER_STORAGE_ENGINE=lmdb npm run test:integration -- "integrationTests/resources/etag-version-monotonicity.test.ts" + */ +import { suite, test, before, after } from 'node:test'; +import { ok, equal } from 'node:assert/strict'; +import { resolve } from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { + setupHarperWithFixture, + teardownHarper, + sendOperation, + type ContextWithHarper, +} from '@harperfast/integration-testing'; + +const FIXTURE_PATH = resolve(import.meta.dirname, 'etag-version-monotonicity'); +const SCHEMA = 'data'; +const TABLE = 'Doc'; +const ENGINE = process.env.HARPER_STORAGE_ENGINE ?? 'rocksdb'; +// Worker count via QA145_WORKERS so one file covers single + multi-worker across runs. +const WORKERS = Number(process.env.QA145_WORKERS ?? '1'); +const skipSuite = process.platform === 'win32'; + +// Storm sizing. +const ROUNDS = 6; +const STORM = 250; // concurrent same-key set()s per round + +const summary: string[] = []; + +function authHeader(ctx: ContextWithHarper): string { + const { username, password } = ctx.harper.admin; + return 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64'); +} + +// Forward etag encoder — byte-for-byte mirror of server/REST.ts ETag encoding. +const _eb = new Uint8Array(8); +const _ef = new Float64Array(_eb.buffer, 0, 1); +function encodeEtag(ms: number): string { + _ef[0] = ms; + const b = _eb; + return String.fromCharCode( + 34, + (b[0] & 0x3f) + 62, + (b[0] >> 6) + ((b[1] << 2) & 0x3f) + 62, + (b[1] >> 4) + ((b[2] << 4) & 0x3f) + 62, + (b[2] >> 2) + 62, + (b[3] & 0x3f) + 62, + (b[3] >> 6) + ((b[4] << 2) & 0x3f) + 62, + (b[4] >> 4) + ((b[5] << 4) & 0x3f) + 62, + (b[5] >> 2) + 62, + (b[6] & 0x3f) + 62, + (b[6] >> 6) + ((b[7] << 2) & 0x3f) + 62, + 34 + ); +} + +/** + * Decode an ETag back to the Float64 ms it encodes. Byte7 (MSB) is lossy in the + * forward encoding but constant (0x42=66) across the current epoch era, so we fix + * it; this round-trips exactly for all in-era timestamps (verified 500k samples). + */ +function decodeEtag(etag: string | null): number | null { + if (!etag) return null; + const s = etag.replace(/"/g, ''); + if (s.length !== 10) return null; + const c = (i: number) => s.charCodeAt(i) - 62; + const dec3 = (c0: number, c1: number, c2: number, c3: number) => [ + c0 | ((c1 & 0x03) << 6), + ((c1 >> 2) & 0x0f) | ((c2 & 0x0f) << 4), + ((c2 >> 4) & 0x03) | (c3 << 2), + ]; + const g0 = dec3(c(0), c(1), c(2), c(3)); + const g1 = dec3(c(4), c(5), c(6), c(7)); + const b6 = c(8) | ((c(9) & 0x03) << 6); + const b = new Uint8Array([g0[0], g0[1], g0[2], g1[0], g1[1], g1[2], b6, 66]); + return new Float64Array(b.buffer, 0, 1)[0]; +} + +async function op(ctx: ContextWithHarper, operation: any): Promise { + return sendOperation(ctx.harper, operation); +} + +/** ops read of one row, full attributes (incl. __updatedtime__). */ +async function opsGet(ctx: ContextWithHarper, id: string): Promise | null> { + const res = await op(ctx, { + operation: 'search_by_conditions', + schema: SCHEMA, + table: TABLE, + operator: 'and', + conditions: [{ search_attribute: 'id', search_type: 'equals', search_value: id }], + get_attributes: ['*'], + }); + const rows = Array.isArray(res) ? res : []; + return rows[0] ?? null; +} + +/** Audit history for a key, oldest-first. */ +async function readHistory(ctx: ContextWithHarper, id: string): Promise { + const res = await op(ctx, { + operation: 'read_audit_log', + schema: SCHEMA, + table: TABLE, + search_type: 'hash_value', + search_values: [id], + }); + const entries = res?.[id]; + return Array.isArray(entries) ? entries : []; +} + +suite(`version/ETag monotonicity under same-record storm [engine=${ENGINE}]`, { skip: skipSuite }, (ctx: ContextWithHarper) => { + let httpURL: string; + let auth: string; + + before(async () => { + await setupHarperWithFixture(ctx, FIXTURE_PATH, { config: { threads: { count: WORKERS } }, env: {} }); + httpURL = ctx.harper.httpURL; + auth = authHeader(ctx); + // readiness poll + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + try { + const probe = await fetch(`${httpURL}/${TABLE}/`, { headers: { Authorization: auth } }); + if (probe.status !== 404) break; + } catch { + /* not ready */ + } + await sleep(250); + } + }); + + after(async () => { + console.log(`\n===== version/ETag monotonicity SUMMARY [engine=${ENGINE}, workers=${WORKERS}] =====`); + for (const line of summary) console.log(line); + console.log('=====================================================================\n'); + await teardownHarper(ctx); + }); + + // One PUT (plain full-replace set) -> { status, etag }. + async function put(key: string, value: number, tag: string): Promise<{ status: number; etag: string | null }> { + const res = await fetch(`${httpURL}/${TABLE}/${key}`, { + method: 'PUT', + headers: { Authorization: auth, 'Content-Type': 'application/json' }, + body: JSON.stringify({ id: key, value, tag }), + }); + // drain body so the socket frees + await res.text(); + return { status: res.status, etag: res.headers.get('etag') }; + } + + // REST GET -> { status, etag, body }. + async function get(key: string): Promise<{ status: number; etag: string | null; body: any }> { + const res = await fetch(`${httpURL}/${TABLE}/${key}`, { headers: { Authorization: auth, Accept: 'application/json' } }); + const body = res.status === 200 ? await res.json() : (await res.text(), null); + return { status: res.status, etag: res.headers.get('etag'), body }; + } + + test(`storm: lastModified monotonic + collision-free + real final + cross-surface [engine=${ENGINE}]`, async () => { + const key = `storm-${ENGINE}-${WORKERS}`; + const observedReads: number[] = []; // lastModified (ms) seen on successive read-backs (monotonicity) + // lastModified -> set of distinct committed (value|tag) states observed at that version (collision) + const versionStates = new Map>(); + const allSubmitted = new Set(); + + // self-test the etag codec before trusting M4 + let codecOk = false; + + for (let round = 0; round < ROUNDS; round++) { + // Fire STORM concurrent plain set()s, each with a DISTINCT value so any two + // committed states are distinguishable. + const reqs: Promise<{ status: number; etag: string | null }>[] = []; + for (let i = 0; i < STORM; i++) { + const value = round * STORM + i; // globally unique across rounds + const tag = `r${round}i${i}`; + allSubmitted.add(`${value}|${tag}`); + reqs.push(put(key, value, tag)); + } + const results = await Promise.all(reqs); + const ok2xx = results.filter((r) => r.status >= 200 && r.status < 300).length; + + // Read the committed state back from up to THREE surfaces for this round. + const g = await get(key); + const row = await opsGet(ctx, key); + const hist = await readHistory(ctx, key); + const auditTs = hist.length ? hist[hist.length - 1].timestamp : NaN; + + const etagMs = decodeEtag(g.etag); + // __updatedtime__ may live on the ops row or the REST body depending on schema; best-effort. + const utRow = typeof row?.__updatedtime__ === 'number' ? row.__updatedtime__ : NaN; + const utBody = typeof g.body?.__updatedtime__ === 'number' ? g.body.__updatedtime__ : NaN; + const ut = Number.isFinite(utRow) ? utRow : utBody; + + // codec self-check: decoded ETag must match the audit-log timestamp of the same write + // (the audit ts is the canonical lastModification; etag is derived from it). + if (etagMs != null && Number.isFinite(auditTs) && Math.abs(etagMs - auditTs) < 1e-6) codecOk = true; + + // committed state identity for this round + const state = `${g.body?.value}|${g.body?.tag}`; + + // Monotonicity/collision anchor: the ETag-decoded version (the cache surface). + const ver = etagMs ?? NaN; + if (Number.isFinite(ver)) { + observedReads.push(ver); + if (!versionStates.has(ver)) versionStates.set(ver, new Set()); + versionStates.get(ver)!.add(state); + } + + summary.push( + `[round ${round}] storm=${STORM} 2xx=${ok2xx} committed state='${state}' ` + + `etag=${g.etag} etagMs=${etagMs} auditTs=${auditTs} __updatedtime__=${Number.isFinite(ut) ? ut : 'absent'} ` + + `etag==audit=${etagMs != null && Number.isFinite(auditTs) ? Math.abs(etagMs - auditTs) < 1e-6 : 'n/a'}` + ); + } + + // ---- M1: monotonicity of the committed version across rounds ---- + let monotonic = true; + let regression = ''; + for (let i = 1; i < observedReads.length; i++) { + if (observedReads[i] < observedReads[i - 1]) { + monotonic = false; + regression = `round ${i}: ${observedReads[i]} < round ${i - 1}: ${observedReads[i - 1]}`; + break; + } + } + + // ---- M2: collision — any single version mapped to >1 distinct committed state ---- + // Round-final read samples (6) PLUS the full audit-log mining below for the strong test. + const collisions: string[] = []; + for (const [ver, states] of versionStates) { + if (states.size > 1) collisions.push(`version ${ver} -> {${[...states].join(', ')}}`); + } + + // ---- M2-strong: mine the FULL audit log (every committed write) for a timestamp shared by + // two DISTINCT record states. This is the definitive ETag-collision probe: the etag is a + // pure function of the write's timestamp, so two distinct committed images at one timestamp + // => one etag for two states. ~1500 committed writes per run across both keys touched. ---- + const fullHist = await readHistory(ctx, key); + const tsToStates = new Map>(); + let auditMonotonic = true; + let auditInversions = 0; + let maxInversionMs = 0; + let auditRegression = ''; + let prevTs = -Infinity; + for (const e of fullHist) { + const ts = e.timestamp; + if (ts < prevTs) { + auditMonotonic = false; + auditInversions++; + maxInversionMs = Math.max(maxInversionMs, prevTs - ts); + if (!auditRegression) auditRegression = `first: audit entry ts ${ts} < prior ${prevTs}`; + } + prevTs = ts; + const rec = Array.isArray(e.records) ? e.records[0] : e.records; + // committed image identity (value|tag); deletes have no record image + const img = rec ? `${rec.value}|${rec.tag}` : `<${e.operation}>`; + if (!tsToStates.has(ts)) tsToStates.set(ts, new Set()); + tsToStates.get(ts)!.add(img); + } + const auditCollisions: string[] = []; + let sharedTsCount = 0; + for (const [ts, states] of tsToStates) { + if (states.size > 1) { + auditCollisions.push(`ts ${ts} -> {${[...states].join(', ')}}`); + } + } + // How many distinct committed timestamps vs total writes (collision *pressure*). + const distinctTs = tsToStates.size; + const totalEntries = fullHist.length; + sharedTsCount = totalEntries - distinctTs; + + // ---- M3: final committed state is a real submitted value ---- + const finalGet = await get(key); + const finalState = `${finalGet.body?.value}|${finalGet.body?.tag}`; + const finalIsReal = allSubmitted.has(finalState); + + // ---- M4: cross-surface consistency on the FINAL write (ETag vs audit ts vs __updatedtime__) ---- + const finalRow = await opsGet(ctx, key); + const finalEtagMs = decodeEtag(finalGet.etag); + const hist = await readHistory(ctx, key); + const lastAuditTs = hist.length ? hist[hist.length - 1].timestamp : NaN; + // __updatedtime__ is best-effort (ops row or REST body); on this schema it may be absent. + const finalUtRow = typeof finalRow?.__updatedtime__ === 'number' ? finalRow.__updatedtime__ : NaN; + const finalUtBody = typeof finalGet.body?.__updatedtime__ === 'number' ? finalGet.body.__updatedtime__ : NaN; + const finalUt = Number.isFinite(finalUtRow) ? finalUtRow : finalUtBody; + const utPresent = Number.isFinite(finalUt); + + // Anchor: ETag-decoded ms must equal the canonical audit-log timestamp. + const etagVsAudit = finalEtagMs != null && Number.isFinite(lastAuditTs) ? Math.abs(finalEtagMs - lastAuditTs) : NaN; + const utVsAudit = utPresent && Number.isFinite(lastAuditTs) ? Math.abs(finalUt - lastAuditTs) : NaN; + // Independent string-level check: forward-encode the canonical audit ts -> must equal the served ETag. + const expectedEtag = Number.isFinite(lastAuditTs) ? encodeEtag(lastAuditTs) : null; + const etagStringMatch = expectedEtag != null && expectedEtag === finalGet.etag; + + summary.push( + `[M1 monotonic] ${monotonic}${monotonic ? '' : ' REGRESSION: ' + regression} (samples=${observedReads.length})` + ); + summary.push(`[M2 collisions/round-final] count=${collisions.length}${collisions.length ? ' :: ' + collisions.join(' ; ') : ''}`); + summary.push( + `[M2-strong audit-mined] entries=${totalEntries} distinctTimestamps=${distinctTs} ` + + `sharedTimestampWrites=${sharedTsCount} stateCollisions=${auditCollisions.length}` + + (auditCollisions.length ? ' :: ' + auditCollisions.slice(0, 5).join(' ; ') : '') + ); + summary.push( + `[M-audit-order] auditArrayMonotonic=${auditMonotonic} inversions=${auditInversions} ` + + `maxInversionMs=${maxInversionMs}${auditMonotonic ? '' : ' (' + auditRegression + ')'} ` + + `[NOTE: audit-array ordering, distinct from live-ETag M1]` + ); + summary.push(`[M3 final-real] ${finalIsReal} finalState='${finalState}'`); + summary.push( + `[M4 cross-surface] codecOk=${codecOk} finalEtagMs=${finalEtagMs} lastAuditTs=${lastAuditTs} ` + + `__updatedtime__=${utPresent ? finalUt : 'absent-on-this-schema'} |etag-audit|=${etagVsAudit} ` + + `|ut-audit|=${utPresent ? utVsAudit : 'n/a'} etagStr=${finalGet.etag} expectedEtag=${expectedEtag} ` + + `etagStringMatch=${etagStringMatch} auditEntries=${hist.length}` + ); + + // Hard assertions: the version-integrity contract. + ok(codecOk, 'ETag codec self-check failed (decoded ETag never matched the audit timestamp) — cannot trust M4'); + ok(monotonic, `MONOTONICITY VIOLATION: committed lastModified/ETag regressed (${regression})`); + // NOTE: auditArrayMonotonic is recorded but NOT asserted — sub-ms audit-array reordering of + // two near-simultaneous multi-worker writes is an ordering artifact, not a live-version + // regression. The cache-correctness contract is M1 (live ETag) + the no-collision check below. + equal(collisions.length, 0, `VERSION COLLISION (round-final): distinct committed states share one lastModified/ETag -> ${collisions.join(' ; ')}`); + equal( + auditCollisions.length, + 0, + `VERSION COLLISION (audit-mined): two distinct committed states share one timestamp/ETag -> ${auditCollisions.slice(0, 5).join(' ; ')}` + ); + ok(finalIsReal, `TORN/PHANTOM FINAL STATE: '${finalState}' was never submitted`); + // Cross-surface: ETag must agree with the canonical audit timestamp (both ms-value and string form). + ok(Number.isFinite(etagVsAudit) && etagVsAudit < 1e-6, `cross-surface: final ETag-ms (${finalEtagMs}) != audit ts (${lastAuditTs})`); + ok(etagStringMatch, `cross-surface: served ETag (${finalGet.etag}) != etag(auditTs) (${expectedEtag})`); + // If __updatedtime__ surfaced at all, it too must agree with the audit ts (else divergent surfaces). + if (utPresent) ok(utVsAudit < 2, `cross-surface: __updatedtime__ (${finalUt}) != audit ts (${lastAuditTs})`); + }); +}); diff --git a/integrationTests/resources/etag-version-monotonicity/config.yaml b/integrationTests/resources/etag-version-monotonicity/config.yaml new file mode 100644 index 0000000000..8005c9df90 --- /dev/null +++ b/integrationTests/resources/etag-version-monotonicity/config.yaml @@ -0,0 +1,3 @@ +graphqlSchema: + files: '*.graphql' +rest: true diff --git a/integrationTests/resources/etag-version-monotonicity/schema.graphql b/integrationTests/resources/etag-version-monotonicity/schema.graphql new file mode 100644 index 0000000000..5004d4d63b --- /dev/null +++ b/integrationTests/resources/etag-version-monotonicity/schema.graphql @@ -0,0 +1,13 @@ +# QA-145 — Version / ETag monotonicity under a same-record update storm. +# +# A plain @table (NOT @sealed) with audit ON so we can cross-check the audit-log +# timestamp against the REST ETag and the record's __updatedtime__. Each PUT/upsert +# of the same id is a plain set() (full replace, NOT addTo) — so lost-update under +# concurrency is EXPECTED; what we probe is the integrity of the VERSION metadata +# (lastModified -> ETag) across the storm: monotonicity, collision-freedom, +# final-state-is-a-real-submitted-value, and cross-surface consistency. +type Doc @table(audit: true) @export { + id: ID @primaryKey + value: Int + tag: String +} diff --git a/integrationTests/resources/schema-evolution.test.ts b/integrationTests/resources/schema-evolution.test.ts new file mode 100644 index 0000000000..28fe63447a --- /dev/null +++ b/integrationTests/resources/schema-evolution.test.ts @@ -0,0 +1,453 @@ +/** + * Schema evolution of @indexed / @relationship on a POPULATED table. + * + * Seed Author (parent) + Book (child) with data FIRST, then evolve the schema in + * place via set_component_file (rewrite schema.graphql) + restart_service + * http_workers, and verify the PRE-EXISTING rows behave correctly after each step. + * + * Model: Author 1──* Book (Book.authorId). Deterministic seed so every expected + * id-set is computable in-test. + * + * Evolution steps (each: edit schema -> restart -> verify over old+new rows): + * S1 ADD @indexed on Book.genre (was declared-but-unindexed) -> does search_by_value + * on the new index BACKFILL every pre-existing matching row? (index == in-test + * expected == full-scan via search_by_conditions). + * S2 ADD @relationship Author.books (to: authorId) + Book.author (from: authorId) + * referencing PRE-EXISTING rows -> do parent->child and child->parent edges + * RESOLVE over rows written before the relationship existed? + * S3 REMOVE @indexed (Book.tag) AND REMOVE @relationship (Author.tagBooks) -> + * are stale index entries / dangling edge resolutions cleaned up, or do they + * linger (ghost index hits via search_by_value on the now-unindexed attr; + * phantom edge in REST/GraphQL selecting the removed relationship)? + * S4 RETYPE Book.year String->Int over coercible ("2001") and non-coercible + * ("MMXXIV") existing values -> crash, silent data loss, or clean read? + * + * Mechanism: set_component_file (schema.graphql) + restart_service http_workers + * (re-poll readiness). sendOperation/.expect(200) throws on + * non-200 — for expected-error paths we use raw fetch / no .expect. + */ +import { suite, test, before, after } from 'node:test'; +import { ok, strictEqual } from 'node:assert/strict'; +import { resolve } from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { setupHarperWithFixture, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing'; +// @ts-expect-error utils/client.mjs has no type declarations; runtime resolves fine +import { createApiClient } from '../apiTests/utils/client.mjs'; +// @ts-expect-error utils/lifecycle.mjs has no type declarations; runtime resolves fine +import { restartHttpWorkers } from '../apiTests/utils/lifecycle.mjs'; + +const FIXTURE_PATH = resolve(import.meta.dirname, 'schema-evolution'); +const PROJECT = 'schema-evolution'; +const SCHEMA = 'data'; + +const skipSuite = process.platform === 'win32'; + +// --- deterministic seed ---------------------------------------------------------------- +const AUTHORS = 20; +const BOOKS_PER_AUTHOR = 25; // 500 books total +const GENRES = ['scifi', 'fantasy', 'mystery', 'romance', 'history']; // 5 buckets +const REGIONS = ['na', 'eu', 'apac']; + +const authorId = (a: number) => `au-${a}`; +const bookId = (a: number, b: number) => `bk-${a}-${b}`; +const genreFor = (a: number, b: number) => GENRES[(a * 7 + b) % GENRES.length]; +// tag holds an AUTHOR id so Author.tagBooks = @relationship(to: tag) actually resolves +// to a non-empty edge set BEFORE removal (gives the S3 phantom-edge probe real teeth). +// Books of author a are tagged with author ((a+1)%AUTHORS)'s id -> a clean cross-link. +const tagFor = (a: number, _b: number) => authorId((a + 1) % AUTHORS); +const yearStrFor = (a: number, b: number) => { + // Most years are coercible numeric strings; a deterministic minority are NOT. + if ((a * 3 + b) % 11 === 0) return 'MMXXIV'; // non-coercible roman-ish + return String(1990 + ((a * 5 + b) % 35)); // "1990".."2024" +}; + +// --- schema builder: toggle features on/off so each step is a controlled edit ---------- +interface SchemaOpts { + genreIndexed: boolean; // S1 + authorBooksRel: boolean; // S2 (Author.books to:authorId + Book.author from:authorId) + tagIndexed: boolean; // S3 removes + tagBooksRel: boolean; // S3 removes (Author.tagBooks to:tag) + yearInt: boolean; // S4 retype +} +function genSchema(o: SchemaOpts): string { + const authorRel = o.authorBooksRel ? '\n\tbooks: [Book] @relationship(to: authorId)' : ''; + const tagRel = o.tagBooksRel ? '\n\ttagBooks: [Book] @relationship(to: tag)' : ''; + const genreLine = `genre: String${o.genreIndexed ? ' @indexed' : ''}`; + const tagLine = `tag: ID${o.tagIndexed ? ' @indexed' : ''}`; + const yearLine = `year: ${o.yearInt ? 'Int' : 'String'}`; + const bookAuthorRel = o.authorBooksRel ? '\n\tauthor: Author @relationship(from: authorId)' : ''; + return [ + `type Author @table @export {`, + `\tid: ID @primaryKey`, + `\tname: String`, + `\tregion: String @indexed${authorRel}${tagRel}`, + `}`, + ``, + `type Book @table @export {`, + `\tid: ID @primaryKey`, + `\tauthorId: ID @indexed`, + `\t${genreLine}`, + `\t${tagLine}`, + `\t${yearLine}`, + `\ttitle: String${bookAuthorRel}`, + `}`, + ``, + ].join('\n'); +} + +// v0 == the on-disk fixture: genre NOT indexed, no author/book rel, tag indexed, +// tagBooks rel present, year is String. +const V0: SchemaOpts = { + genreIndexed: false, + authorBooksRel: false, + tagIndexed: true, + tagBooksRel: true, + yearInt: false, +}; + +suite('schema evolution (@indexed/@relationship) on populated table', { skip: skipSuite }, (ctx: ContextWithHarper) => { + let client: ReturnType; + let httpURL: string; + let auth: string; + let state: SchemaOpts = { ...V0 }; + + before(async () => { + await setupHarperWithFixture(ctx, FIXTURE_PATH, { config: { threads: { count: 1 } }, env: {} }); + client = createApiClient(ctx.harper); + httpURL = ctx.harper.httpURL; + auth = client.headers.Authorization; + + // readiness poll + const deadline = Date.now() + 90_000; + while (Date.now() < deadline) { + try { + const probe = await client.reqRest('/Book/').timeout(3_000); + if (probe.status !== 404) break; + } catch { + /* not ready */ + } + await sleep(250); + } + + // ---- seed POPULATED tables under v0 schema (BEFORE any evolution) ---- + const authors: any[] = []; + for (let a = 0; a < AUTHORS; a++) { + authors.push({ id: authorId(a), name: `author-${a}`, region: REGIONS[a % REGIONS.length] }); + } + await client.req().send({ operation: 'insert', schema: SCHEMA, table: 'Author', records: authors }).timeout(60_000).expect(200); + + const books: any[] = []; + for (let a = 0; a < AUTHORS; a++) { + for (let b = 0; b < BOOKS_PER_AUTHOR; b++) { + books.push({ + id: bookId(a, b), + authorId: authorId(a), + genre: genreFor(a, b), + tag: tagFor(a, b), + year: yearStrFor(a, b), + title: `book ${b} by ${authorId(a)}`, + }); + } + } + for (let i = 0; i < books.length; i += 200) { + await client.req().send({ operation: 'insert', schema: SCHEMA, table: 'Book', records: books.slice(i, i + 200) }).timeout(60_000).expect(200); + } + await sleep(800); // settle + }); + + after(async () => { + await teardownHarper(ctx); + }); + + // ---- helpers -------------------------------------------------------------------------- + async function evolve(next: SchemaOpts, probePath = '/Book/'): Promise { + state = next; + await client + .req() + .send({ operation: 'set_component_file', project: PROJECT, file: 'schema.graphql', payload: genSchema(next) }) + .timeout(30_000) + .expect(200); + await restartHttpWorkers(client, probePath, 120_000); + await sleep(1_500); // allow index build/backfill to settle + } + + /** search_by_value -> set of ids. Returns {ids,status} (does not assert 200). */ + async function searchByValue(table: string, attribute: string, value: unknown): Promise<{ ids: Set; status: number }> { + const r = await client + .req() + .send({ operation: 'search_by_value', schema: SCHEMA, table, search_attribute: attribute, search_value: value, get_attributes: ['id'] }) + .timeout(60_000); + const rows: any[] = Array.isArray(r.body) ? r.body : []; + return { ids: new Set(rows.map((row) => String(row.id))), status: r.status }; + } + + /** full-scan equals via search_by_conditions -> {ids,status}. Independent of secondary index. */ + async function scanEquals(table: string, attribute: string, value: unknown): Promise<{ ids: Set; status: number }> { + const r = await client + .req() + .send({ + operation: 'search_by_conditions', + schema: SCHEMA, + table, + operator: 'and', + conditions: [{ search_attribute: attribute, search_type: 'equals', search_value: value }], + get_attributes: ['id'], + }) + .timeout(60_000); + const rows: any[] = Array.isArray(r.body) ? r.body : []; + return { ids: new Set(rows.map((row) => String(row.id))), status: r.status }; + } + + async function getById(table: string, id: string): Promise { + const r = await client.req().send({ operation: 'search_by_hash', schema: SCHEMA, table, hash_values: [id], get_attributes: ['*'] }).timeout(15_000); + const rows: any[] = Array.isArray(r.body) ? r.body : []; + return rows[0]; + } + + /** raw REST GET (no throw-on-non-200). */ + async function restGet(path: string): Promise<{ status: number; body: any; raw: string }> { + const r = await fetch(`${httpURL}${path}`, { headers: { Authorization: auth } }); + const raw = await r.text(); + let body: any = null; + try { + body = JSON.parse(raw); + } catch { + /* leave null */ + } + return { status: r.status, body, raw }; + } + + const diff = (a: Set, b: Set) => [...a].filter((x) => !b.has(x)); + + // ======================================================================================== + // S1 — ADD @indexed on Book.genre (declared-but-unindexed) over POPULATED rows. + // ======================================================================================== + test('S1: ADD @indexed on Book.genre backfills every pre-existing matching row', async () => { + await evolve({ ...state, genreIndexed: true }); + + const dt = await client.req().send({ operation: 'describe_table', schema: SCHEMA, table: 'Book' }).expect(200); + const ga = (dt.body?.attributes ?? []).find((x: any) => (x.attribute ?? x.name) === 'genre'); + console.log(`[schema-evolution S1] post-ALTER genre attr=${JSON.stringify(ga)}`); + + let failures = 0; + const fails: string[] = []; + for (const g of GENRES) { + const expected = new Set(); + for (let a = 0; a < AUTHORS; a++) for (let b = 0; b < BOOKS_PER_AUTHOR; b++) if (genreFor(a, b) === g) expected.add(bookId(a, b)); + + const idx = await searchByValue('Book', 'genre', g); + const scan = await scanEquals('Book', 'genre', g); + const miss = idx.status === 200 ? diff(expected, idx.ids) : [-1 as any]; + const extra = idx.status === 200 ? diff(idx.ids, expected) : [-1 as any]; + const idxVsScan = idx.status === 200 && scan.status === 200 ? [...diff(idx.ids, scan.ids), ...diff(scan.ids, idx.ids)] : [-1 as any]; + console.log( + `[schema-evolution S1] genre=${g}: expected=${expected.size} index=${idx.ids.size}(st=${idx.status}) scan=${scan.ids.size}(st=${scan.status}) ` + + `missing=${miss.length} extra=${extra.length} idx!=scan=${idxVsScan.length}` + ); + if (idx.status !== 200 || miss.length || extra.length || idxVsScan.length) { + failures++; + fails.push(`genre=${g}: st=${idx.status} missing=${miss.length} extra=${extra.length} idxVsScan=${idxVsScan.length}`); + } + } + strictEqual(failures, 0, `S1 BACKFILL DEFECT (genre index): ${fails.join(' | ')}`); + }); + + // ======================================================================================== + // S2 — ADD @relationship referencing PRE-EXISTING rows; edges must resolve both ways. + // ======================================================================================== + test('S2: ADD @relationship Author.books / Book.author resolves over pre-existing rows', async () => { + await evolve({ ...state, authorBooksRel: true }); + + let failures = 0; + const fails: string[] = []; + + // parent -> child: Author.books must contain exactly the pre-existing books for each author. + for (let a = 0; a < AUTHORS; a++) { + const expected = new Set(); + for (let b = 0; b < BOOKS_PER_AUTHOR; b++) expected.add(bookId(a, b)); + const r = await restGet(`/Author/${authorId(a)}?select(id,books{id})`); + const obj = Array.isArray(r.body) ? r.body[0] : r.body; + const got = new Set((obj?.books ?? []).map((x: any) => String(x.id))); + const miss = diff(expected, got); + const extra = diff(got, expected); + if (r.status !== 200 || miss.length || extra.length) { + failures++; + fails.push(`Author.books ${authorId(a)}: st=${r.status} got=${got.size}/${expected.size} miss=${miss.length} extra=${extra.length}`); + if (a < 2) console.log(`[schema-evolution S2] sample Author ${authorId(a)} raw=${r.raw.slice(0, 200)}`); + } + } + + // child -> parent: Book.author must resolve to the correct pre-existing author. + for (let a = 0; a < AUTHORS; a++) { + const probe = bookId(a, a % BOOKS_PER_AUTHOR); + const r = await restGet(`/Book/${probe}?select(id,authorId,author{id,name})`); + const obj = Array.isArray(r.body) ? r.body[0] : r.body; + const resolvedAuthor = obj?.author?.id; + if (r.status !== 200 || resolvedAuthor !== authorId(a)) { + failures++; + fails.push(`Book.author ${probe}: st=${r.status} resolved=${JSON.stringify(resolvedAuthor)} expected=${authorId(a)}`); + } + } + + console.log(`[schema-evolution S2] relationship-over-old-rows failures=${failures}` + (fails.length ? `\n ${fails.slice(0, 8).join('\n ')}` : ' — all resolved')); + strictEqual(failures, 0, `S2 RELATIONSHIP-OVER-EXISTING DEFECT: ${fails.slice(0, 8).join(' | ')}`); + }); + + // ======================================================================================== + // S3 — REMOVE @indexed (Book.tag) AND @relationship (Author.tagBooks); no ghosts/phantoms. + // ======================================================================================== + test('S3: REMOVE @indexed + @relationship leaves no ghost index hits / phantom edges', async () => { + // tag == author-id cross-link: books tagged with au-T are exactly author ((T-1+N)%N)'s books. + const probeTag = authorId(1); // tag value to probe in the index (books of author 0 carry tag au-1) + const expectedForProbeTag = new Set(); + for (let a = 0; a < AUTHORS; a++) for (let b = 0; b < BOOKS_PER_AUTHOR; b++) if (tagFor(a, b) === probeTag) expectedForProbeTag.add(bookId(a, b)); + // tagBooks of author au-T (join Book.tag == au-T) -> books of author ((T-1+N)%N). + const relAuthor = authorId(1); // -> its tagBooks are author 0's books + const expectedTagBooks = new Set(); + for (let b = 0; b < BOOKS_PER_AUTHOR; b++) expectedTagBooks.add(bookId(0, b)); + + // Sanity BEFORE removal: tag index works and tagBooks relationship resolves (non-empty!). + const beforeIdx = await searchByValue('Book', 'tag', probeTag); + const beforeRel = await restGet(`/Author/${relAuthor}?select(id,tagBooks{id})`); + const beforeRelObj = Array.isArray(beforeRel.body) ? beforeRel.body[0] : beforeRel.body; + const beforeEdges = new Set((beforeRelObj?.tagBooks ?? []).map((x: any) => String(x.id))); + console.log( + `[schema-evolution S3] BEFORE remove: tag-index st=${beforeIdx.status} hits=${beforeIdx.ids.size}/${expectedForProbeTag.size} ; ` + + `tagBooks rel st=${beforeRel.status} edges=${beforeEdges.size}/${expectedTagBooks.size}` + ); + ok(beforeIdx.ids.size === expectedForProbeTag.size, `S3 precondition: tag index must resolve before removal (${beforeIdx.ids.size} vs ${expectedForProbeTag.size})`); + ok(beforeEdges.size === expectedTagBooks.size, `S3 precondition: tagBooks relationship must resolve non-empty before removal (${beforeEdges.size} vs ${expectedTagBooks.size})`); + + await evolve({ ...state, tagIndexed: false, tagBooksRel: false }); + + // describe_table: is tag still reported as indexed? + const dt = await client.req().send({ operation: 'describe_table', schema: SCHEMA, table: 'Book' }).expect(200); + const tagAttr = (dt.body?.attributes ?? []).find((x: any) => (x.attribute ?? x.name) === 'tag'); + console.log(`[schema-evolution S3] AFTER remove: describe_table tag attr=${JSON.stringify(tagAttr)}`); + + // GHOST INDEX: search_by_value on the now-unindexed tag. Two defensible outcomes: + // (a) the index is dropped -> search_by_value errors (no such index) OR + // (b) it falls back to a scan and returns the CORRECT current set. + // A DEFECT is: returns STALE/partial results that disagree with a full scan + // (search_by_conditions, which does not use the secondary index). + const ghost = await searchByValue('Book', 'tag', probeTag); + const scan = await scanEquals('Book', 'tag', probeTag); + console.log( + `[schema-evolution S3] ghost-probe search_by_value(tag=${probeTag}) st=${ghost.status} hits=${ghost.ids.size} | ` + + `scan st=${scan.status} hits=${scan.ids.size} | expected=${expectedForProbeTag.size}` + ); + // scan must remain authoritative & complete regardless. + ok(scan.status === 200, `S3: full scan over tag must still work, got st=${scan.status}`); + strictEqual(scan.ids.size, expectedForProbeTag.size, `S3: full scan over tag changed after index removal (data loss?): ${scan.ids.size} vs ${expectedForProbeTag.size}`); + if (ghost.status === 200) { + const ghostMiss = diff(expectedForProbeTag, ghost.ids); + const ghostExtra = diff(ghost.ids, expectedForProbeTag); + ok( + ghostMiss.length === 0 && ghostExtra.length === 0, + `S3 GHOST-INDEX DEFECT: search_by_value(tag) after index removal returned stale set (miss=${ghostMiss.length} extra=${ghostExtra.length})` + ); + } else { + console.log(`[schema-evolution S3] search_by_value on removed index returned st=${ghost.status} (index dropped — acceptable)`); + } + + // PHANTOM EDGE: selecting the removed tagBooks relationship (which DID resolve before removal). + const phantom = await restGet(`/Author/${relAuthor}?select(id,tagBooks{id})`); + const phantomObj = Array.isArray(phantom.body) ? phantom.body[0] : phantom.body; + const phantomEdges = phantomObj?.tagBooks; + console.log( + `[schema-evolution S3] phantom-edge select(tagBooks) st=${phantom.status} value=${JSON.stringify(phantomEdges)?.slice(0, 120)} ` + + `raw=${phantom.raw.slice(0, 160)}` + ); + // Acceptable: 200 with tagBooks absent/undefined (relationship gone), or a 4xx for unknown field. + // DEFECT: 200 that still materializes the removed edges as a populated array. + const stillPopulated = Array.isArray(phantomEdges) && phantomEdges.length > 0; + ok(!stillPopulated, `S3 PHANTOM-EDGE DEFECT: removed relationship tagBooks still resolved ${phantomEdges?.length} edges`); + + // The author row itself + plain attrs must remain readable & intact. + const authorRow = await getById('Author', relAuthor); + ok(authorRow && authorRow.region === REGIONS[1 % REGIONS.length], `S3: author row corrupted after relationship removal: ${JSON.stringify(authorRow)}`); + }); + + // ======================================================================================== + // S4 — RETYPE Book.year String->Int over coercible + non-coercible existing values. + // ======================================================================================== + test('S4: RETYPE Book.year String->Int — no crash, no silent loss on existing rows', async () => { + // Capture a coercible and a non-coercible pre-existing row id. + let coercibleId = ''; + let coercibleStr = ''; + let nonCoercibleId = ''; + outer: for (let a = 0; a < AUTHORS; a++) { + for (let b = 0; b < BOOKS_PER_AUTHOR; b++) { + const y = yearStrFor(a, b); + if (y === 'MMXXIV' && !nonCoercibleId) nonCoercibleId = bookId(a, b); + else if (y !== 'MMXXIV' && !coercibleId) { + coercibleId = bookId(a, b); + coercibleStr = y; + } + if (coercibleId && nonCoercibleId) break outer; + } + } + console.log(`[schema-evolution S4] probes: coercible=${coercibleId}("${coercibleStr}") nonCoercible=${nonCoercibleId}("MMXXIV")`); + + // Read BEFORE retype. + const beforeC = await getById('Book', coercibleId); + const beforeN = await getById('Book', nonCoercibleId); + console.log(`[schema-evolution S4] BEFORE: coercible.year=${JSON.stringify(beforeC?.year)} nonCoercible.year=${JSON.stringify(beforeN?.year)}`); + + // RETYPE may legitimately FAIL the restart (validation rejects non-coercible existing + // data) OR succeed. set_component_file uses .expect(200); restart via restartHttpWorkers + // may throw if readiness never returns — capture both. + let retypeError: string | undefined; + try { + await evolve({ ...state, yearInt: true }); + } catch (err: any) { + retypeError = err?.message ?? String(err); + } + console.log(`[schema-evolution S4] retype restart error=${JSON.stringify(retypeError ?? null)}`); + + // Whatever the migration outcome, the instance must still answer (no brick/crash-loop). + const ops = await client.req().send({ operation: 'describe_table', schema: SCHEMA, table: 'Book' }).timeout(15_000); + ok(ops.status === 200, `S4 BRICK: ops API unresponsive after retype, st=${ops.status}`); + + // Pre-existing rows must remain READABLE (no silent loss / unreadable row). + const afterC = await getById('Book', coercibleId); + const afterN = await getById('Book', nonCoercibleId); + console.log( + `[schema-evolution S4] AFTER: coercible.year=${JSON.stringify(afterC?.year)} (typeof ${typeof afterC?.year}) ` + + `nonCoercible.year=${JSON.stringify(afterN?.year)} (typeof ${typeof afterN?.year})` + ); + ok(afterC, `S4 DATA-LOSS: coercible row ${coercibleId} unreadable after retype`); + ok(afterN, `S4 DATA-LOSS: non-coercible row ${nonCoercibleId} unreadable after retype`); + // values must not have silently vanished (year still present in some form). + ok(afterC.year !== undefined && afterC.year !== null, `S4 DATA-LOSS: coercible year vanished: ${JSON.stringify(afterC)}`); + ok(afterN.year !== undefined && afterN.year !== null, `S4 DATA-LOSS: non-coercible year vanished: ${JSON.stringify(afterN)}`); + + // Total book count must be unchanged (no rows dropped by the retype). + const all = await scanEquals('Book', 'authorId', authorId(0)); // one author's books as a cheap liveness probe + console.log(`[schema-evolution S4] author0 book count after retype=${all.ids.size} (expected ${BOOKS_PER_AUTHOR})`); + strictEqual(all.ids.size, BOOKS_PER_AUTHOR, `S4 DATA-LOSS: author0 lost books after retype: ${all.ids.size} vs ${BOOKS_PER_AUTHOR}`); + + // NEW writes under the Int type: coercible-number ok; what about a string into Int? + const goodWrite = await client.req().send({ operation: 'insert', schema: SCHEMA, table: 'Book', records: [{ id: 'bk-new-int', authorId: authorId(0), year: 2030, genre: 'scifi', tag: 'tag-0', title: 'new int year' }] }).timeout(15_000); + const goodRow = await getById('Book', 'bk-new-int'); + // raw fetch for the string-into-Int write (may be rejected -> non-200). + const badResp = await fetch(`${httpURL}/Book/`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Authorization: auth }, + body: JSON.stringify({ id: 'bk-new-str', authorId: authorId(0), year: 'not-a-year', genre: 'scifi', tag: 'tag-0', title: 'bad str year' }), + }); + const badRaw = (await badResp.text()).slice(0, 200); + const badRow = await getById('Book', 'bk-new-str'); + console.log( + `[schema-evolution S4] new-write year=2030 -> op-st=${goodWrite.status} readback=${JSON.stringify(goodRow?.year)} (typeof ${typeof goodRow?.year}) ; ` + + `new-write year="not-a-year" -> rest-st=${badResp.status} readback=${JSON.stringify(badRow?.year)} raw=${badRaw}` + ); + ok(goodWrite.status === 200, `S4: a clean Int write should succeed after retype, got ${goodWrite.status} ${JSON.stringify(goodWrite.body)}`); + + console.log( + `[schema-evolution S4] VERDICT retype-restart-failed=${!!retypeError} ` + + `existing-coercible-typeof=${typeof afterC?.year} existing-noncoercible-typeof=${typeof afterN?.year} ` + + `(Harper stores values as-encoded; declared type is a validation/index hint, not a stored cast)` + ); + }); +}); diff --git a/integrationTests/resources/schema-evolution/config.yaml b/integrationTests/resources/schema-evolution/config.yaml new file mode 100644 index 0000000000..084b5e0fec --- /dev/null +++ b/integrationTests/resources/schema-evolution/config.yaml @@ -0,0 +1,4 @@ +graphqlSchema: + files: '*.graphql' +rest: true +graphql: true diff --git a/integrationTests/resources/schema-evolution/schema.graphql b/integrationTests/resources/schema-evolution/schema.graphql new file mode 100644 index 0000000000..faf084538c --- /dev/null +++ b/integrationTests/resources/schema-evolution/schema.graphql @@ -0,0 +1,32 @@ +# QA-143 — schema evolution of @indexed / @relationship on a POPULATED table. +# +# This is the v0 (initial) schema. The test rewrites it live via set_component_file +# + restart_service http_workers to evolve indexes/relationships, then verifies the +# PRE-EXISTING (already-seeded) data behaves correctly after each evolution. +# +# Model: Author 1──* Book (Book.authorId). +# +# v0 deliberately: +# - leaves Book.genre declared but NOT @indexed (step 1 ADDs @indexed -> backfill) +# - has NO Author.books / Book.author relationship (step 2 ADDs it -> resolve old rows) +# - HAS Author.region @indexed and an Author.tagBooks relationship + Book.tag @indexed +# (step 3 REMOVEs these -> ghost index / phantom edge check) +# - stores Book.year as String (step 4 RETYPEs String->Int) + +type Author @table @export { + id: ID @primaryKey + name: String + region: String @indexed + # tagBooks: a relationship present at v0 so step-3 can REMOVE it and probe phantom edges. + tagBooks: [Book] @relationship(to: tag) +} + +type Book @table @export { + id: ID @primaryKey + authorId: ID @indexed + genre: String + # tag: indexed at v0; the join key for Author.tagBooks. Step 3 REMOVEs this index. + tag: ID @indexed + year: String + title: String +} diff --git a/integrationTests/resources/sourcedfrom-write-through.test.ts b/integrationTests/resources/sourcedfrom-write-through.test.ts new file mode 100644 index 0000000000..9dbd5d065f --- /dev/null +++ b/integrationTests/resources/sourcedfrom-write-through.test.ts @@ -0,0 +1,432 @@ +/** + * sourcedFrom cache WRITE / write-through semantics. + * + * Prior waves covered the READ side of a sourcedFrom cache. This probes the WRITE + * side: when you PUT / PATCH / DELETE a record on a cache table that is sourcedFrom() + * an external origin, does the write reach the ORIGIN (write-through), only update + * the local cache (cache-only / silent divergence from the system of record), or error? + * + * The in-test origin (Node http.createServer) is the SYSTEM OF RECORD. It owns a Map + * store and a write log; every GET/PUT/DELETE it receives is recorded with op+id+value, + * so the test reads back EXACTLY which writes propagated, independent of Harper's HTTP + * response. + * + * Two cache tables share that origin but differ in the SOURCE resource (fixture resources.js): + * CacheWT — source defines get() + put() + delete() => write-through expected + * CacheRO — source defines ONLY get() => write-through impossible + * + * Matrix probed (create / update / delete) x (resident / cold) x {write-through, cache-only, rejected}, + * plus read-after-write consistency on the cache. resident = key was read from origin first (cache + * populated); cold = key was NEVER read from origin before the write. + */ +import { suite, test, before, after } from 'node:test'; +import { ok } from 'node:assert/strict'; +import { resolve } from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; +import * as http from 'node:http'; +import { setupHarperWithFixture, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing'; +// @ts-expect-error utils/client.mjs has no type declarations; runtime resolves fine +import { createApiClient } from '../apiTests/utils/client.mjs'; + +const FIXTURE_PATH = resolve(import.meta.dirname, 'sourcedfrom-write-through'); +const ORIGIN_PORT = 39147; +const skipSuite = process.platform === 'win32'; + +interface WriteLogEntry { + op: 'PUT' | 'DELETE'; + id: string; + value?: unknown; +} + +interface ResultRow { + scenario: string; + table: string; + op: string; + state: string; + httpStatus: number | string; + originWrites: string; // what the origin actually received for this id + cacheAfter: string; // what the cache returns after the write + originAfter: string; // what the origin store holds after the write + verdict: string; +} + +suite('sourcedFrom cache WRITE semantics', { skip: skipSuite }, (ctx: ContextWithHarper) => { + let client: ReturnType; + let restURL: string; + let headers: Record; + let server: http.Server; + + // ---- ORIGIN = system of record: a store + a write log, all in the test process ---- + const store = new Map(); + const writeLog: WriteLogEntry[] = []; + const getHits = new Map(); + + const results: ResultRow[] = []; + + function startServer(): Promise { + return new Promise((r) => server.listen(ORIGIN_PORT, '127.0.0.1', r)); + } + function stopServer(): Promise { + return new Promise((r) => server.close(() => r())); + } + function writesFor(id: string): WriteLogEntry[] { + return writeLog.filter((w) => w.id === id); + } + + /** Raw REST request so we see exact status without supertest throwing. */ + async function rest( + method: string, + path: string, + body?: unknown, + timeoutMs = 15_000 + ): Promise<{ status: number | string; body: string }> { + const ac = new AbortController(); + const t = setTimeout(() => ac.abort(), timeoutMs); + try { + const res = await fetch(restURL + path, { + method, + headers: { ...headers, 'content-type': 'application/json' }, + body: body == null ? undefined : JSON.stringify(body), + signal: ac.signal, + }); + const text = await res.text(); + return { status: res.status, body: text }; + } catch (e: any) { + return { status: e?.name === 'AbortError' ? 'CLIENT-ABORT' : `ERR:${e?.message}`, body: '' }; + } finally { + clearTimeout(t); + } + } + + function originSnapshot(id: string): string { + const v = store.get(id); + return v ? `value=${JSON.stringify(v.value)}` : 'ABSENT'; + } + + before(async () => { + server = http.createServer((req, res) => { + const m = /^\/item\/(.+)$/.exec(req.url ?? ''); + if (!m) { + res.writeHead(404).end('no'); + return; + } + const id = decodeURIComponent(m[1]); + const chunks: Buffer[] = []; + req.on('data', (c) => chunks.push(c as Buffer)); + req.on('end', () => { + const raw = Buffer.concat(chunks).toString('utf8'); + if (req.method === 'GET') { + getHits.set(id, (getHits.get(id) ?? 0) + 1); + const rec = store.get(id); + if (!rec) { + res.writeHead(404).end('not found'); + return; + } + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify(rec)); + return; + } + if (req.method === 'PUT') { + let parsed: any = {}; + try { + parsed = raw ? JSON.parse(raw) : {}; + } catch { + parsed = { value: raw }; + } + store.set(id, { id, value: parsed.value }); + writeLog.push({ op: 'PUT', id, value: parsed.value }); + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ id, value: parsed.value })); + return; + } + if (req.method === 'DELETE') { + store.delete(id); + writeLog.push({ op: 'DELETE', id }); + res.writeHead(204).end(); + return; + } + res.writeHead(405).end('method not allowed'); + }); + }); + await startServer(); + + await setupHarperWithFixture(ctx, FIXTURE_PATH, { + config: {}, + env: { QA147_ORIGIN_PORT: String(ORIGIN_PORT) }, + }); + client = createApiClient(ctx.harper); + restURL = client.restURL.replace(/\/$/, ''); + headers = client.headers; + + // readiness poll + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + const r = await rest('GET', '/CacheWT/__ready__', undefined, 4_000); + if (r.status === 200 || r.status === 404) break; + await sleep(250); + } + }); + + after(async () => { + console.log('\n================ sourcedFrom CACHE-WRITE SEMANTICS MATRIX ================'); + console.log( + 'scenario | table | op | state | http | origin-writes-for-id | cache-after | origin-after | verdict' + ); + console.log( + '---------------------------------+---------+--------+----------+-------+-----------------------------+------------------------+-----------------------+--------------------' + ); + for (const r of results) { + console.log( + `${r.scenario.padEnd(32)} | ${r.table.padEnd(7)} | ${r.op.padEnd(6)} | ${r.state.padEnd(8)} | ${String( + r.httpStatus + ).padEnd(5)} | ${r.originWrites.padEnd(27)} | ${r.cacheAfter.padEnd(22)} | ${r.originAfter.padEnd(21)} | ${r.verdict}` + ); + } + console.log('====================================================================\n'); + try { + await teardownHarper(ctx); + } finally { + await stopServer(); + } + }); + + /** Make a key cache-RESIDENT by seeding the origin and doing a cold read so the cache populates. */ + async function makeResident(table: string, id: string, seedValue: string) { + store.set(id, { id, value: seedValue }); + const r = await rest('GET', `/${table}/${id}`); + ok(r.status === 200, `expected residency read 200 for ${table}/${id}, got ${r.status}`); + } + + function verdictFor( + opTookEffectAtOrigin: boolean, + httpStatus: number | string, + expectOriginWrite: boolean + ): string { + const http2xx = typeof httpStatus === 'number' && httpStatus >= 200 && httpStatus < 300; + if (!http2xx) return opTookEffectAtOrigin ? 'REJECTED(but-origin-wrote!)' : 'REJECTED'; + if (opTookEffectAtOrigin) return 'WRITE-THROUGH'; + return expectOriginWrite ? 'SILENT-DIVERGENCE' : 'CACHE-ONLY'; + } + + // ============================================================ CacheWT (write-through source) + test('WT-1 CREATE cold key (never read): PUT new id -> reaches origin?', async () => { + const id = 'wt-create-cold'; + writeLog.length = 0; + const r = await rest('PUT', `/CacheWT/${id}`, { id, value: 'created-cold' }); + await sleep(150); + const cache = await rest('GET', `/CacheWT/${id}`); + const w = writesFor(id); + results.push({ + scenario: 'WT-1 create cold', + table: 'CacheWT', + op: 'PUT', + state: 'cold', + httpStatus: r.status, + originWrites: w.map((x) => x.op).join(',') || 'NONE', + cacheAfter: `${cache.status}:${cache.body.slice(0, 30)}`, + originAfter: originSnapshot(id), + verdict: verdictFor(w.some((x) => x.op === 'PUT'), r.status, true), + }); + ok(true); + }); + + test('WT-2 CREATE then READ-AFTER-WRITE consistency', async () => { + const id = 'wt-raw'; + writeLog.length = 0; + const w0 = await rest('PUT', `/CacheWT/${id}`, { id, value: 'v-raw' }); + const cache = await rest('GET', `/CacheWT/${id}`); + let consistent = false; + try { + consistent = JSON.parse(cache.body)?.value === 'v-raw'; + } catch { + /* ignore */ + } + results.push({ + scenario: 'WT-2 read-after-write', + table: 'CacheWT', + op: 'PUT', + state: 'cold', + httpStatus: w0.status, + originWrites: writesFor(id).map((x) => x.op).join(',') || 'NONE', + cacheAfter: `${cache.status}:${cache.body.slice(0, 30)}`, + originAfter: originSnapshot(id), + verdict: consistent ? 'RAW-CONSISTENT' : 'RAW-STALE/INCONSISTENT', + }); + ok(true); + }); + + test('WT-3 UPDATE resident key: PUT over a populated key -> reaches origin?', async () => { + const id = 'wt-update-resident'; + await makeResident('CacheWT', id, 'origin-seed'); + writeLog.length = 0; + const r = await rest('PUT', `/CacheWT/${id}`, { id, value: 'updated-resident' }); + await sleep(150); + const cache = await rest('GET', `/CacheWT/${id}`); + const w = writesFor(id); + results.push({ + scenario: 'WT-3 update resident', + table: 'CacheWT', + op: 'PUT', + state: 'resident', + httpStatus: r.status, + originWrites: w.map((x) => `${x.op}(${JSON.stringify(x.value)})`).join(',') || 'NONE', + cacheAfter: `${cache.status}:${cache.body.slice(0, 30)}`, + originAfter: originSnapshot(id), + verdict: verdictFor(w.some((x) => x.op === 'PUT'), r.status, true), + }); + ok(true); + }); + + test('WT-4 DELETE resident key -> deletes at origin?', async () => { + const id = 'wt-delete-resident'; + await makeResident('CacheWT', id, 'to-be-deleted'); + writeLog.length = 0; + const r = await rest('DELETE', `/CacheWT/${id}`); + await sleep(150); + const w = writesFor(id); + results.push({ + scenario: 'WT-4 delete resident', + table: 'CacheWT', + op: 'DELETE', + state: 'resident', + httpStatus: r.status, + originWrites: w.map((x) => x.op).join(',') || 'NONE', + cacheAfter: 'n/a', + originAfter: originSnapshot(id), + verdict: verdictFor(w.some((x) => x.op === 'DELETE'), r.status, true), + }); + ok(true); + }); + + test('WT-5 DELETE cold key (never read) -> deletes at origin?', async () => { + const id = 'wt-delete-cold'; + store.set(id, { id, value: 'exists-at-origin-only' }); // present at origin, never cached + writeLog.length = 0; + const r = await rest('DELETE', `/CacheWT/${id}`); + await sleep(150); + const w = writesFor(id); + results.push({ + scenario: 'WT-5 delete cold', + table: 'CacheWT', + op: 'DELETE', + state: 'cold', + httpStatus: r.status, + originWrites: w.map((x) => x.op).join(',') || 'NONE', + cacheAfter: 'n/a', + originAfter: originSnapshot(id), + verdict: verdictFor(w.some((x) => x.op === 'DELETE'), r.status, true), + }); + ok(true); + }); + + // ============================================================ CacheRO (read-only source: no put/delete) + test('RO-1 CREATE cold key on read-only source -> cache-only, rejected, or SILENT divergence?', async () => { + const id = 'ro-create-cold'; + writeLog.length = 0; + const r = await rest('PUT', `/CacheRO/${id}`, { id, value: 'ro-created' }); + await sleep(150); + const cache = await rest('GET', `/CacheRO/${id}`); + const w = writesFor(id); + results.push({ + scenario: 'RO-1 create cold (no source.put)', + table: 'CacheRO', + op: 'PUT', + state: 'cold', + httpStatus: r.status, + originWrites: w.map((x) => x.op).join(',') || 'NONE', + cacheAfter: `${cache.status}:${cache.body.slice(0, 30)}`, + originAfter: originSnapshot(id), + verdict: verdictFor(w.length > 0, r.status, true), + }); + ok(true); + }); + + test('RO-2 UPDATE resident key on read-only source -> silent divergence?', async () => { + const id = 'ro-update-resident'; + await makeResident('CacheRO', id, 'ro-origin-seed'); + writeLog.length = 0; + const r = await rest('PUT', `/CacheRO/${id}`, { id, value: 'ro-updated' }); + await sleep(150); + const cache = await rest('GET', `/CacheRO/${id}`); + let cacheVal: unknown; + try { + cacheVal = JSON.parse(cache.body)?.value; + } catch { + /* ignore */ + } + const w = writesFor(id); + const cacheShowsNew = cacheVal === 'ro-updated'; + const originStillOld = store.get(id)?.value === 'ro-origin-seed'; + results.push({ + scenario: 'RO-2 update resident (no source.put)', + table: 'CacheRO', + op: 'PUT', + state: 'resident', + httpStatus: r.status, + originWrites: w.map((x) => x.op).join(',') || 'NONE', + cacheAfter: `${cache.status}:val=${JSON.stringify(cacheVal)}`, + originAfter: originSnapshot(id), + verdict: + w.length === 0 && cacheShowsNew && originStillOld + ? 'SILENT-DIVERGENCE' + : verdictFor(w.length > 0, r.status, true), + }); + ok(true); + }); + + test('RO-3 DELETE resident key on read-only source -> origin still has it?', async () => { + const id = 'ro-delete-resident'; + await makeResident('CacheRO', id, 'ro-keep-at-origin'); + writeLog.length = 0; + const r = await rest('DELETE', `/CacheRO/${id}`); + await sleep(150); + const cache = await rest('GET', `/CacheRO/${id}`); // may re-populate from origin + const w = writesFor(id); + results.push({ + scenario: 'RO-3 delete resident (no source.delete)', + table: 'CacheRO', + op: 'DELETE', + state: 'resident', + httpStatus: r.status, + originWrites: w.map((x) => x.op).join(',') || 'NONE', + cacheAfter: `${cache.status}:${cache.body.slice(0, 30)}`, + originAfter: originSnapshot(id), + verdict: + w.length === 0 && store.has(id) + ? 'CACHE-ONLY-DELETE(origin-retains)' + : verdictFor(w.some((x) => x.op === 'DELETE'), r.status, true), + }); + ok(true); + }); + + test('RO-4 PATCH resident key on read-only source -> divergence?', async () => { + const id = 'ro-patch-resident'; + await makeResident('CacheRO', id, 'ro-patch-seed'); + writeLog.length = 0; + const r = await rest('PATCH', `/CacheRO/${id}`, { value: 'ro-patched' }); + await sleep(150); + const cache = await rest('GET', `/CacheRO/${id}`); + let cacheVal: unknown; + try { + cacheVal = JSON.parse(cache.body)?.value; + } catch { + /* ignore */ + } + const w = writesFor(id); + results.push({ + scenario: 'RO-4 patch resident (no source.patch)', + table: 'CacheRO', + op: 'PATCH', + state: 'resident', + httpStatus: r.status, + originWrites: w.map((x) => x.op).join(',') || 'NONE', + cacheAfter: `${cache.status}:val=${JSON.stringify(cacheVal)}`, + originAfter: originSnapshot(id), + verdict: + w.length === 0 && cacheVal === 'ro-patched' && store.get(id)?.value === 'ro-patch-seed' + ? 'SILENT-DIVERGENCE' + : verdictFor(w.length > 0, r.status, true), + }); + ok(true); + }); +}); diff --git a/integrationTests/resources/sourcedfrom-write-through/config.yaml b/integrationTests/resources/sourcedfrom-write-through/config.yaml new file mode 100644 index 0000000000..af1852743a --- /dev/null +++ b/integrationTests/resources/sourcedfrom-write-through/config.yaml @@ -0,0 +1,6 @@ +graphqlSchema: + files: '*.graphql' +jsResource: + files: resources.js +rest: true +graphql: true diff --git a/integrationTests/resources/sourcedfrom-write-through/resources.js b/integrationTests/resources/sourcedfrom-write-through/resources.js new file mode 100644 index 0000000000..c5ca4dc58c --- /dev/null +++ b/integrationTests/resources/sourcedfrom-write-through/resources.js @@ -0,0 +1,113 @@ +// QA-147 — source resources for CacheWT (write-through) and CacheRO (read-only get()). +// +// The in-test origin (started by the test) is the SYSTEM OF RECORD. Each source method issues +// an HTTP request to http://127.0.0.1:/item/: +// GET -> read-through population (both tables define this) +// PUT -> write-through create/update (CacheWT only) +// DELETE -> write-through delete (CacheWT only) +// +// CacheRO deliberately omits put/delete so we can observe what Harper does with a cache write +// when the source CANNOT write through (the silent-divergence candidate per QA-124's read-only +// source shape). +// +// All origin mutations are recorded server-side via the /__writes counter endpoint, so the test +// reads back EXACTLY which writes propagated (op + id + value), independent of Harper's response. + +const http = require('node:http'); + +const { CacheWT, CacheRO } = tables; + +function originPort() { + return Number(process.env.QA147_ORIGIN_PORT || 0); +} + +// Generic origin request helper. method GET/PUT/DELETE; body optional JSON. +function originRequest(method, id, body) { + return new Promise((resolve, reject) => { + const data = body == null ? null : Buffer.from(JSON.stringify(body), 'utf8'); + const req = http.request( + { + host: '127.0.0.1', + port: originPort(), + method, + path: `/item/${encodeURIComponent(id)}`, + agent: false, + headers: data ? { 'content-type': 'application/json', 'content-length': data.length } : {}, + }, + (res) => { + const chunks = []; + res.on('data', (c) => chunks.push(c)); + res.on('end', () => { + const text = Buffer.concat(chunks).toString('utf8'); + if (res.statusCode >= 200 && res.statusCode < 300) { + let parsed; + try { + parsed = text ? JSON.parse(text) : undefined; + } catch { + parsed = { value: text }; + } + resolve({ statusCode: res.statusCode, parsed }); + } else if (res.statusCode === 404) { + resolve({ statusCode: 404, parsed: undefined }); + } else { + const err = new Error(`origin ${method} returned ${res.statusCode}: ${text.slice(0, 120)}`); + err.statusCode = res.statusCode; + reject(err); + } + }); + } + ); + req.on('error', (err) => reject(err)); + if (data) req.write(data); + req.end(); + }); +} + +async function getFromOrigin(id) { + const { statusCode, parsed } = await originRequest('GET', id); + if (statusCode === 404 || !parsed) return undefined; + return { id: String(id), value: parsed.value, fetchedAt: Date.now() }; +} + +// CacheWT — full read-through + write-through (put/delete forwarded to origin). +CacheWT.sourcedFrom( + class extends Resource { + async get() { + return getFromOrigin(this.getId()); + } + // Table._writeUpdate calls the static Resource.put(id, recordUpdate, context); the static + // wrapper resolves the resource by id and invokes THIS instance method as put(record, query). + // So arg1 is the record being written; this.getId() gives the key. (resources/Resource.ts:113-134) + async put(record /*, query */) { + const id = this.getId(); + // The record arrives as the request envelope ({contentType, data:}); + // decode the body so the origin stores the real value (proves value-fidelity of write-through). + let value; + try { + if (record && record.data && (Buffer.isBuffer(record.data) || record.data.type === 'Buffer')) { + const buf = Buffer.isBuffer(record.data) ? record.data : Buffer.from(record.data.data); + value = JSON.parse(buf.toString('utf8')).value; + } else if (record && typeof record === 'object' && 'value' in record) { + value = record.value; + } else { + value = record; + } + } catch { + value = undefined; + } + await originRequest('PUT', id, { id: String(id), value }); + } + async delete(/* query */) { + await originRequest('DELETE', this.getId()); + } + } +); + +// CacheRO — read-through ONLY. No put/delete defined: write-through is impossible. +CacheRO.sourcedFrom( + class extends Resource { + async get() { + return getFromOrigin(this.getId()); + } + } +); diff --git a/integrationTests/resources/sourcedfrom-write-through/schema.graphql b/integrationTests/resources/sourcedfrom-write-through/schema.graphql new file mode 100644 index 0000000000..cd937c6389 --- /dev/null +++ b/integrationTests/resources/sourcedfrom-write-through/schema.graphql @@ -0,0 +1,22 @@ +# QA-147 — sourcedFrom cache WRITE / write-through semantics. +# +# Two cache tables share one in-test HTTP origin but differ in what the SOURCE resource +# implements (resources.js): +# CacheWT — source defines get() + put() + delete() => expect WRITE-THROUGH to origin +# CacheRO — source defines ONLY get() => write-through impossible; probe +# whether a cache write is cache-only/silent-divergence or rejected. +# +# expiration is long (3600s) so a read-after-write does NOT trigger a refresh that could mask +# a stale local value — we want to see exactly what the write left in the cache. + +type CacheWT @table(expiration: 3600) @export { + id: ID @primaryKey + value: String + fetchedAt: Float +} + +type CacheRO @table(expiration: 3600) @export { + id: ID @primaryKey + value: String + fetchedAt: Float +} diff --git a/integrationTests/server/subscription-across-redeploy.test.ts b/integrationTests/server/subscription-across-redeploy.test.ts new file mode 100644 index 0000000000..d987b6377b --- /dev/null +++ b/integrationTests/server/subscription-across-redeploy.test.ts @@ -0,0 +1,556 @@ +/** + * Live-tail subscription correctness across a mid-subscription schema + * change / component redeploy. + * + * Open SSE (REST CONNECT) AND MQTT subscriptions to a table, then WHILE subscribed + * redeploy the component: add a `note` attribute + `@indexed seq` to schema.graphql + * and rewrite resources.js (the Writer now stamps `note`), via set_component_file + + * restart_service http_workers. + * + * The redeploy recycles the HTTP workers. The questions: + * - Does the live subscription SURVIVE (stay open + keep delivering), DROP cleanly + * (client sees disconnect/EOF), or get STUCK (silently dead, no events, no error)? + * - Are events written DURING and AFTER the swap delivered with the NEW record shape + * (the added `note` attribute present)? + * - Across the restart boundary: any DUPLICATE / DROPPED / mis-ordered events? + * - If the subscription drops, is a ?startTime= resume across the boundary consistent + * (no gap, no dup)? + * + * Each published event carries a unique, monotonically increasing `seq`; we track the + * client-side delivered seq set per transport to detect dup/drop/misorder across the + * restart boundary. We note whether the SSE socket EOFs and whether MQTT emits 'close'. + * + * NOTE: a Table SSE subscription withholds response headers until the initial query + * yields >= 1 row — so we SEED a row before opening SSE, otherwise openSse never + * resolves. (MQTT has no such constraint but we seed for symmetry.) + * + * Run single-worker (deterministic teardown/replacement of the only worker -> expected + * socket gap) AND multi-worker (rolling restart keeps peers serving) to see whether the + * transports behave differently across the recycle. + */ +import { suite, test, before, after } from 'node:test'; +import { ok, strictEqual } from 'node:assert/strict'; +import { resolve } from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; +import http from 'node:http'; +import https from 'node:https'; +import { URL } from 'node:url'; + +import request from 'supertest'; +import mqtt, { type IClientOptions, type MqttClient } from 'mqtt'; + +import { setupHarperWithFixture, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing'; +// @ts-expect-error utils/client.mjs has no type declarations; runtime resolves fine +import { createApiClient } from '../apiTests/utils/client.mjs'; +// @ts-expect-error utils/lifecycle.mjs has no type declarations; runtime resolves fine +import { restartHttpWorkers } from '../apiTests/utils/lifecycle.mjs'; + +const FIXTURE_PATH = resolve(import.meta.dirname, 'subscription-across-redeploy'); +const PROJECT = 'subscription-across-redeploy'; + +const skipSuite = process.env.HARPER_RUNTIME === 'bun' || process.platform === 'win32'; + +const ADMIN_USER = 'admin'; +const ADMIN_PASS = 'Abc1234!'; + +// ---- v2 component: schema adds `note` + @indexed seq; Writer stamps `note` ---------------- +const SCHEMA_V2 = `type Event @table @export { + id: ID @primaryKey + seq: Int @indexed + value: Int + note: String +} +`; +const RESOURCES_V2 = `export class Writer extends Resource { + static loadAsInstance = false; + async post(query, body) { + const { id, seq, value } = body; + await tables.Event.put({ id, seq, value, note: 'v2-shape' }); + return { version: 'v2', wrote: await tables.Event.get(id) }; + } +} +`; + +// ---------------- SSE plumbing (raw socket so we can detect EOF) --------------------------- +interface SseEvent { + data?: string; + id?: string; + at: number; +} +interface SseStream { + events: SseEvent[]; + status: number; + contentType: string; + closed: boolean; // server EOF'd the response stream + closedAt: number; // ms epoch of EOF, 0 if still open + errored: boolean; + destroy: () => void; +} + +function openSse(urlStr: string, headers: Record): Promise { + const url = new URL(urlStr); + const lib = url.protocol === 'https:' ? https : http; + const events: SseEvent[] = []; + let buffer = ''; + const parseFrame = (frame: string, stream: SseStream) => { + if (!frame.trim()) return; + const ev: SseEvent = { at: Date.now() }; + for (const line of frame.split('\n')) { + const idx = line.indexOf(':'); + if (idx < 0) continue; + const field = line.slice(0, idx); + const val = line.slice(idx + 1).replace(/^ /, ''); + if (field === 'data') ev.data = (ev.data ? ev.data + '\n' : '') + val; + else if (field === 'id') ev.id = val; + } + stream.events.push(ev); + }; + return new Promise((resolve, reject) => { + const req = lib.request( + url, + { method: 'GET', headers: { ...headers, Accept: 'text/event-stream' }, rejectUnauthorized: false } as any, + (res) => { + const stream: SseStream = { + events, + status: res.statusCode ?? 0, + contentType: res.headers['content-type'] ?? '', + closed: false, + closedAt: 0, + errored: false, + destroy: () => { + try { + res.destroy(); + req.destroy(); + } catch { + /* ignore */ + } + }, + }; + res.setEncoding('utf8'); + res.on('data', (chunk: string) => { + buffer += chunk; + let sep: number; + while ((sep = buffer.indexOf('\n\n')) >= 0) { + parseFrame(buffer.slice(0, sep), stream); + buffer = buffer.slice(sep + 2); + } + }); + res.on('end', () => { + if (!stream.closed) { + stream.closed = true; + stream.closedAt = Date.now(); + } + }); + res.on('close', () => { + if (!stream.closed) { + stream.closed = true; + stream.closedAt = Date.now(); + } + }); + res.on('error', () => { + stream.errored = true; + }); + resolve(stream); + } + ); + req.on('error', reject); + req.end(); + }); +} + +/** Pull the seq we stamped out of an SSE native-message envelope (envelope.value.seq). */ +function sseSeq(ev: SseEvent): number | undefined { + if (!ev.data) return undefined; + try { + const obj = JSON.parse(ev.data); + const inner = obj?.value; + if (inner && typeof inner === 'object' && typeof inner.seq === 'number') return inner.seq; + if (typeof obj?.seq === 'number') return obj.seq; + } catch { + /* not json */ + } + return undefined; +} +function sseHasNote(ev: SseEvent): boolean { + if (!ev.data) return false; + try { + const obj = JSON.parse(ev.data); + const inner = obj?.value; + return Boolean(inner && typeof inner === 'object' && inner.note != null); + } catch { + return false; + } +} +function sseLocalTime(ev: SseEvent): number | undefined { + if (!ev.data) return undefined; + try { + const obj = JSON.parse(ev.data); + if (typeof obj?.localTime === 'number') return obj.localTime; + if (typeof obj?.version === 'number') return obj.version; + } catch { + /* ignore */ + } + return undefined; +} + +// ---------------- MQTT plumbing ----------------------------------------------------------- +function baseOpts(overrides: Partial = {}): IClientOptions { + return { + protocolVersion: 5, + reconnectPeriod: 0, + connectTimeout: 8000, + clean: true, + username: ADMIN_USER, + password: ADMIN_PASS, + ...overrides, + }; +} +function connect(url: string, opts: IClientOptions): Promise { + return new Promise((resolve, reject) => { + const client = mqtt.connect(url, opts); + const onError = (err: Error) => { + client.removeListener('connect', onConnect); + client.end(true); + reject(err); + }; + const onConnect = () => { + client.removeListener('error', onError); + resolve(client); + }; + client.once('error', onError); + client.once('connect', onConnect); + }); +} +function subscribe(client: MqttClient, topic: string, qos: 0 | 1 | 2 = 1): Promise { + return new Promise((resolve, reject) => { + client.subscribe(topic, { qos }, (err) => (err ? reject(err) : resolve())); + }); +} +function endQuiet(client: MqttClient | undefined): Promise { + return new Promise((resolve) => { + if (!client) return resolve(); + client.end(true, {}, () => resolve()); + }); +} +interface MqttObserver { + seqs: Array<{ seq: number; note: boolean; at: number }>; + closes: number; + offlines: number; + stop: () => void; +} +function observeMqtt(client: MqttClient): MqttObserver { + const seqs: Array<{ seq: number; note: boolean; at: number }> = []; + const onMsg = (_topic: string, payload: Buffer) => { + try { + const obj = JSON.parse(payload.toString()); + if (typeof obj?.seq === 'number') seqs.push({ seq: obj.seq, note: obj.note != null, at: Date.now() }); + } catch { + /* ignore */ + } + }; + const obs: MqttObserver = { seqs, closes: 0, offlines: 0, stop: () => {} }; + const onClose = () => obs.closes++; + const onOffline = () => obs.offlines++; + client.on('message', onMsg); + client.on('close', onClose); + client.on('offline', onOffline); + obs.stop = () => { + client.removeListener('message', onMsg); + client.removeListener('close', onClose); + client.removeListener('offline', onOffline); + }; + return obs; +} + +async function waitFor(predicate: () => boolean, timeoutMs = 8000, intervalMs = 25): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate()) return true; + await sleep(intervalMs); + } + return predicate(); +} + +// Analyze a delivered seq list against the set of seqs we PUBLISHED, scoped to our test +// (other tests/keys can't collide because every test uses a unique id + seq base). +function continuity(delivered: number[], published: number[]) { + const counts = new Map(); + for (const s of delivered) counts.set(s, (counts.get(s) ?? 0) + 1); + const missing = published.filter((s) => !counts.has(s)); + const dups = published.filter((s) => (counts.get(s) ?? 0) > 1); + // first-seen order over the published set only + const firstSeen: number[] = []; + const seen = new Set(); + for (const s of delivered) { + if (published.includes(s) && !seen.has(s)) { + seen.add(s); + firstSeen.push(s); + } + } + const ascending = firstSeen.every((v, i) => i === 0 || v > firstSeen[i - 1]); + return { missing, dups, firstSeen, ascending }; +} + +function runSuite(label: string, workers: number) { + suite(`subscription across redeploy [${label}]`, { skip: skipSuite }, (ctx: ContextWithHarper) => { + let client: ReturnType; + let restBase = ''; + let mqttURL = ''; + let authHeaders: Record = {}; + let mqttUsable = false; + let mqttSkipReason = ''; + const openStreams = new Set(); + const openMqtt = new Set(); + + const restPost = (body: object) => + request(restBase).post('/Writer/').set(client.headers).send(body); + + before(async () => { + await setupHarperWithFixture(ctx, FIXTURE_PATH, { config: { threads: { count: workers } }, env: {} }); + client = createApiClient(ctx.harper); + restBase = client.restURL; + authHeaders = { Authorization: client.headers.Authorization as string }; + const wsScheme = restBase.startsWith('https') ? 'wss' : 'ws'; + mqttURL = `${restBase.replace(/^https?/, wsScheme)}/mqtt`; + + // Poll until the custom Writer route is actually serving (not just the Event + // table) — the seed POST goes to /Writer/, which registers slightly after the + // table is initialized. + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + try { + const probe = await request(restBase).post('/Writer/').set(client.headers).send({}).timeout(3_000); + // 404 = route not registered yet; anything else (200/4xx/5xx) = serving. + if (probe.status !== 404) break; + } catch { + /* not ready */ + } + await sleep(250); + } + + try { + const probe = await connect(mqttURL, baseOpts({ clientId: `${label}-probe` })); + mqttUsable = probe.connected === true; + await endQuiet(probe); + } catch (err) { + mqttUsable = false; + mqttSkipReason = `MQTT connect probe failed on ${mqttURL}: ${(err as Error)?.message}`; + console.log(`[sub-redeploy][${label}] HARNESS: ${mqttSkipReason}`); + } + }); + + after(async () => { + for (const s of openStreams) s.destroy(); + openStreams.clear(); + for (const c of openMqtt) await endQuiet(c); + openMqtt.clear(); + await teardownHarper(ctx); + }); + + async function setComponentFile(file: string, payload: string) { + await client + .req() + .send({ operation: 'set_component_file', project: PROJECT, file, payload }) + .timeout(30_000) + .expect(200); + } + + // Main experiment: subscribe(SSE+MQTT) -> pre-writes -> redeploy -> during/post writes, + // then characterize survival + continuity + new-shape delivery per transport. + test('SSE & MQTT live subscriptions across a mid-subscription redeploy', async () => { + const id = `e-${label}-${Date.now()}`; // ONE key; same-key updates so subs see a stream + let nextSeq = 1; + const published: number[] = []; + const write = async (note?: string) => { + const seq = nextSeq++; + const r = await restPost({ id, seq, value: seq * 10 }); + published.push(seq); + return { seq, status: r.status, version: r.body?.version, note: r.body?.wrote?.note }; + }; + + // --- SEED a row first (Table SSE withholds headers until >=1 row) --- + const seed = await restPost({ id, seq: 0, value: 0 }); + strictEqual(seed.status, 200, `seed write should be 200: ${seed.status}`); + + // --- Open BOTH subscriptions on the live (v1) component --- + const sse = await openSse(`${restBase}/Event/${id}`, authHeaders); + openStreams.add(sse); + ok(sse.status >= 200 && sse.status < 300, `SSE should open 2xx, got ${sse.status} ${sse.contentType}`); + ok(sse.contentType.includes('text/event-stream'), `SSE content-type, got ${sse.contentType}`); + + let mqttObs: MqttObserver | undefined; + let mqttClient: MqttClient | undefined; + if (mqttUsable) { + mqttClient = await connect(mqttURL, baseOpts({ clientId: `${label}-sub` })); + openMqtt.add(mqttClient); + mqttObs = observeMqtt(mqttClient); + await subscribe(mqttClient, `Event/${id}`, 1); + } + await sleep(400); // let both subscriptions attach + + // --- PRE-swap writes (v1 shape: no note) --- + const preWrites = []; + for (let i = 0; i < 4; i++) preWrites.push(await write()); + const lastPreSeq = nextSeq - 1; + // wait for pre-writes to land on SSE so we have a resume token from before the swap + await waitFor(() => sse.events.some((e) => sseSeq(e) === lastPreSeq), 8000); + const resumeEv = [...sse.events].reverse().find((e) => sseSeq(e) != null); + const resumeLocalTime = resumeEv ? sseLocalTime(resumeEv) : undefined; + const sseClosedBeforeSwap = sse.closed; + + // --- THE SWAP: rewrite schema + resources, recycle workers --- + await setComponentFile('schema.graphql', SCHEMA_V2); + await setComponentFile('resources.js', RESOURCES_V2); + const swapAt = Date.now(); + await restartHttpWorkers(client, '/Event/', 120_000); + const readyAt = Date.now(); + + // give the recycled workers / sockets a moment to settle, observe SSE EOF if any + await sleep(800); + const sseEofWithinWindow = sse.closed; + const sseEofRelToReady = sse.closedAt ? sse.closedAt - readyAt : null; + + // --- POST-swap writes (v2 shape: note present). Verify handler is v2. --- + const postWrites = []; + for (let i = 0; i < 4; i++) postWrites.push(await write()); + const lastPostSeq = nextSeq - 1; + const v2HandlerActive = postWrites.every((w) => w.version === 'v2'); + const v2NotePersisted = postWrites.every((w) => w.note === 'v2-shape'); + + // Allow time for post writes to flow to whatever is still attached. + await sleep(1500); + + // =========================== SSE characterization =========================== + const sseDelivered = sse.events.map(sseSeq).filter((v): v is number => typeof v === 'number'); + const ssePre = continuity(sseDelivered, preWrites.map((w) => w.seq)); + const ssePost = continuity(sseDelivered, postWrites.map((w) => w.seq)); + const sseFull = continuity(sseDelivered, published); + const ssePostDeliveredLive = ssePost.missing.length === 0; + const ssePostNoteShape = sse.events.filter((e) => postWrites.some((w) => w.seq === sseSeq(e))).every(sseHasNote); + // Timing: did the last post-swap event arrive on the wire BEFORE the socket EOF'd? + const lastPostEv = [...sse.events].reverse().find((e) => sseSeq(e) === lastPostSeq); + const lastPostDeliveredAt = lastPostEv?.at ?? 0; + const deliveredBeforeEof = ssePostDeliveredLive && (!sse.closed || sse.closedAt === 0 || lastPostDeliveredAt <= sse.closedAt); + + let sseVerdict: string; + if (!sse.closed && ssePostDeliveredLive) sseVerdict = 'SURVIVES (open + delivered all post-swap live)'; + else if (sse.closed && ssePostDeliveredLive && deliveredBeforeEof) + sseVerdict = 'SURVIVED-THEN-EOF (all post-swap delivered live, socket EOF after; client reconnects)'; + else if (sse.closed && ssePost.missing.length === postWrites.length) sseVerdict = 'CLEAN-DROP (EOF, no post events)'; + else if (sse.closed) sseVerdict = 'DROPPED (EOF, partial post delivery)'; + else if (!sse.closed && ssePost.missing.length === postWrites.length) sseVerdict = 'STUCK-DEAD (open but silent, no post events)'; + else sseVerdict = 'PARTIAL (open, partial post delivery)'; + + // Always probe a ?startTime= resume across the boundary (whether or not the live + // stream got everything) — resume FROM the last delivered pre-swap event should + // catch up the post-swap writes with no gap and no dup. + let sseResumeVerdict = 'n/a (no resume token)'; + let sseResumeDelivered: number[] = []; + if (resumeLocalTime != null) { + const s2 = await openSse(`${restBase}/Event/${id}?startTime=${resumeLocalTime}`, authHeaders); + openStreams.add(s2); + await sleep(2500); + sseResumeDelivered = s2.events.map(sseSeq).filter((v): v is number => typeof v === 'number'); + const resumeCont = continuity(sseResumeDelivered, postWrites.map((w) => w.seq)); + // NOTE: on a SINGLE-KEY record subscription, ?startTime= replay coalesces + // same-key updates to the LATEST version — so a resume that returns only the + // final seq (not every intermediate) is EXPECTED coalescing, not a gap. + sseResumeVerdict = + resumeCont.missing.length === 0 + ? `RESUMABLE (?startTime= replayed all post-swap; dups=${JSON.stringify(resumeCont.dups)})` + : sseResumeDelivered.includes(lastPostSeq) + ? `RESUME-COALESCED-TO-LATEST (got ${JSON.stringify(sseResumeDelivered)}; same-key coalescing, expected — live stream got all)` + : `RESUME-GAP (missing post-swap seqs=${JSON.stringify(resumeCont.missing)}; got=${JSON.stringify(sseResumeDelivered)})`; + s2.destroy(); + } + + console.log( + `\n[sub-redeploy][${label}] ===== SSE =====\n` + + ` swap@+0 ready@+${readyAt - swapAt}ms\n` + + ` delivered seqs = ${JSON.stringify(sseDelivered)}\n` + + ` pre-swap missing/dups = ${JSON.stringify(ssePre.missing)} / ${JSON.stringify(ssePre.dups)}\n` + + ` post-swap missing/dups = ${JSON.stringify(ssePost.missing)} / ${JSON.stringify(ssePost.dups)}\n` + + ` full first-seen ascending = ${sseFull.ascending}\n` + + ` socket EOF? before/after swap= ${sseClosedBeforeSwap} / ${sse.closed} (eof rel ready=${sse.closedAt ? sse.closedAt - readyAt : sseEofRelToReady}ms)\n` + + ` all post delivered b/4 EOF = ${deliveredBeforeEof}\n` + + ` post events carry note shape = ${ssePostNoteShape}\n` + + ` ?startTime= resume boundary = ${sseResumeVerdict}\n` + + ` => SSE ${sseVerdict}` + ); + + // =========================== MQTT characterization =========================== + let mqttVerdict = 'SKIPPED (harness MQTT unusable)'; + if (mqttObs) { + const mDelivered = mqttObs.seqs.map((s) => s.seq); + const mPre = continuity(mDelivered, preWrites.map((w) => w.seq)); + const mPost = continuity(mDelivered, postWrites.map((w) => w.seq)); + const mFull = continuity(mDelivered, published); + const mPostLive = mPost.missing.length === 0; + const mPostNoteShape = mqttObs.seqs.filter((s) => postWrites.some((w) => w.seq === s.seq)).every((s) => s.note); + const mqttConnected = mqttClient?.connected === true; + + if (mqttConnected && mPostLive) mqttVerdict = 'SURVIVES (connected + delivered all post-swap live)'; + else if (!mqttConnected && mPostLive) + mqttVerdict = 'SURVIVED-THEN-CLOSE (all post-swap delivered live, WS closed after; client reconnects)'; + else if (!mqttConnected && mPost.missing.length === postWrites.length) mqttVerdict = 'CLEAN-DROP (close, no post events)'; + else if (!mqttConnected) mqttVerdict = 'DROPPED (close, partial post)'; + else if (mqttConnected && mPost.missing.length === postWrites.length) mqttVerdict = 'STUCK-DEAD (connected but silent post-swap)'; + else mqttVerdict = 'PARTIAL (connected, partial post)'; + + console.log( + `\n[sub-redeploy][${label}] ===== MQTT =====\n` + + ` delivered seqs = ${JSON.stringify(mDelivered)}\n` + + ` pre-swap missing/dups = ${JSON.stringify(mPre.missing)} / ${JSON.stringify(mPre.dups)}\n` + + ` post-swap missing/dups = ${JSON.stringify(mPost.missing)} / ${JSON.stringify(mPost.dups)}\n` + + ` full first-seen ascending = ${mFull.ascending}\n` + + ` client connected after swap = ${mqttConnected} (close events=${mqttObs.closes} offline=${mqttObs.offlines})\n` + + ` post events carry note shape = ${mPostNoteShape}\n` + + ` => MQTT ${mqttVerdict}` + ); + mqttObs.stop(); + } else { + console.log(`\n[sub-redeploy][${label}] ===== MQTT ===== SKIPPED: ${mqttSkipReason}`); + } + + // ---- Cross-cutting assertions on what SHOULD hold regardless of survive/drop ---- + // 1. The redeploy must actually have taken effect (v2 handler + new attr durable). + ok(v2HandlerActive, `post-swap writes should hit the v2 handler: ${JSON.stringify(postWrites)}`); + ok(v2NotePersisted, `new 'note' attribute must persist post-swap: ${JSON.stringify(postWrites)}`); + + // 2. SSE correctness invariants: + // - never silently STUCK-DEAD (open but no post events, no error) — the worst find; + // - no mis-order across the boundary; no duplicate delivery; + // - any post-swap event that WAS delivered must carry the new 'note' shape. + // SURVIVED-THEN-EOF (full live delivery then a late socket close) is DEFENSIBLE. + ok(sseVerdict !== 'STUCK-DEAD (open but silent, no post events)', `SSE STUCK-DEAD defect: ${sseVerdict}`); + ok(sseFull.ascending, `SSE delivered order must be ascending across the boundary: ${JSON.stringify(sseDelivered)}`); + strictEqual(ssePre.dups.length, 0, `SSE must not duplicate pre-swap events: ${JSON.stringify(ssePre.dups)}`); + strictEqual(ssePost.dups.length, 0, `SSE must not duplicate post-swap events: ${JSON.stringify(ssePost.dups)}`); + ok(ssePostNoteShape, `any delivered post-swap SSE event must carry the new 'note' shape`); + // If the LIVE stream actually lost post-swap events (genuine drop/partial, not a + // post-delivery EOF), then ?startTime= resume MUST cleanly cover the gap. + if (!ssePostDeliveredLive) { + ok( + sseResumeVerdict.startsWith('RESUMABLE'), + `SSE lost post-swap events live AND ?startTime= did not resume them: ${sseResumeVerdict}` + ); + } + + // 3. MQTT (if usable): no silent stuck-dead, no dup, ascending order, new shape on + // any delivered post-swap event. SURVIVED-THEN-CLOSE is defensible like SSE. + if (mqttObs) { + const mDelivered = mqttObs.seqs.map((s) => s.seq); + const mPostDup = continuity(mDelivered, postWrites.map((w) => w.seq)).dups; + const mAscending = continuity(mDelivered, published).ascending; + const mPostNote = mqttObs.seqs.filter((s) => postWrites.some((w) => w.seq === s.seq)).every((s) => s.note); + ok(!mqttVerdict.startsWith('STUCK-DEAD'), `MQTT STUCK-DEAD defect: ${mqttVerdict}`); + ok(mAscending, `MQTT delivered order must be ascending across the boundary: ${JSON.stringify(mDelivered)}`); + strictEqual(mPostDup.length, 0, `MQTT must not duplicate post-swap events: ${JSON.stringify(mPostDup)}`); + ok(mPostNote, `any delivered post-swap MQTT event must carry the new 'note' shape`); + } + }); + }); +} + +runSuite('single-worker', 1); +runSuite('multi-worker', 4); diff --git a/integrationTests/server/subscription-across-redeploy/config.yaml b/integrationTests/server/subscription-across-redeploy/config.yaml new file mode 100644 index 0000000000..3f729348c9 --- /dev/null +++ b/integrationTests/server/subscription-across-redeploy/config.yaml @@ -0,0 +1,6 @@ +graphqlSchema: + files: '*.graphql' +jsResource: + files: resources.js +rest: true +mqtt: true diff --git a/integrationTests/server/subscription-across-redeploy/resources.js b/integrationTests/server/subscription-across-redeploy/resources.js new file mode 100644 index 0000000000..1228ce23e8 --- /dev/null +++ b/integrationTests/server/subscription-across-redeploy/resources.js @@ -0,0 +1,18 @@ +// QA-153 — live-tail subscription correctness across a mid-subscription schema +// change / component redeploy (v1 resources). +// +// `Event` is exported as an automatic table resource (no custom Resource needed +// for the subscription path — SSE CONNECT and MQTT both attach to Table.subscribe). +// A tiny custom `Writer` Resource lets the test write rows via REST POST using +// only v1 attributes (id, seq, value). After the v2 redeploy adds `note`, the v2 +// Writer additionally stamps `note`, so the test can assert the NEW shape is +// delivered to the still/again-attached subscribers. + +export class Writer extends Resource { + static loadAsInstance = false; + async post(query, body) { + const { id, seq, value } = body; + await tables.Event.put({ id, seq, value }); + return { version: 'v1', wrote: await tables.Event.get(id) }; + } +} diff --git a/integrationTests/server/subscription-across-redeploy/schema.graphql b/integrationTests/server/subscription-across-redeploy/schema.graphql new file mode 100644 index 0000000000..891c549ade --- /dev/null +++ b/integrationTests/server/subscription-across-redeploy/schema.graphql @@ -0,0 +1,14 @@ +# QA-153 — live-tail subscription correctness across a mid-subscription schema +# change / component redeploy (v1 schema). +# +# Event is a plain table the test subscribes to over BOTH SSE (REST CONNECT) and +# MQTT. The redeploy probe rewrites this file via set_component_file to add a +# `note` attribute and `@indexed seq`, then restart_service http_workers recycles +# the workers. The test watches whether the live subscription survives, drops +# cleanly, or goes stuck-dead, and whether post-swap events carry the new shape. + +type Event @table @export { + id: ID @primaryKey + seq: Int + value: Int +} diff --git a/integrationTests/server/websocket-transport.test.ts b/integrationTests/server/websocket-transport.test.ts new file mode 100644 index 0000000000..16c82f1083 --- /dev/null +++ b/integrationTests/server/websocket-transport.test.ts @@ -0,0 +1,458 @@ +/** + * WebSocket subscription transport: delivery, disconnect teardown, backpressure. + * + * Harper exposes table subscriptions over WebSocket. The WS subscribe URL is the + * SAME REST path (e.g. /Burst/ collection or /Burst/ record), upgraded to + * WebSocket; the connection itself IS the subscription. We send Authorization + + * Content-Type: application/json on the upgrade request so frames arrive as JSON + * change envelopes: {id, localTime, value:, version, type}. + * + * QUESTIONS: + * Q1 SAME-KEY BURST: fire N rapid PUTs to ONE key; subscriber must see monotonic + * (no reorder), no value past final, last==final. (Coalescing is legitimate; + * we assert order.) + * Q2 MANY-KEY BURST: one PUT to each of K distinct keys; every key delivered + * EXACTLY ONCE, no drop/dup. (Parity with QA-140's exactly-once SSE/MQTT.) + * Q3 DISCONNECT TEARDOWN: open a WS sub, receive an event, then HARD disconnect + * the client mid-stream. Does the subscription-layer teardown fire PROMPTLY + * (activeSubs decrements without any further write), confirming WS is leak-free? + * Or is it deferred until the next event (F-024-style)? + * Q4 SLOW / STALLED CONSUMER: open a WS sub and NEVER read (pause the socket), + * drive sustained large writes, sample per-worker heap/handles. Bounded + * (kernel backpressure) vs unbounded. Liveness: a healthy WS + REST client + * stays responsive throughout. + * + * TEARDOWN SAFETY: every WS is terminate()'d in finally/after. An undrained WS + * sub would block teardownHarper, so we never rely on the server to close it. + */ +import { suite, test, before, after } from 'node:test'; +import { ok, strictEqual } from 'node:assert/strict'; +import { resolve } from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; + +import WebSocket from 'ws'; +import request from 'supertest'; + +import { setupHarperWithFixture, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing'; +// @ts-expect-error utils/client.mjs has no type declarations; runtime resolves fine +import { createApiClient } from '../apiTests/utils/client.mjs'; + +const FIXTURE_PATH = resolve(import.meta.dirname, 'websocket-transport'); +const skipSuite = process.env.HARPER_RUNTIME === 'bun' || process.platform === 'win32'; + +interface WsEvent { + raw: string; + id?: string; + value?: number; + type?: string; + at: number; +} + +/** + * An open WS subscription. `events` is mutated live as frames arrive. `consume:false` pauses + * the socket (stalled slow consumer — never read). `terminate()` hard-kills the socket (the + * mid-stream client-disconnect simulation; ws.terminate() is an abrupt RST, no close frame). + */ +interface WsSub { + ws: WebSocket; + events: WsEvent[]; + opened: boolean; + terminate: () => void; +} + +function parseFrame(raw: string): WsEvent { + const ev: WsEvent = { raw, at: Date.now() }; + try { + const obj = JSON.parse(raw); + if (obj && typeof obj === 'object') { + if (obj.id != null) ev.id = String(obj.id); + ev.type = obj.type; + const inner = obj.value; + if (inner && typeof inner === 'object' && typeof inner.value === 'number') ev.value = inner.value; + else if (typeof inner === 'number') ev.value = inner; + else if (typeof obj.value === 'number') ev.value = obj.value; + } + } catch { + /* not json */ + } + return ev; +} + +function openWs( + wsBase: string, + path: string, + auth: string, + opts: { consume?: boolean } = {} +): Promise { + const consume = opts.consume ?? true; + const events: WsEvent[] = []; + const ws = new WebSocket(`${wsBase}${path}`, { + headers: { Authorization: auth, 'Content-Type': 'application/json' }, + // loopback; no TLS in this env but tolerate self-signed if it ever is. + rejectUnauthorized: false, + }); + const sub: WsSub = { + ws, + events, + opened: false, + terminate: () => { + try { + ws.terminate(); + } catch { + /* ignore */ + } + }, + }; + ws.on('message', (data: Buffer) => { + if (!consume) return; // stalled consumer: drop without parsing (socket still drains; see Q4 note) + events.push(parseFrame(data.toString())); + }); + ws.on('error', () => {}); + return new Promise((resolve, reject) => { + ws.once('open', () => { + sub.opened = true; + if (!consume) ws.pause(); // truly stall: stop reading the underlying socket + resolve(sub); + }); + ws.once('error', (err) => reject(err)); + // guard: never hang forever waiting for an upgrade + setTimeout(() => (sub.opened ? resolve(sub) : reject(new Error('ws open timeout'))), 8000); + }); +} + +async function waitFor( + predicate: () => boolean | Promise, + timeoutMs = 15_000, + intervalMs = 25 +): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await predicate()) return true; + await sleep(intervalMs); + } + return await predicate(); +} + +function wsSuite(label: string, threadCount: number | undefined) { + suite(`WebSocket transport [${label}]`, { skip: skipSuite }, (ctx: ContextWithHarper) => { + let client: ReturnType; + let restBase = ''; + let wsBase = ''; + let auth = ''; + const open = new Set(); + + const track = (s: WsSub) => { + open.add(s); + return s; + }; + const drop = (s: WsSub) => { + s.terminate(); + open.delete(s); + }; + + const restPut = (id: string, body: object) => request(restBase).put(`/Burst/${id}`).set(client.headers).send(body); + + // PERSISTENT latest-per-pid ledger. A single /Probe/ GET lands on ONE worker, + // and one probe() call's round-robin may not cover all workers — so we keep + // the latest reading per pid ACROSS calls. Monotonic counters (opened/closed) + // are always correct; gauges (activeSubs/heap/handles) reflect each worker's + // last-seen value. + const perPid = new Map(); + + /** Sum per-worker Probe ledgers (latest reading per pid, accumulated across calls). */ + async function probe(samples = 40): Promise<{ + activeSubs: number; + opened: number; + closed: number; + maxConcurrent: number; + teardownErrors: number; + activeHandles: number; + heapUsed: number; + pids: number; + }> { + for (let i = 0; i < samples; i++) { + try { + // Connection: close forces a FRESH TCP connection per GET so the OS (SO_REUSEPORT) + // spreads our probes across ALL worker threads — a keep-alive agent would pin every + // GET to one worker (workers=1) and hide the worker holding the WS sub. + const r = await client.reqRest('/Probe/').set('Connection', 'close').timeout(3000); + if (r.status === 200 && r.body?.pid != null) perPid.set(r.body.pid, r.body); + } catch { + /* ignore */ + } + } + const agg = { + activeSubs: 0, + opened: 0, + closed: 0, + maxConcurrent: 0, + teardownErrors: 0, + activeHandles: 0, + heapUsed: 0, + pids: perPid.size, + }; + for (const v of perPid.values()) { + agg.activeSubs += v.activeSubs ?? 0; + agg.opened += v.opened ?? 0; + agg.closed += v.closed ?? 0; + agg.maxConcurrent += v.maxConcurrent ?? 0; + agg.teardownErrors += v.teardownErrors ?? 0; + agg.activeHandles += v.activeHandles ?? 0; + agg.heapUsed += v.heapUsed ?? 0; + } + return agg; + } + + before(async () => { + const config: any = {}; + if (threadCount) config.threads = { count: threadCount }; + await setupHarperWithFixture(ctx, FIXTURE_PATH, { config, env: {} }); + client = createApiClient(ctx.harper); + restBase = client.restURL; + wsBase = restBase.replace(/^http/, 'ws'); + auth = client.headers.Authorization as string; + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + try { + const probeReq = await client.reqRest('/Burst/').timeout(3000); + if (probeReq.status !== 404) break; + } catch { + /* not ready */ + } + await sleep(250); + } + }); + + after(async () => { + for (const s of open) s.terminate(); + open.clear(); + await teardownHarper(ctx); + }); + + // Q1 — same-key burst: monotonic, last==final. + test('Q1: same-key burst delivers in monotonic order, ending on final', { timeout: 60_000 }, async () => { + const N = 30; + const id = `q1-${Date.now()}`; + const sub = track(await openWs(wsBase, `/Burst/${id}`, auth)); + ok(sub.opened, 'WS record subscription should open'); + await sleep(300); + for (let v = 1; v <= N; v++) await restPut(id, { id, value: v, tag: 'q1' }).expect(204); + + await waitFor(() => sub.events.filter((e) => e.id === id && e.value === N).length >= 1, 12_000); + const mine = sub.events.filter((e) => e.id === id && typeof e.value === 'number'); + const values = mine.map((e) => e.value as number); + const monotonic = values.every((v, i) => i === 0 || v >= values[i - 1]); + const last = values[values.length - 1]; + console.log( + `\n[ws-transport][${label}][Q1] same-key burst (sent ${N}):\n` + + ` frames for key = ${mine.length}\n` + + ` values = ${JSON.stringify(values.slice(0, 40))}\n` + + ` monotonic = ${monotonic}\n` + + ` last / final = ${last} / ${N}\n` + + ` classification = ${values.length >= N ? 'FULL STREAM' : values.length <= 3 ? 'COALESCED' : 'PARTIAL'}` + ); + ok(values.length >= 1, 'WS subscriber should receive at least one frame for our key'); + ok(monotonic, `WS values must be monotonic (no reorder); got ${JSON.stringify(values)}`); + ok(values.length <= N, `must not deliver MORE than sent (no dup); got ${values.length}`); + strictEqual(last, N, `last delivered value must be the final write ${N}, got ${last}`); + drop(sub); + }); + + // Q2 — many-key burst: exactly-once per key, no drop/dup. + test('Q2: many-key burst delivers each key exactly once (no drop/dup)', { timeout: 60_000 }, async () => { + const K = 50; + const run = Date.now(); + const sub = track(await openWs(wsBase, `/Burst/`, auth)); // collection sub + ok(sub.opened, 'WS collection subscription should open'); + await sleep(400); + const keys: string[] = []; + for (let i = 0; i < K; i++) { + const id = `q2-${run}-${i}`; + keys.push(id); + await restPut(id, { id, value: i, tag: 'q2' }).expect(204); + } + const keySet = new Set(keys); + await waitFor(() => { + const seen = new Set(sub.events.filter((e) => e.id && keySet.has(e.id)).map((e) => e.id)); + return seen.size >= K; + }, 15_000); + + const mine = sub.events.filter((e) => e.id && keySet.has(e.id)); + const counts = new Map(); + for (const e of mine) counts.set(e.id!, (counts.get(e.id!) ?? 0) + 1); + const delivered = counts.size; + const dups = [...counts.entries()].filter(([, c]) => c > 1); + const missing = keys.filter((k) => !counts.has(k)); + console.log( + `\n[ws-transport][${label}][Q2] many-key burst (sent ${K} distinct keys):\n` + + ` distinct keys delivered = ${delivered}/${K}\n` + + ` duplicated keys = ${dups.length} ${dups.length ? JSON.stringify(dups.slice(0, 5)) : ''}\n` + + ` missing keys = ${missing.length} ${missing.length ? JSON.stringify(missing.slice(0, 5)) : ''}\n` + + ` classification = ${delivered === K && dups.length === 0 ? 'EXACTLY-ONCE' : delivered < K ? 'DROP' : 'DUP'}` + ); + strictEqual(delivered, K, `every key must be delivered; missing=${JSON.stringify(missing.slice(0, 10))}`); + strictEqual(dups.length, 0, `no key may be duplicated; dups=${JSON.stringify(dups.slice(0, 10))}`); + drop(sub); + }); + + // Q3 — DISCONNECT TEARDOWN. Sample subscription-layer activeSubs + // BEFORE / WHILE-CONNECTED / AFTER-DISCONNECT with NO intervening write. + test('Q3: WS disconnect tears down the subscription PROMPTLY (no F-024 deferral)', { timeout: 90_000 }, async () => { + const base = await probe(); + const id = `q3-${Date.now()}`; + // connect to the instrumented Live resource so we observe the REAL subscription lifecycle. + const sub = track(await openWs(wsBase, `/Live/${id}`, auth)); + ok(sub.opened, 'Live WS subscription should open'); + // receive at least one event to prove the sub is live, then we will NOT write again. + await sleep(300); + await restPut(id, { id, value: 1, tag: 'q3' }).expect(204); + await waitFor(() => sub.events.length >= 1, 6000); // confirm the sub is actually live + // Poll until the Probe ledger reflects the new sub. In multi-worker the WS lands on ONE + // worker and our round-robin /Probe/ GETs may not sample that pid until a later pass — + // so wait for the monotonic `opened` to register rather than snapshotting once (which + // races the worker that holds the sub). + let whileConnected = await probe(); + await waitFor(async () => { + whileConnected = await probe(12); + return whileConnected.opened - base.opened >= 1 && whileConnected.activeSubs - base.activeSubs >= 1; + }, 10_000, 150); + const openedDelta = whileConnected.opened - base.opened; + const activeWhile = whileConnected.activeSubs - base.activeSubs; + ok(openedDelta >= 1, `Live.connect should register a subscription; openedDelta=${openedDelta}`); + ok(activeWhile >= 1, `activeSubs should rise by >=1 while connected; got ${activeWhile}`); + + // HARD disconnect mid-stream. CRITICAL: do NOT write anything after this — a deferred + // teardown would only fire on the NEXT event, so a clean decrement here with + // zero further writes proves PROMPT teardown. + const tDisc = Date.now(); + drop(sub); + + // Poll for teardown to register. The DEFINITIVE leak-free signal is the activeSubs + // gauge returning to baseline (opened-closed back to 0 on the holding worker) AND the + // monotonic `closed` counter catching up. + let tornAt = 0; + await waitFor(async () => { + const p = await probe(60); + if (p.activeSubs - base.activeSubs <= 0 && p.closed - base.closed >= openedDelta) { + tornAt = Date.now(); + return true; + } + return false; + }, 15_000, 150); + const after = await probe(60); + const closedDelta = after.closed - base.closed; + const activeAfter = after.activeSubs - base.activeSubs; + const teardownMs = tornAt ? tornAt - tDisc : -1; + console.log( + `\n[ws-transport][${label}][Q3] disconnect teardown (NO write after disconnect):\n` + + ` baseline activeSubs = ${base.activeSubs} (workers=${base.pids})\n` + + ` while-connected openedDelta = ${openedDelta}, activeDelta = ${activeWhile}\n` + + ` after-disconnect closedDelta = ${closedDelta}, activeDelta = ${activeAfter}\n` + + ` teardown latency = ${teardownMs}ms (no intervening event)\n` + + ` teardownErrors = ${after.teardownErrors - base.teardownErrors}\n` + + ` => ${ + activeAfter <= 0 && closedDelta >= openedDelta + ? 'PROMPT teardown (WS leak-free; iterator.return fired on disconnect)' + : 'DEFERRED / LEAK (subscription survived disconnect — F-024-style)' + }` + ); + strictEqual(after.teardownErrors - base.teardownErrors, 0, 'no teardown errors'); + ok( + closedDelta >= openedDelta, + `WS disconnect must tear down the sub WITHOUT a following event; closed=${closedDelta} opened=${openedDelta}` + ); + ok(activeAfter <= 0, `activeSubs must return to baseline after disconnect; residual=${activeAfter}`); + }); + + // Q3b — churn: open/receive/disconnect many WS subs; activeSubs must return to baseline. + test('Q3b: WS subscribe churn returns activeSubs to baseline (no leak)', { timeout: 120_000 }, async () => { + const base = await probe(); + const ROUNDS = 25; + for (let i = 0; i < ROUNDS; i++) { + const id = `q3b-${Date.now()}-${i}`; + const s = await openWs(wsBase, `/Live/${id}`, auth); + await restPut(id, { id, value: i, tag: 'q3b' }).expect(204); + await sleep(40); + s.terminate(); // disconnect; no write after + } + await waitFor(async () => { + const p = await probe(8); + return p.activeSubs - base.activeSubs <= 0; + }, 20_000, 200); + const after = await probe(); + const residual = after.activeSubs - base.activeSubs; + console.log( + `\n[ws-transport][${label}][Q3b] churn x${ROUNDS}:\n` + + ` opened delta = ${after.opened - base.opened}, closed delta = ${after.closed - base.closed}\n` + + ` residual activeSubs = ${residual}, maxConcurrent observed = ${after.maxConcurrent}\n` + + ` => ${residual <= 1 ? 'CLEAN (no listener/handle leak across churn)' : 'LEAK (subs survive disconnect)'}` + ); + ok(residual <= 1, `activeSubs must settle to baseline after churn; residual=${residual}`); + }); + + // Q4 — stalled WS consumer: never read, drive large writes, sample heap/handles + liveness. + test('Q4: stalled WS consumer is bounded; server stays responsive', { timeout: 90_000 }, async () => { + const before = await probe(); + const id = `q4-${Date.now()}`; + const big = 'x'.repeat(4096); + // Open a stalled collection sub: ws.pause() so we never read the socket -> server's + // send loop should hit writableNeedDrain and await 'drain' (kernel backpressure), + // not buffer unboundedly in-process. + const stalled = track(await openWs(wsBase, `/Burst/`, auth, { consume: false })); + ok(stalled.opened, 'stalled WS sub should open'); + await sleep(300); + const N = 200; + let writeErrors = 0; + const t0 = Date.now(); + for (let v = 1; v <= N; v++) { + try { + await restPut(id, { id, value: v, tag: big }).timeout(5000).expect(204); + } catch (e) { + writeErrors++; + if (writeErrors <= 2) console.log(`[ws-transport][${label}][Q4] write ${v} err: ${(e as Error).message}`); + } + } + const writeMs = Date.now() - t0; + + // healthy clients stay responsive: a fresh REST GET and a fresh (draining) WS sub. + const hT0 = Date.now(); + let healthStatus = 0; + let healthValue: unknown; + try { + const h = await client.reqRest(`/Burst/${id}`).timeout(5000); + healthStatus = h.status; + healthValue = h.body?.value; + } catch (e) { + console.log(`[ws-transport][${label}][Q4] concurrent GET err: ${(e as Error).message}`); + } + const healthMs = Date.now() - hT0; + + const after = await probe(); + const heapGrowth = after.heapUsed - before.heapUsed; + const handleGrowth = after.activeHandles - before.activeHandles; + // Bounded-buffering signal: the send loop awaits the socket 'drain' + // when writableNeedDrain, so a stalled (paused) consumer applies KERNEL TCP backpressure + // rather than buffering frames unboundedly in-process. The definitive evidence is (a) + // activeHandles does NOT grow per-write and (b) heap does not track total write volume. + const boundedHeap = heapGrowth < 64 * 1e6; // generous ceiling; normal GC churn lives well under this + const boundedHandles = handleGrowth <= 2; // just the stalled socket (+ maybe the fresh GET) + console.log( + `\n[ws-transport][${label}][Q4] stalled-consumer summary:\n` + + ` ${N} large (~4KB) PUTs in ${writeMs}ms, writeErrors=${writeErrors}\n` + + ` concurrent GET status=${healthStatus} latency=${healthMs}ms value=${healthValue}\n` + + ` worker heapUsed growth (summed) = ${(heapGrowth / 1e6).toFixed(1)}MB\n` + + ` activeHandles before/after = ${before.activeHandles}/${after.activeHandles} (growth ${handleGrowth})\n` + + ` => server ${healthMs < 4000 && healthStatus === 200 ? 'RESPONSIVE (not wedged)' : 'DEGRADED / wedged'}; ` + + `buffering ${boundedHeap && boundedHandles ? 'BOUNDED (kernel backpressure; no handle/heap blowup)' : 'SUSPECT-UNBOUNDED'}` + ); + strictEqual(writeErrors, 0, 'writes must not fail because one WS consumer is stalled'); + strictEqual(healthStatus, 200, 'server must serve normal requests with a stalled WS reader'); + ok(healthMs < 4000, `concurrent request must stay responsive; took ${healthMs}ms`); + strictEqual(healthValue, N, `final persisted value should be ${N}`); + drop(stalled); + }); + }); +} + +wsSuite('single-worker', undefined); +wsSuite('multi-worker', 4); diff --git a/integrationTests/server/websocket-transport/config.yaml b/integrationTests/server/websocket-transport/config.yaml new file mode 100644 index 0000000000..efffc0833f --- /dev/null +++ b/integrationTests/server/websocket-transport/config.yaml @@ -0,0 +1,5 @@ +graphqlSchema: + files: '*.graphql' +jsResource: + files: resources.js +rest: true diff --git a/integrationTests/server/websocket-transport/resources.js b/integrationTests/server/websocket-transport/resources.js new file mode 100644 index 0000000000..cf68ffaf61 --- /dev/null +++ b/integrationTests/server/websocket-transport/resources.js @@ -0,0 +1,93 @@ +// QA-159 — WebSocket subscription transport instrumentation. +// +// GOAL: observe, per worker process, the EXACT subscription-layer teardown timing for the +// WebSocket subscribe path, to test the F-024 contrast (WS = prompt iterator.return() on +// disconnect, vs SSE deferral). +// +// MECHANISM (verified against harper 7aaa5a152): +// server/REST.ts `scope.server.ws(...)` handler on a WS 'close' event calls +// `iterator.return()` (REST.ts:341) — that is the prompt teardown wiring QA-127 noted. +// `iterator.return()` on a core Subscription (resources/transactionBroadcast.ts) emits +// 'close', which runs Subscription.end() and decrements databaseSubscriptions.activeCount. +// +// The core activeCount registry (allSubscriptions) is module-private, so instead of +// reaching into it we wrap the REAL subscription: `Live` extends the Burst table and its +// connect() delegates to the REAL Table.subscribe (super.subscribe -> core Subscription), +// then we attach our own once('close') to that very subscription object. Our counter +// therefore decrements at the SAME instant core's activeCount does — i.e. it is a faithful +// proxy for the subscription-layer teardown, with NO core changes. +// +// All counters live on a process-global so single-worker and multi-worker (threads.count:N) +// runs are both observable; Probe returns this worker's slice + pid so the test can sum +// across workers and detect per-worker leaks. + +import { threadId } from 'node:worker_threads'; + +const G = (globalThis.__QA159__ ??= { + // Harper runs HTTP workers as worker_threads, which SHARE process.pid — so we key per-worker + // state by threadId (unique per worker thread) for correct cross-worker aggregation. + pid: `${process.pid}:t${threadId}`, + opened: 0, // # of Live WS subscriptions that attached (subscribe returned) + closed: 0, // # whose 'close' teardown fired (iterator.return on disconnect / end) + teardownErrors: 0, + maxConcurrent: 0, // high-water mark of (opened-closed) + lastCloseAt: 0, // wall-clock of most recent teardown (latency probing) + lastOpenAt: 0, +}); + +// Live: custom resource over the Burst store. connect() is what the WS path invokes; we route +// to the REAL Table.subscribe so the returned iterable is the genuine core Subscription, then +// instrument its lifecycle. Filtering/delivery semantics are unchanged from a plain table sub. +export class Live extends tables.Burst { + connect(target, incomingMessages, request) { + // Mirror the default Resource.connect query selection (resources/Resource.ts:405): for a + // loadAsInstance resource the subscribe query is `incomingMessages` unless that's not a + // subscription target; for our record/collection WS path `target` carries the id. Pass + // `target` (the RequestTarget) — Table.subscribe reads its id via requestTargetToId. + // super.subscribe is resources/Table.ts subscribe() -> core Subscription (IterableEventQueue). + const subPromise = Promise.resolve(super.subscribe(target)); + return subPromise.then((sub) => { + G.opened++; + G.lastOpenAt = Date.now(); + const live = G.opened - G.closed; + if (live > G.maxConcurrent) G.maxConcurrent = live; + let torn = false; + const onClose = () => { + if (torn) return; + torn = true; + try { + G.closed++; + G.lastCloseAt = Date.now(); + } catch { + G.teardownErrors++; + } + }; + // 'close' is emitted by the core Subscription's iterator.return()/throw() — exactly + // the REST.ts WS 'close' -> iterator.return() teardown (and end-of-stream). This is + // the F-024 detection hook: it fires PROMPTLY for WS, deferred for SSE. + if (typeof sub.once === 'function') sub.once('close', onClose); + return sub; + }); + } +} + +// Probe: read this worker's WS-subscription lifecycle ledger. +export class Probe extends Resource { + static loadAsInstance = false; + async get() { + return { + pid: G.pid, + opened: G.opened, + closed: G.closed, + activeSubs: G.opened - G.closed, // should settle to (currently-connected WS subs) + maxConcurrent: G.maxConcurrent, + teardownErrors: G.teardownErrors, + lastOpenAt: G.lastOpenAt, + lastCloseAt: G.lastCloseAt, + // out-of-band transport-layer leak signal: open socket handles in this worker. + activeHandles: typeof process._getActiveHandles === 'function' ? process._getActiveHandles().length : -1, + rss: process.memoryUsage().rss, + heapUsed: process.memoryUsage().heapUsed, + }; + } +} diff --git a/integrationTests/server/websocket-transport/schema.graphql b/integrationTests/server/websocket-transport/schema.graphql new file mode 100644 index 0000000000..65cbcd04ef --- /dev/null +++ b/integrationTests/server/websocket-transport/schema.graphql @@ -0,0 +1,11 @@ +# QA-159 — WebSocket subscription transport: delivery, disconnect teardown, backpressure. +# +# Burst: plain exported table. A WS client connects to /Burst/ or /Burst/ (collection), +# which Harper maps to the WS subscribe path (server/REST.ts: scope.server.ws -> +# resources.getMatch(url,'ws') -> resource.connect -> Table.subscribe). Each write surfaces +# as one change frame on the socket. We mutate via REST PUT and observe WS delivery/ordering. +type Burst @table @export { + id: ID @primaryKey + value: Int + tag: String +}