Skip to content

Commit 097e1bc

Browse files
authored
fix(ci): prevent flaky smoke test timeouts from failing the build (#8837)
Signed-off-by: Michael Neale <michael.neale@gmail.com>
1 parent e790d37 commit 097e1bc

1 file changed

Lines changed: 36 additions & 11 deletions

File tree

ui/desktop/tests/integration/test_providers_lib.ts

Lines changed: 36 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -313,13 +313,19 @@ function registerTests(label: string, cases: TestCase[], fn: ProviderTestFn): vo
313313
}
314314

315315
if (flaky.length > 0) {
316-
test.each(flaky)(`${label} — $provider / $model (flaky)`, async (tc) => {
317-
try {
318-
await fn(tc);
319-
} catch (err) {
320-
console.warn(`Flaky test ${tc.provider}/${tc.model} failed (allowed): ${err}`);
321-
}
322-
});
316+
// Use a longer vitest timeout (90s) so the internal runGoose timeout (55s)
317+
// fires first — that rejection is catchable and the test passes as "allowed".
318+
test.each(flaky)(
319+
`${label} — $provider / $model (flaky)`,
320+
async (tc) => {
321+
try {
322+
await fn(tc);
323+
} catch (err) {
324+
console.warn(`Flaky test ${tc.provider}/${tc.model} failed (allowed): ${err}`);
325+
}
326+
},
327+
90_000
328+
);
323329
}
324330

325331
if (skipped.length > 0) {
@@ -357,9 +363,10 @@ export function runGoose(
357363
cwd: string,
358364
prompt: string,
359365
builtins: string,
360-
env: Record<string, string>
366+
env: Record<string, string>,
367+
timeoutMs: number = 55_000
361368
): Promise<string> {
362-
return new Promise((resolve) => {
369+
return new Promise((resolve, reject) => {
363370
const child: ChildProcess = spawn(
364371
gooseBin,
365372
['run', '--text', prompt, '--with-builtin', builtins],
@@ -371,6 +378,16 @@ export function runGoose(
371378
);
372379

373380
let output = '';
381+
let settled = false;
382+
383+
const timer = setTimeout(() => {
384+
if (!settled) {
385+
settled = true;
386+
child.kill('SIGKILL');
387+
reject(new Error(`goose timed out after ${timeoutMs}ms\n\nPartial output:\n${output}`));
388+
}
389+
}, timeoutMs);
390+
374391
child.stdout?.on('data', (d) => {
375392
output += String(d);
376393
});
@@ -379,11 +396,19 @@ export function runGoose(
379396
});
380397

381398
child.on('close', () => {
382-
resolve(output);
399+
if (!settled) {
400+
settled = true;
401+
clearTimeout(timer);
402+
resolve(output);
403+
}
383404
});
384405

385406
child.on('error', (err) => {
386-
resolve(`spawn error: ${err.message}`);
407+
if (!settled) {
408+
settled = true;
409+
clearTimeout(timer);
410+
resolve(`spawn error: ${err.message}`);
411+
}
387412
});
388413
});
389414
}

0 commit comments

Comments
 (0)