From d40648d684be0336731cbccfa9e38ef296cb782b Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 24 Jun 2026 06:50:42 -0600 Subject: [PATCH 1/3] test(northwind): poll for async auth-cache propagation after alter_role Three stale-auth-cache races caused ~50% CI failure rate with varying line numbers. alter_role returns HTTP 200 after the DB write but before the async ITC permission-cache invalidation reaches all worker threads, so the next request can hit a worker with stale permissions. Affected spots: - CSV upsert with newly granted attribute permissions - Two cross-schema SQL JOINs that require the updated role to be visible Each affected request is now wrapped in the existing waitFor() poll helper, retrying until the permission change is visibly effective. No product code changed. Test-only stabilization. Verified locally 5/5 runs (560 pass). Co-Authored-By: Claude Opus 4.8 (1M context) --- integrationTests/apiTests/northwind.test.mjs | 110 ++++++++++--------- 1 file changed, 61 insertions(+), 49 deletions(-) diff --git a/integrationTests/apiTests/northwind.test.mjs b/integrationTests/apiTests/northwind.test.mjs index ec61d41687..f8a9138791 100644 --- a/integrationTests/apiTests/northwind.test.mjs +++ b/integrationTests/apiTests/northwind.test.mjs @@ -8747,6 +8747,19 @@ suite('Northwind operations', { skip: skipSuite }, (ctx) => { }); test('CSV Data Load upsert to table w/ full perms', { skip: bunSkipNewAttrCsvLoad }, async () => { + // 'Alter non-SU bulk_load_role' above grants unrestricted attribute access on + // northnwd.suppliers, but alter_role propagates to worker auth-cache via async + // ITC — the next request can land on a worker that hasn't cleared its cache yet. + // Poll a read of a previously-restricted column (contactname) as bulk_load_user + // until the new permissions are visible, then proceed with the CSV load (#1222). + await waitFor( + () => + client.reqAs(headersBulkLoadUser).send({ + operation: 'sql', + sql: 'SELECT contactname FROM northnwd.suppliers LIMIT 1', + }), + { until: (r) => r.status === 200, timeoutSeconds: 10 } + ); await csvDataLoad( headersBulkLoadUser, 'upsert', @@ -11336,29 +11349,29 @@ suite('Northwind operations', { skip: skipSuite }, (ctx) => { }); test('Select two table CROSS SCHEMA JOIN as test_user', async () => { - await client - .reqAs(headersTestUser) - .send({ - operation: 'sql', - sql: 'SELECT d.id, d.dog_name, d.age, d.adorable, o.id, o.name FROM dev.dog AS d INNER JOIN other.owner AS o ON d.owner_id = o.id', - }) - .expect((r) => { - assert.equal(r.body.length, 8, r.text); - const expected_attributes = ['id', 'dog_name', 'age', 'adorable', 'id1', 'name']; - //Important to test that only the id (returned as id1) and name attributes come back for 'other.owner' - // since user only has access to those two attributes - r.body.forEach((row) => { - expected_attributes.forEach((attr) => { - assert.ok(row.hasOwnProperty(attr), r.text); - }); - }); - }) - .expect((r) => { - assert.equal(r.body[1].name, 'David', r.text); - assert.equal(r.body[3].id1, 1, r.text); - assert.equal(r.body[4].id1, 2, r.text); - }) - .expect(200); + // 'SQL ALTER non SU role' above adds read access for other.owner attributes, + // but the permission propagates to worker auth-cache via async ITC — poll + // until the join returns the expected row count before asserting structure (#1222). + const r = await waitFor( + () => + client.reqAs(headersTestUser).send({ + operation: 'sql', + sql: 'SELECT d.id, d.dog_name, d.age, d.adorable, o.id, o.name FROM dev.dog AS d INNER JOIN other.owner AS o ON d.owner_id = o.id', + }), + { until: (res) => res.status === 200 && Array.isArray(res.body) && res.body.length === 8, timeoutSeconds: 10 } + ); + assert.equal(r.body.length, 8, r.text); + const expected_attributes = ['id', 'dog_name', 'age', 'adorable', 'id1', 'name']; + //Important to test that only the id (returned as id1) and name attributes come back for 'other.owner' + // since user only has access to those two attributes + r.body.forEach((row) => { + expected_attributes.forEach((attr) => { + assert.ok(row.hasOwnProperty(attr), r.text); + }); + }); + assert.equal(r.body[1].name, 'David', r.text); + assert.equal(r.body[3].id1, 1, r.text); + assert.equal(r.body[4].id1, 2, r.text); }); test('Select * w/ two table CROSS SCHEMA JOIN as test_user', async () => { @@ -11515,32 +11528,31 @@ suite('Northwind operations', { skip: skipSuite }, (ctx) => { }); test('Select with ALL RESTRICTED complex CROSS 3 SCHEMA JOINS as test_user', async () => { - await client - .reqAs(headersTestUser) - .send({ - operation: 'sql', - sql: 'SELECT d.age AS dog_age, AVG(d.weight_lbs) AS dog_weight, o.name AS owner_name, b.name, b.country FROM dev.dog AS d INNER JOIN other.owner AS o ON d.owner_id = o.id INNER JOIN another.breed AS b ON d.breed_id = b.id GROUP BY o.name, b.name, d.age ORDER BY b.name', - }) - .expect((r) => { - assert.equal( - r.body.error, - 'This operation is not authorized due to role restrictions and/or invalid database items', - r.text - ); - assert.equal(r.body.unauthorized_access.length, 1, r.text); - assert.equal(r.body.unauthorized_access[0].required_table_permissions.length, 1, r.text); - assert.equal(r.body.unauthorized_access[0].required_table_permissions[0], 'read', r.text); - assert.equal(r.body.unauthorized_access[0].schema, 'dev', r.text); - assert.equal(r.body.unauthorized_access[0].table, 'dog', r.text); - assert.equal(r.body.invalid_schema_items.length, 3, r.text); - assert.ok(r.body.invalid_schema_items.includes("Attribute 'id' does not exist on 'other.owner'"), r.text); - assert.ok(r.body.invalid_schema_items.includes("Attribute 'name' does not exist on 'other.owner'"), r.text); - assert.ok( - r.body.invalid_schema_items.includes("Attribute 'country' does not exist on 'another.breed'"), - r.text - ); - }) - .expect(403); + // 'SQL ALTER non SU role with multi table join restrictions' above restricts + // dev.dog read access, but propagates via async ITC — poll until the + // restriction is enforced before asserting the 403 response body (#1222). + const r = await waitFor( + () => + client.reqAs(headersTestUser).send({ + operation: 'sql', + sql: 'SELECT d.age AS dog_age, AVG(d.weight_lbs) AS dog_weight, o.name AS owner_name, b.name, b.country FROM dev.dog AS d INNER JOIN other.owner AS o ON d.owner_id = o.id INNER JOIN another.breed AS b ON d.breed_id = b.id GROUP BY o.name, b.name, d.age ORDER BY b.name', + }), + { until: (res) => res.status === 403, timeoutSeconds: 10 } + ); + assert.equal( + r.body.error, + 'This operation is not authorized due to role restrictions and/or invalid database items', + r.text + ); + assert.equal(r.body.unauthorized_access.length, 1, r.text); + assert.equal(r.body.unauthorized_access[0].required_table_permissions.length, 1, r.text); + assert.equal(r.body.unauthorized_access[0].required_table_permissions[0], 'read', r.text); + assert.equal(r.body.unauthorized_access[0].schema, 'dev', r.text); + assert.equal(r.body.unauthorized_access[0].table, 'dog', r.text); + assert.equal(r.body.invalid_schema_items.length, 3, r.text); + assert.ok(r.body.invalid_schema_items.includes("Attribute 'id' does not exist on 'other.owner'"), r.text); + assert.ok(r.body.invalid_schema_items.includes("Attribute 'name' does not exist on 'other.owner'"), r.text); + assert.ok(r.body.invalid_schema_items.includes("Attribute 'country' does not exist on 'another.breed'"), r.text); }); test('SQL drop test user', async () => { From 23768597b2a7577eca865991a689825fbe9ee0aa Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 24 Jun 2026 08:16:16 -0600 Subject: [PATCH 2/3] test(northwind): assert status explicitly after waitFor so a timeout fails clearly Addresses Gemini review: waitFor returns the last value on timeout; assert the expected status (200/403) before body assertions so a propagation timeout surfaces a clear failure instead of a confusing TypeError or silent pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- integrationTests/apiTests/northwind.test.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/integrationTests/apiTests/northwind.test.mjs b/integrationTests/apiTests/northwind.test.mjs index f8a9138791..bcb4d5b245 100644 --- a/integrationTests/apiTests/northwind.test.mjs +++ b/integrationTests/apiTests/northwind.test.mjs @@ -8752,7 +8752,7 @@ suite('Northwind operations', { skip: skipSuite }, (ctx) => { // ITC — the next request can land on a worker that hasn't cleared its cache yet. // Poll a read of a previously-restricted column (contactname) as bulk_load_user // until the new permissions are visible, then proceed with the CSV load (#1222). - await waitFor( + const propagated = await waitFor( () => client.reqAs(headersBulkLoadUser).send({ operation: 'sql', @@ -8760,6 +8760,7 @@ suite('Northwind operations', { skip: skipSuite }, (ctx) => { }), { until: (r) => r.status === 200, timeoutSeconds: 10 } ); + assert.equal(propagated.status, 200, propagated.text); await csvDataLoad( headersBulkLoadUser, 'upsert', @@ -11360,6 +11361,7 @@ suite('Northwind operations', { skip: skipSuite }, (ctx) => { }), { until: (res) => res.status === 200 && Array.isArray(res.body) && res.body.length === 8, timeoutSeconds: 10 } ); + assert.equal(r.status, 200, r.text); assert.equal(r.body.length, 8, r.text); const expected_attributes = ['id', 'dog_name', 'age', 'adorable', 'id1', 'name']; //Important to test that only the id (returned as id1) and name attributes come back for 'other.owner' @@ -11539,6 +11541,7 @@ suite('Northwind operations', { skip: skipSuite }, (ctx) => { }), { until: (res) => res.status === 403, timeoutSeconds: 10 } ); + assert.equal(r.status, 403, r.text); assert.equal( r.body.error, 'This operation is not authorized due to role restrictions and/or invalid database items', From c0b256bf17f54213084587e2dcd586e84d363e78 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 25 Jun 2026 18:39:28 -0600 Subject: [PATCH 3/3] test(northwind): tighten ALL-RESTRICTED join poll to full expected body MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous poll predicate (status === 403) was satisfied at an intermediate/stale role-propagation state — a 403 with empty unauthorized_access and only one invalid_schema_item — so the test still flaked after the waitFor was added (CI on the fix commit still failed: "0 !== 1" on unauthorized_access.length). alter_role propagates to worker auth-caches via async ITC, and a 403 alone does not prove the *current* restriction is fully visible. Poll until the full expected shape is present (1 unauthorized table = dog AND all 3 invalid_schema_items) before asserting. Verified locally 3/3. Co-Authored-By: Claude Opus 4.8 (1M context) --- integrationTests/apiTests/northwind.test.mjs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/integrationTests/apiTests/northwind.test.mjs b/integrationTests/apiTests/northwind.test.mjs index bcb4d5b245..ba1be0d491 100644 --- a/integrationTests/apiTests/northwind.test.mjs +++ b/integrationTests/apiTests/northwind.test.mjs @@ -11533,13 +11533,24 @@ suite('Northwind operations', { skip: skipSuite }, (ctx) => { // 'SQL ALTER non SU role with multi table join restrictions' above restricts // dev.dog read access, but propagates via async ITC — poll until the // restriction is enforced before asserting the 403 response body (#1222). + // A bare `status === 403` is NOT sufficient: a stale/partially-propagated + // role can also produce a 403 with a different body (e.g. empty + // unauthorized_access and only one invalid_schema_item), so poll until the + // full expected shape is visible — 1 unauthorized table + all 3 invalid items. const r = await waitFor( () => client.reqAs(headersTestUser).send({ operation: 'sql', sql: 'SELECT d.age AS dog_age, AVG(d.weight_lbs) AS dog_weight, o.name AS owner_name, b.name, b.country FROM dev.dog AS d INNER JOIN other.owner AS o ON d.owner_id = o.id INNER JOIN another.breed AS b ON d.breed_id = b.id GROUP BY o.name, b.name, d.age ORDER BY b.name', }), - { until: (res) => res.status === 403, timeoutSeconds: 10 } + { + until: (res) => + res.status === 403 && + res.body?.unauthorized_access?.length === 1 && + res.body.unauthorized_access[0]?.table === 'dog' && + res.body?.invalid_schema_items?.length === 3, + timeoutSeconds: 10, + } ); assert.equal(r.status, 403, r.text); assert.equal(